Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): preserve redirect response provenance
ldm0c1412b76b98a
diff --git a/moli-core/tests/scripts/integrity.rs b/moli-core/tests/scripts/integrity.rsindex 9b6892259..dd03a94f8 100644--- a/moli-core/tests/scripts/integrity.rs+++ b/moli-core/tests/scripts/integrity.rs@@ -153,6 +153,16 @@ fn fixture_response(path: &str, origin: &str, cross: &str) -> (&'static str, Strevent.respondWith(Response.redirect('{cross}/script.js'));else if (path === '/sw-roundtrip.js')event.respondWith(Response.redirect('{cross}/redirect-home.js'));+ else if (path === '/sw-cors-redirect.js')+ event.respondWith(Response.redirect('{origin}/cors.js'));+ else if (path === '/sw-cors-redirect-null.js')+ event.respondWith(Response.redirect('{origin}/cors-null.js'));+ else if (path === '/sw-cors-redirect-origin.js')+ event.respondWith(Response.redirect('{origin}/cors-origin.js'));+ else if (path === '/sw-cors-redirect-no-acao.js')+ event.respondWith(Response.redirect('{origin}/script.js'));+ else if (path === '/sw-cors-redirect-unapproved-hop.js')+ event.respondWith(Response.redirect('{cross}/unapproved-redirect.js'));}});"#,script = serde_json::to_string(SCRIPT).unwrap()@@ -301,3 +311,29 @@ async fn service_worker_script_integrity_preserves_response_filter() -> Result<(assert_integrity_results(result, &cases);Ok(())}++#[tokio::test(flavor = "multi_thread")]+async fn service_worker_cross_origin_redirect_integrity_checks_network_responses() -> Result<()> {+ let servers = IntegrityServers::spawn().await?;+ let browser = Browser::new(AppConfig::default())?;+ let mut page = browser+ .fetch(&format!("{}/page.html", servers.origin))+ .await?;+ let cross = &servers.cross_origin;+ // These requests start cross-origin, so the worker's synthetic response+ // must be distinguished from a cross-origin HTTP redirect without ACAO.+ let cases = serde_json::json!([+ {"name": "classic-cors", "src": format!("{cross}/sw-cors-redirect.js"), "crossOrigin": "anonymous", "integrity": INTEGRITY, "expected": "load"},+ {"name": "module-cors", "src": format!("{cross}/sw-cors-redirect.js"), "type": "module", "integrity": INTEGRITY, "expected": "load"},+ {"name": "module-null-origin", "src": format!("{cross}/sw-cors-redirect-null.js"), "type": "module", "integrity": INTEGRITY, "expected": "load"},+ {"name": "module-old-origin", "src": format!("{cross}/sw-cors-redirect-origin.js"), "type": "module", "integrity": INTEGRITY, "expected": "error"},+ {"name": "module-no-acao", "src": format!("{cross}/sw-cors-redirect-no-acao.js"), "type": "module", "integrity": INTEGRITY, "expected": "error"},+ {"name": "module-unapproved-hop", "src": format!("{cross}/sw-cors-redirect-unapproved-hop.js"), "type": "module", "integrity": INTEGRITY, "expected": "error"},+ {"name": "classic-opaque", "src": format!("{cross}/sw-cors-redirect.js"), "integrity": INTEGRITY, "expected": "error"}+ ]);+ let result = page+ .evaluate_runtime_expression_with_await_async(&dynamic_probe(&cases, true), true)+ .await?;+ assert_integrity_results(result, &cases);+ Ok(())+}diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rsindex 2c70179eb..e9bfc3850 100644--- a/moli-fetch/src/blocking/mod.rs+++ b/moli-fetch/src/blocking/mod.rs@@ -1177,6 +1177,7 @@ mod tests {fn redirect(from_url: &Url, to_url: &Url) -> RedirectInfo {RedirectInfo {+ source: crate::RedirectSource::Network,from_url: from_url.clone(),to_url: to_url.clone(),status: 302,diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rsindex ccfefaabf..6b5ccf33e 100644--- a/moli-fetch/src/lib.rs+++ b/moli-fetch/src/lib.rs@@ -65,7 +65,7 @@ pub use request::{pub use request_policy::{is_bad_port, should_request_be_blocked_due_to_bad_port};pub use response::{NegotiatedHttpVersion, NetworkRequestExtraInfo, NetworkResponseExtraInfo, RawResponse,- RedirectInfo, Response, ResponseBody, ResponseHead,+ RedirectInfo, RedirectSource, Response, ResponseBody, ResponseHead,};pub use runtime::PendingStreamingRawResponse;pub use runtime::{diff --git a/moli-fetch/src/response.rs b/moli-fetch/src/response.rsindex 395699979..3613e5782 100644--- a/moli-fetch/src/response.rs+++ b/moli-fetch/src/response.rs@@ -601,8 +601,21 @@ pub struct NetworkResponseExtraInfo {pub cookie_set_reports: Vec<StoredCookieSetReport>,}+/// The producer of a redirect response, independent of cache and network+/// observation metadata. Only network responses are subject to a CORS check.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum RedirectSource {+ /// An HTTP response, including one reused from the HTTP cache.+ Network,+ /// A response returned by a service worker's FetchEvent.respondWith().+ ServiceWorker,+ /// A browser-generated redirect such as an HTTPS upgrade or client hint retry.+ Internal,+}+#[derive(Debug, Clone, PartialEq, Eq)]pub struct RedirectInfo {+ pub source: RedirectSource,pub from_url: Url,pub to_url: Url,pub status: u16,diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rsindex 27d74bf92..c3e17459a 100644--- a/moli-fetch/src/runtime.rs+++ b/moli-fetch/src/runtime.rs@@ -1532,6 +1532,7 @@ impl RuntimeOwner {{let redirect_has_extra_info = request_extra_info.is_some() && !response.from_cache;job.redirect_chain.push(RedirectInfo {+ source: crate::RedirectSource::Network,from_url: response.final_url.clone(),to_url: next_url.clone(),status: response.status,@@ -1819,6 +1820,7 @@ impl RuntimeOwner {}let redirect_has_extra_info = request_extra_info.is_some();job.redirect_chain.push(RedirectInfo {+ source: crate::RedirectSource::Network,from_url: final_url,to_url: next_url.clone(),status,@@ -2199,6 +2201,7 @@ impl RuntimeOwner {}let redirect_has_extra_info = request_extra_info.is_some();job.redirect_chain.push(RedirectInfo {+ source: crate::RedirectSource::Network,from_url: final_url,to_url: next_url.clone(),status,@@ -2339,6 +2342,7 @@ impl RuntimeOwner {&& job.request.follow_redirects{job.redirect_chain.push(RedirectInfo {+ source: crate::RedirectSource::Network,from_url: final_url,to_url: next_url.clone(),status: cached.status,@@ -2400,6 +2404,7 @@ impl RuntimeOwner {&& job.request.follow_redirects{job.redirect_chain.push(RedirectInfo {+ source: crate::RedirectSource::Network,from_url: final_url,to_url: next_url.clone(),status: cached.status,@@ -3295,6 +3300,7 @@ fn https_upgrade_redirect_info(request_cookie_report: Option<StoredCookieQueryReport>,) -> RedirectInfo {RedirectInfo {+ source: crate::RedirectSource::Internal,from_url,headers: vec![("location".to_owned(), to_url.to_string()),@@ -3346,6 +3352,7 @@ fn critical_client_hint_restart_redirect_info(response_extra_info: NetworkResponseExtraInfo,) -> RedirectInfo {RedirectInfo {+ source: crate::RedirectSource::Internal,from_url: url.clone(),to_url: url.clone(),status: 307,diff --git a/moli-page-types/src/lib.rs b/moli-page-types/src/lib.rsindex b89a0f950..901ca6648 100644--- a/moli-page-types/src/lib.rs+++ b/moli-page-types/src/lib.rs@@ -252,6 +252,7 @@ impl DomScrollIntoViewRect {#[derive(Debug, Clone, PartialEq, Eq)]pub struct NavigationRedirect {+ pub source: moli_fetch::RedirectSource,pub from_url: Url,pub to_url: Url,pub status: u16,@@ -269,6 +270,7 @@ pub struct NavigationRedirect {impl From<RedirectInfo> for NavigationRedirect {fn from(value: RedirectInfo) -> Self {Self {+ source: value.source,from_url: value.from_url,to_url: value.to_url,status: value.status,@@ -288,6 +290,7 @@ impl From<RedirectInfo> for NavigationRedirect {impl From<NavigationRedirect> for RedirectInfo {fn from(value: NavigationRedirect) -> Self {Self {+ source: value.source,from_url: value.from_url,to_url: value.to_url,status: value.status,diff --git a/moli-protocol/src/conn/tests/cookie_surfaces/snapshots/projection.rs b/moli-protocol/src/conn/tests/cookie_surfaces/snapshots/projection.rsindex 6a3076f52..88bab28d0 100644--- a/moli-protocol/src/conn/tests/cookie_surfaces/snapshots/projection.rs+++ b/moli-protocol/src/conn/tests/cookie_surfaces/snapshots/projection.rs@@ -287,6 +287,7 @@ async fn browser_context_cookie_manager_surface_tracks_redirected_navigation_tra);response.redirected = true;response.redirect_chain = vec![moli_core::page::NavigationRedirect {+ source: moli_fetch::RedirectSource::Network,from_url: requested_url.clone(),to_url: final_url.clone(),status: 302,diff --git a/moli-protocol/src/domains/network/main_document_progress/tests.rs b/moli-protocol/src/domains/network/main_document_progress/tests.rsindex 961415fb4..0e9922e88 100644--- a/moli-protocol/src/domains/network/main_document_progress/tests.rs+++ b/moli-protocol/src/domains/network/main_document_progress/tests.rs@@ -122,6 +122,7 @@ fn failed_redirect_transport_emits_completed_hop_and_final_request_extra_info());let final_url = Url::parse("http://example.test/reset").unwrap();let redirect = RedirectInfo {+ source: moli_fetch::RedirectSource::Network,from_url: Url::parse("http://example.test/start").unwrap(),to_url: final_url.clone(),status: 302,@@ -404,6 +405,7 @@ fn live_progress_source_serializes_through_progress_emissions() {effective_same_site: None,}];let redirect = RedirectInfo {+ source: moli_fetch::RedirectSource::Network,from_url: Url::parse("http://example.test/start").unwrap(),to_url: final_url.clone(),status: 302,@@ -757,6 +759,7 @@ fn completed_body_http_redirect_emits_correlated_no_cookie_extra_info() {let mut events = completed_events();events.network_extra_info_available = true;events.redirect_chain = vec![moli_core::page::NavigationRedirect {+ source: moli_fetch::RedirectSource::Network,from_url: Url::parse("http://example.test/start").unwrap(),to_url: final_url.clone(),status: 302,@@ -852,6 +855,7 @@ fn completed_body_redirect_without_transport_extra_info_keeps_flag_false() {let final_url = Url::parse("http://example.test/final").unwrap();let mut events = completed_events();events.redirect_chain = vec![moli_core::page::NavigationRedirect {+ source: moli_fetch::RedirectSource::Network,from_url: Url::parse("http://example.test/start").unwrap(),to_url: final_url.clone(),status: 302,@@ -920,6 +924,7 @@ fn critical_client_hint_restart_keeps_discarded_response_extra_info_separate_fro.with_network_observation_journal(journal);events.network_extra_info_available = true;events.redirect_chain = vec![moli_core::page::NavigationRedirect {+ source: moli_fetch::RedirectSource::Internal,from_url: navigation_url.clone(),to_url: navigation_url.clone(),status: 307,@@ -996,6 +1001,7 @@ fn completed_body_uses_negotiated_protocol_for_redirect_and_final_response() {let mut events =completed_events().with_negotiated_http_version(Some(NegotiatedHttpVersion::Http2));events.redirect_chain = vec![moli_core::page::NavigationRedirect {+ source: moli_fetch::RedirectSource::Network,from_url: Url::parse("https://example.test/start").unwrap(),to_url: final_url.clone(),status: 302,@@ -1073,6 +1079,7 @@ fn cached_completed_body_redirect_emits_cache_event_before_next_request() {let final_url = Url::parse("http://example.test/final").unwrap();let mut events = completed_events();events.redirect_chain = vec![moli_core::page::NavigationRedirect {+ source: moli_fetch::RedirectSource::Network,from_url: Url::parse("http://example.test/start").unwrap(),to_url: final_url.clone(),status: 302,diff --git a/moli-protocol/src/domains/network/output_queue.rs b/moli-protocol/src/domains/network/output_queue.rsindex c7639e8f9..d7eac84f7 100644--- a/moli-protocol/src/domains/network/output_queue.rs+++ b/moli-protocol/src/domains/network/output_queue.rs@@ -3716,6 +3716,7 @@ mod tests {SubresourceResourceType::Fetch,Some(StoredCookieQueryReport::default()),vec![NavigationRedirect {+ source: moli_fetch::RedirectSource::Network,from_url: start_url.clone(),to_url: final_url.clone(),status: 307,diff --git a/moli-renderer-v8/src/network/request_client/tests.rs b/moli-renderer-v8/src/network/request_client/tests.rsindex e09e4fedb..4748079d1 100644--- a/moli-renderer-v8/src/network/request_client/tests.rs+++ b/moli-renderer-v8/src/network/request_client/tests.rs@@ -1365,6 +1365,7 @@ async fn cached_raw_subresource_marks_redirect_hops_from_cache() -> Result<()> {cookie_set_reports: Vec::new(),redirected: true,redirect_chain: vec![RedirectInfo {+ source: moli_fetch::RedirectSource::Network,from_url: start_url,to_url: final_url,status: 301,diff --git a/moli-renderer-v8/src/network_host/async_fetch.rs b/moli-renderer-v8/src/network_host/async_fetch.rsindex 814757e1f..9c84a48cc 100644--- a/moli-renderer-v8/src/network_host/async_fetch.rs+++ b/moli-renderer-v8/src/network_host/async_fetch.rs@@ -204,6 +204,7 @@ impl ManualCorsRedirectState {}let redirect_status = head.status;self.redirect_chain.push(RedirectInfo {+ source: moli_fetch::RedirectSource::Network,from_url: head.final_url,to_url: next_url.clone(),status: redirect_status,@@ -1255,6 +1256,7 @@ mod tests {.with_initiator_url(&Url::parse(&format!("http://{addr}/page"))?).with_browser_request_metadata(BrowserRequestMetadata::Fetch);let initial_redirect_chain = vec![RedirectInfo {+ source: moli_fetch::RedirectSource::ServiceWorker,from_url: source_url.clone(),to_url: target_url.clone(),status: 302,diff --git a/moli-renderer-v8/src/network_host/response/cors.rs b/moli-renderer-v8/src/network_host/response/cors.rsindex 087a45961..341211568 100644--- a/moli-renderer-v8/src/network_host/response/cors.rs+++ b/moli-renderer-v8/src/network_host/response/cors.rs@@ -1,5 +1,5 @@use moli_cookie_jar::same_site_urls;-use moli_fetch::{RequestCredentialsMode, RequestMode};+use moli_fetch::{RedirectSource, RequestCredentialsMode, RequestMode};use moli_url::{origin_ascii_serialization, same_origin};use moli_web_mime::{response_header_value, response_header_values, should_opaque_response_be_blocked_by_orb,@@ -53,7 +53,9 @@ pub(crate) fn validate_cors_response(validate_cors_response_for_origin(&origin, response_headers, credentials_mode)}-/// Validates an already fetched network response, including each redirect.+/// Validates an already fetched network response and its network redirects.+/// Service worker and browser-generated redirects still contribute to taint,+/// but their synthetic response headers are not subject to the CORS check./// Once CORS-tainted, a redirect across origins changes the request origin to/// null, even when the final URL returns to the initiating document's origin.pub(crate) fn validate_cors_response_chain(@@ -66,7 +68,9 @@ pub(crate) fn validate_cors_response_chain(for redirect in &head.redirect_chain {cors_tainted |= !same_origin(document_url, &redirect.from_url);if cors_tainted {- validate_cors_response_for_origin(&origin, &redirect.headers, credentials_mode)?;+ if redirect.source == RedirectSource::Network {+ validate_cors_response_for_origin(&origin, &redirect.headers, credentials_mode)?;+ }if !same_origin(&redirect.from_url, &redirect.to_url) {origin = "null".to_owned();}@@ -571,6 +575,47 @@ mod tests {url::Url::parse(value).expect("valid URL")}+ #[test]+ fn cors_response_chain_checks_network_redirects_without_extra_info() {+ for from_cache in [false, true] {+ let mut head = moli_fetch::ResponseHead {+ final_url: url("https://final.test/script.js"),+ status: 200,+ headers: vec![("Access-Control-Allow-Origin".to_owned(), "*".to_owned())],+ request_cookie_report: None,+ cookie_set_reports: Vec::new(),+ redirected: true,+ redirect_chain: vec![moli_fetch::RedirectInfo {+ source: RedirectSource::Network,+ from_url: url("https://redirect.test/script.js"),+ to_url: url("https://final.test/script.js"),+ status: 302,+ headers: Vec::new(),+ network_extra_info_available: false,+ request_extra_info: None,+ response_extra_info: None,+ redirect_has_extra_info: false,+ request_cookie_report: None,+ cookie_set_reports: Vec::new(),+ from_cache,+ negotiated_http_version: None,+ }],+ from_cache: false,+ negotiated_http_version: None,+ };+ let document_url = url("https://document.test/page.html");+ validate_cors_response_chain(&document_url, &head, RequestCredentialsMode::SameOrigin)+ .expect_err("HTTP redirects require ACAO even without network ExtraInfo");++ head.redirect_chain[0].headers.push((+ "Access-Control-Allow-Origin".to_owned(),+ "https://document.test".to_owned(),+ ));+ validate_cors_response_chain(&document_url, &head, RequestCredentialsMode::SameOrigin)+ .expect("authorized HTTP redirect should pass");+ }+ }+#[test]fn cors_preflight_request_headers_detect_unsafe_method_and_headers() {let headers = vec![diff --git a/moli-renderer-v8/src/runtime/tests/redirect_chain.rs b/moli-renderer-v8/src/runtime/tests/redirect_chain.rsindex 6719288da..692e6c45b 100644--- a/moli-renderer-v8/src/runtime/tests/redirect_chain.rs+++ b/moli-renderer-v8/src/runtime/tests/redirect_chain.rs@@ -32,6 +32,7 @@ async fn assert_streaming_redirect_chain(reply_boundary: RendererReplyBoundary).map(|location| {let to_url = requested_url.join(location).unwrap();let redirect = NavigationRedirect {+ source: moli_fetch::RedirectSource::Network,from_url: from_url.clone(),to_url: to_url.clone(),status: 302,diff --git a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rsindex 0f047cccf..cc9fa7844 100644--- a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs+++ b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs@@ -1991,6 +1991,7 @@ async fn streaming_subresource_finish_preserves_response_head_cache_state() {cookie_set_reports: Vec::new(),redirected: true,redirect_chain: vec![moli_fetch::RedirectInfo {+ source: moli_fetch::RedirectSource::Network,from_url: request_url.clone(),to_url: final_url.clone(),status: 301,diff --git a/moli-renderer-v8/src/script_vm/tests/mod.rs b/moli-renderer-v8/src/script_vm/tests/mod.rsindex c64ce4a11..6db5c42a2 100644--- a/moli-renderer-v8/src/script_vm/tests/mod.rs+++ b/moli-renderer-v8/src/script_vm/tests/mod.rs@@ -504,6 +504,7 @@ fn redirected_fetch_response(source_url: &Url, final_url: Url) -> crate::types::);response.redirected = true;response.redirect_chain = vec![crate::types::NavigationRedirect {+ source: moli_fetch::RedirectSource::Network,from_url: source_url.clone(),to_url: final_url,status: 302,diff --git a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rsindex aaaca42ff..4aa54c947 100644--- a/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs+++ b/moli-renderer-v8/src/service_worker_runtime/service/fetch_settlement.rs@@ -169,6 +169,7 @@ fn apply_service_worker_synthetic_redirect(.final_url.unwrap_or_else(|| job.request_url.clone());job.redirect_chain.push(moli_fetch::RedirectInfo {+ source: moli_fetch::RedirectSource::ServiceWorker,from_url,to_url: next_url.clone(),status: response.status,@@ -1896,6 +1897,10 @@ mod tests {);assert_eq!(redirected_job.redirect_count, 1);assert_eq!(redirected_job.redirect_chain.len(), 1);+ assert_eq!(+ redirected_job.redirect_chain[0].source,+ moli_fetch::RedirectSource::ServiceWorker+ );assert_eq!(redirected_job.redirect_chain[0].from_url,url("https://example.test/app/data.txt")diff --git a/moli-renderer-v8/src/worker/script_loading.rs b/moli-renderer-v8/src/worker/script_loading.rsindex 120db4809..c68c8718c 100644--- a/moli-renderer-v8/src/worker/script_loading.rs+++ b/moli-renderer-v8/src/worker/script_loading.rs@@ -41,6 +41,7 @@ mod tests {fn redirect(from_url: &Url, to_url: &Url) -> RedirectInfo {RedirectInfo {+ source: moli_fetch::RedirectSource::Network,from_url: from_url.clone(),to_url: to_url.clone(),status: 302,diff --git a/moli/src/network_trace.rs b/moli/src/network_trace.rsindex 67a06e8f3..b5b2cc3bc 100644--- a/moli/src/network_trace.rs+++ b/moli/src/network_trace.rs@@ -1046,6 +1046,7 @@ mod tests {#[test]fn subresource_trace_diagnostics_classify_login_gate_and_json_error() -> anyhow::Result<()> {let redirect_chain = vec![NavigationRedirect {+ source: moli_fetch::RedirectSource::Network,from_url: "http://example.test/api/detail".parse()?,to_url: "http://login.example.test/security/baxia".parse()?,status: 302,