Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
refactor(fetch): centralize referrer policy parsing
ldm0b406682586d7
diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rsindex 16dc6e114..aa593b25d 100644--- a/moli-fetch/src/lib.rs+++ b/moli-fetch/src/lib.rs@@ -67,8 +67,8 @@ pub use network_fetch_result::{NetworkResponseObservation,};pub use referrer_policy::{- DEFAULT_REFERRER_POLICY, origin_referrer_url, referrer_header_value, referrer_value,- sanitized_referrer_url,+ DEFAULT_REFERRER_POLICY, normalize_referrer_policy, origin_referrer_url, referrer_header_value,+ referrer_value, response_referrer_policy_from_headers, sanitized_referrer_url,};pub use request::{BrowserNavigationRequestKind, BrowserRequestMetadata, FetchPriorityHint, Request, RequestAuth,diff --git a/moli-fetch/src/referrer_policy.rs b/moli-fetch/src/referrer_policy.rsindex e8a041f21..582912c9d 100644--- a/moli-fetch/src/referrer_policy.rs+++ b/moli-fetch/src/referrer_policy.rs@@ -3,6 +3,44 @@ use url::Url;pub const DEFAULT_REFERRER_POLICY: &str = "strict-origin-when-cross-origin";+const VALID_REFERRER_POLICIES: &[&str] = &[+ "no-referrer",+ "no-referrer-when-downgrade",+ "origin",+ "origin-when-cross-origin",+ "same-origin",+ "strict-origin",+ "strict-origin-when-cross-origin",+ "unsafe-url",+];++pub fn normalize_referrer_policy(raw: &str) -> Option<String> {+ raw.split(',')+ .filter_map(|token| {+ let token = token.trim().to_ascii_lowercase();+ VALID_REFERRER_POLICIES+ .contains(&token.as_str())+ .then_some(token)+ })+ .next_back()+}++pub fn response_referrer_policy_from_headers(headers: &[(String, Vec<u8>)]) -> Option<String> {+ let mut combined = String::new();+ for (_, value) in headers+ .iter()+ .filter(|(name, _)| name.eq_ignore_ascii_case("referrer-policy"))+ {+ if !combined.is_empty() {+ combined.push_str(", ");+ }+ combined.push_str(&crate::decode_header_value(value));+ }+ (!combined.is_empty())+ .then(|| normalize_referrer_policy(&combined))+ .flatten()+}+pub fn referrer_header_value(referrer_url: &Url,request_url: &Url,@@ -111,6 +149,39 @@ fn is_downgrade_request(referrer_url: &Url, request_url: &Url) -> bool {mod tests {use super::*;+ #[test]+ fn last_valid_referrer_policy_token_wins() {+ assert_eq!(+ normalize_referrer_policy("not-yet-standardized, no-referrer"),+ Some("no-referrer".to_owned())+ );+ assert_eq!(+ normalize_referrer_policy("origin, not-yet-standardized, strict-origin"),+ Some("strict-origin".to_owned())+ );+ }++ #[test]+ fn invalid_trailing_tokens_do_not_clear_previous_valid_policy() {+ assert_eq!(+ normalize_referrer_policy("same-origin, not-yet-standardized"),+ Some("same-origin".to_owned())+ );+ }++ #[test]+ fn response_referrer_policy_combines_header_instances_before_normalizing() {+ let headers: Vec<(String, Vec<u8>)> = vec![+ ("Referrer-Policy".to_owned(), b"no-referrer".to_vec()),+ ("referrer-policy".to_owned(), b"future-policy".to_vec()),+ ];++ assert_eq!(+ response_referrer_policy_from_headers(&headers),+ Some("no-referrer".to_owned())+ );+ }+fn url(input: &str) -> Url {Url::parse(input).unwrap()}diff --git a/moli-renderer-v8/src/referrer_policy.rs b/moli-renderer-v8/src/referrer_policy.rsindex be12f0116..fcea9cf90 100644--- a/moli-renderer-v8/src/referrer_policy.rs+++ b/moli-renderer-v8/src/referrer_policy.rs@@ -1,77 +1 @@-const VALID_REFERRER_POLICIES: &[&str] = &[- "no-referrer",- "no-referrer-when-downgrade",- "origin",- "origin-when-cross-origin",- "same-origin",- "strict-origin",- "strict-origin-when-cross-origin",- "unsafe-url",-];--pub(crate) fn normalize_referrer_policy(raw: &str) -> Option<String> {- raw.split(',')- .filter_map(|token| {- let token = token.trim().to_ascii_lowercase();- VALID_REFERRER_POLICIES- .contains(&token.as_str())- .then_some(token)- })- .next_back()-}--pub(crate) fn response_referrer_policy_from_headers(- headers: &[(String, Vec<u8>)],-) -> Option<String> {- let mut combined = String::new();- for (_, value) in headers- .iter()- .filter(|(name, _)| name.eq_ignore_ascii_case("referrer-policy"))- {- if !combined.is_empty() {- combined.push_str(", ");- }- combined.push_str(&moli_fetch::decode_header_value(value));- }- (!combined.is_empty())- .then(|| normalize_referrer_policy(&combined))- .flatten()-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn last_valid_referrer_policy_token_wins() {- assert_eq!(- normalize_referrer_policy("not-yet-standardized, no-referrer"),- Some("no-referrer".to_owned())- );- assert_eq!(- normalize_referrer_policy("origin, not-yet-standardized, strict-origin"),- Some("strict-origin".to_owned())- );- }-- #[test]- fn invalid_trailing_tokens_do_not_clear_previous_valid_policy() {- assert_eq!(- normalize_referrer_policy("same-origin, not-yet-standardized"),- Some("same-origin".to_owned())- );- }-- #[test]- fn response_referrer_policy_combines_header_instances_before_normalizing() {- let headers: Vec<(String, Vec<u8>)> = vec![- ("Referrer-Policy".to_owned(), b"no-referrer".to_vec()),- ("referrer-policy".to_owned(), b"future-policy".to_vec()),- ];-- assert_eq!(- response_referrer_policy_from_headers(&headers),- Some("no-referrer".to_owned())- );- }-}+pub(crate) use moli_fetch::{normalize_referrer_policy, response_referrer_policy_from_headers};