Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

refactor(fetch): centralize referrer policy parsing

ldm0b406682586d7
diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rs
index 16dc6e114..aa593b25d 100644
--- a/moli-fetch/src/lib.rs
+++ b/moli-fetch/src/lib.rs
@@ -67,8 +67,8 @@ pub use network_fetch_result::{
NetworkResponseObservation,
};
pub use referrer_policy::{
- DEFAULT_REFERRER_POLICY, origin_referrer_url, referrer_header_value, referrer_value,
- sanitized_referrer_url,
+ DEFAULT_REFERRER_POLICY, normalize_referrer_policy, origin_referrer_url, referrer_header_value,
+ referrer_value, response_referrer_policy_from_headers, sanitized_referrer_url,
};
pub use request::{
BrowserNavigationRequestKind, BrowserRequestMetadata, FetchPriorityHint, Request, RequestAuth,
diff --git a/moli-fetch/src/referrer_policy.rs b/moli-fetch/src/referrer_policy.rs
index e8a041f21..582912c9d 100644
--- a/moli-fetch/src/referrer_policy.rs
+++ b/moli-fetch/src/referrer_policy.rs
@@ -3,6 +3,44 @@ use url::Url;
pub const DEFAULT_REFERRER_POLICY: &str = "strict-origin-when-cross-origin";
+const VALID_REFERRER_POLICIES: &[&str] = &[
+ "no-referrer",
+ "no-referrer-when-downgrade",
+ "origin",
+ "origin-when-cross-origin",
+ "same-origin",
+ "strict-origin",
+ "strict-origin-when-cross-origin",
+ "unsafe-url",
+];
+
+pub fn normalize_referrer_policy(raw: &str) -> Option<String> {
+ raw.split(',')
+ .filter_map(|token| {
+ let token = token.trim().to_ascii_lowercase();
+ VALID_REFERRER_POLICIES
+ .contains(&token.as_str())
+ .then_some(token)
+ })
+ .next_back()
+}
+
+pub fn response_referrer_policy_from_headers(headers: &[(String, Vec<u8>)]) -> Option<String> {
+ let mut combined = String::new();
+ for (_, value) in headers
+ .iter()
+ .filter(|(name, _)| name.eq_ignore_ascii_case("referrer-policy"))
+ {
+ if !combined.is_empty() {
+ combined.push_str(", ");
+ }
+ combined.push_str(&crate::decode_header_value(value));
+ }
+ (!combined.is_empty())
+ .then(|| normalize_referrer_policy(&combined))
+ .flatten()
+}
+
pub fn referrer_header_value(
referrer_url: &Url,
request_url: &Url,
@@ -111,6 +149,39 @@ fn is_downgrade_request(referrer_url: &Url, request_url: &Url) -> bool {
mod tests {
use super::*;
+ #[test]
+ fn last_valid_referrer_policy_token_wins() {
+ assert_eq!(
+ normalize_referrer_policy("not-yet-standardized, no-referrer"),
+ Some("no-referrer".to_owned())
+ );
+ assert_eq!(
+ normalize_referrer_policy("origin, not-yet-standardized, strict-origin"),
+ Some("strict-origin".to_owned())
+ );
+ }
+
+ #[test]
+ fn invalid_trailing_tokens_do_not_clear_previous_valid_policy() {
+ assert_eq!(
+ normalize_referrer_policy("same-origin, not-yet-standardized"),
+ Some("same-origin".to_owned())
+ );
+ }
+
+ #[test]
+ fn response_referrer_policy_combines_header_instances_before_normalizing() {
+ let headers: Vec<(String, Vec<u8>)> = vec![
+ ("Referrer-Policy".to_owned(), b"no-referrer".to_vec()),
+ ("referrer-policy".to_owned(), b"future-policy".to_vec()),
+ ];
+
+ assert_eq!(
+ response_referrer_policy_from_headers(&headers),
+ Some("no-referrer".to_owned())
+ );
+ }
+
fn url(input: &str) -> Url {
Url::parse(input).unwrap()
}
diff --git a/moli-renderer-v8/src/referrer_policy.rs b/moli-renderer-v8/src/referrer_policy.rs
index be12f0116..fcea9cf90 100644
--- a/moli-renderer-v8/src/referrer_policy.rs
+++ b/moli-renderer-v8/src/referrer_policy.rs
@@ -1,77 +1 @@
-const VALID_REFERRER_POLICIES: &[&str] = &[
- "no-referrer",
- "no-referrer-when-downgrade",
- "origin",
- "origin-when-cross-origin",
- "same-origin",
- "strict-origin",
- "strict-origin-when-cross-origin",
- "unsafe-url",
-];
-
-pub(crate) fn normalize_referrer_policy(raw: &str) -> Option<String> {
- raw.split(',')
- .filter_map(|token| {
- let token = token.trim().to_ascii_lowercase();
- VALID_REFERRER_POLICIES
- .contains(&token.as_str())
- .then_some(token)
- })
- .next_back()
-}
-
-pub(crate) fn response_referrer_policy_from_headers(
- headers: &[(String, Vec<u8>)],
-) -> Option<String> {
- let mut combined = String::new();
- for (_, value) in headers
- .iter()
- .filter(|(name, _)| name.eq_ignore_ascii_case("referrer-policy"))
- {
- if !combined.is_empty() {
- combined.push_str(", ");
- }
- combined.push_str(&moli_fetch::decode_header_value(value));
- }
- (!combined.is_empty())
- .then(|| normalize_referrer_policy(&combined))
- .flatten()
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn last_valid_referrer_policy_token_wins() {
- assert_eq!(
- normalize_referrer_policy("not-yet-standardized, no-referrer"),
- Some("no-referrer".to_owned())
- );
- assert_eq!(
- normalize_referrer_policy("origin, not-yet-standardized, strict-origin"),
- Some("strict-origin".to_owned())
- );
- }
-
- #[test]
- fn invalid_trailing_tokens_do_not_clear_previous_valid_policy() {
- assert_eq!(
- normalize_referrer_policy("same-origin, not-yet-standardized"),
- Some("same-origin".to_owned())
- );
- }
-
- #[test]
- fn response_referrer_policy_combines_header_instances_before_normalizing() {
- let headers: Vec<(String, Vec<u8>)> = vec![
- ("Referrer-Policy".to_owned(), b"no-referrer".to_vec()),
- ("referrer-policy".to_owned(), b"future-policy".to_vec()),
- ];
-
- assert_eq!(
- response_referrer_policy_from_headers(&headers),
- Some("no-referrer".to_owned())
- );
- }
-}
+pub(crate) use moli_fetch::{normalize_referrer_policy, response_referrer_policy_from_headers};