Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
perf(site): compile public suffix rules into static trie
ldm05547cb593dad
diff --git a/Cargo.lock b/Cargo.lockindex 767529861..ee4b0646b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2356,7 +2356,7 @@ dependencies = ["document-features","idna","indexmap",- "publicsuffix",+ "psl-types","time","url",]@@ -2883,7 +2883,6 @@ dependencies = ["percent-encoding","png","precomputed-hash",- "publicsuffix","selectors","serde","serde_json",@@ -2954,7 +2953,7 @@ name = "moli-site"version = "0.1.0"dependencies = ["moli-url",- "publicsuffix",+ "psl-types","url",]@@ -3902,16 +3901,6 @@ version = "2.0.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"-[[package]]-name = "publicsuffix"-version = "2.3.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf"-dependencies = [- "idna",- "psl-types",-]-[[package]]name = "pxfm"version = "0.1.30"diff --git a/moli-cookie-jar/src/tests.rs b/moli-cookie-jar/src/tests.rsindex a0fa938e9..c07935cb5 100644--- a/moli-cookie-jar/src/tests.rs+++ b/moli-cookie-jar/src/tests.rs@@ -4704,6 +4704,20 @@ fn upstream_public_suffix_domain_rejection() {assert_eq!(store.cookie_header(&parse("https://bar.co.uk/")), None);}+#[test]+fn deeper_rule_does_not_shadow_parent_wildcard_during_domain_rejection() {+ let mut store = BrowserCookieStore::default();+ store.store_response_headers(+ &parse("https://foo.oci.customer-oci.com/"),+ &[(+ "set-cookie".to_owned(),+ "wide=1; Domain=oci.customer-oci.com; Path=/; Secure".to_owned(),+ )],+ );++ assert!(store.cookies().is_empty());+}+#[test]fn public_suffix_identical_host_domain_downgrades_to_host_only() {let mut store = BrowserCookieStore::default();diff --git a/moli-cookie-store/Cargo.toml b/moli-cookie-store/Cargo.tomlindex d2faf1965..f19fbf130 100644--- a/moli-cookie-store/Cargo.toml+++ b/moli-cookie-store/Cargo.toml@@ -25,8 +25,8 @@ default = ["public_suffix"]## uses `indexmap::IndexMap` in lieu of HashMap internally, so cookies are maintained in insertion/creation orderpreserve_order = ["dep:indexmap"]-## Add support for public suffix lists, as provided by [publicsuffix](https://crates.io/crates/publicsuffix).-public_suffix = ["dep:publicsuffix"]+## Add support for pluggable public suffix lookups.+public_suffix = ["dep:psl-types"]## Enable logging the values of cookies marked 'secure', off by default as values may be sensitivelog_secure_cookie_values = []@@ -39,7 +39,7 @@ url = "2.3.1"indexmap = { version = "2.6.0", optional = true }-publicsuffix = { version = "2.2.3", optional = true }+psl-types = { version = "2.0.11", optional = true }[dependencies.cookie]version = "0.18.0"diff --git a/moli-cookie-store/src/cookie_domain.rs b/moli-cookie-store/src/cookie_domain.rsindex 2a5f4cb28..e94f5273a 100644--- a/moli-cookie-store/src/cookie_domain.rs+++ b/moli-cookie-store/src/cookie_domain.rs@@ -1,6 +1,4 @@use cookie::Cookie as RawCookie;-#[cfg(feature = "public_suffix")]-use publicsuffix::{List, Psl, Suffix};use std::convert::TryFrom;use url::{Host, Url};@@ -112,17 +110,11 @@ impl CookieDomain {}}- /// Tests if the domain-attribute is a public suffix as indicated by the provided- /// `publicsuffix::List`.+ /// Tests if the domain-attribute is a public suffix as indicated by the provided lookup.#[cfg(feature = "public_suffix")]- pub fn is_public_suffix(&self, psl: &List) -> bool {+ pub fn is_public_suffix(&self, psl: &dyn crate::CookiePublicSuffixList) -> bool {if let Some(domain) = self.as_cow().as_ref().map(|d| d.as_bytes()) {- psl.suffix(domain)- // Only consider suffixes explicitly listed in the public suffix list- // to avoid issues like https://github.com/curl/curl/issues/658- .filter(Suffix::is_known)- .filter(|suffix| suffix == &domain)- .is_some()+ psl.is_public_suffix(domain)} else {false}diff --git a/moli-cookie-store/src/cookie_store/mod.rs b/moli-cookie-store/src/cookie_store/mod.rsindex f86227add..d7803f8cf 100644--- a/moli-cookie-store/src/cookie_store/mod.rs+++ b/moli-cookie-store/src/cookie_store/mod.rs@@ -25,8 +25,8 @@ pub struct CookieStore {next_access_index: u64,limits: CookieStoreLimits,#[cfg(feature = "public_suffix")]- /// If set, enables [public suffix](https://datatracker.ietf.org/doc/html/rfc6265#section-5.3) rejection based on the provided `publicsuffix::List`- public_suffix_list: Option<std::sync::Arc<publicsuffix::List>>,+ /// If set, enables [public suffix](https://datatracker.ietf.org/doc/html/rfc6265#section-5.3) rejection based on the provided rule lookup.+ public_suffix_list: Option<std::sync::Arc<dyn crate::CookiePublicSuffixList>>,}#[cfg(test)]diff --git a/moli-cookie-store/src/cookie_store/query.rs b/moli-cookie-store/src/cookie_store/query.rsindex 6c6636a82..b38b2addb 100644--- a/moli-cookie-store/src/cookie_store/query.rs+++ b/moli-cookie-store/src/cookie_store/query.rs@@ -206,16 +206,22 @@ impl CookieStore {.is_some()}- /// Specify a `publicsuffix::List` for the `CookieStore` to allow [public suffix+ /// Specify a public suffix lookup for the `CookieStore` to allow [public suffix/// matching](https://datatracker.ietf.org/doc/html/rfc6265#section-5.3).#[cfg(feature = "public_suffix")]- pub fn with_suffix_list(self, psl: publicsuffix::List) -> CookieStore {+ pub fn with_suffix_list<L>(self, psl: L) -> CookieStore+ where+ L: crate::CookiePublicSuffixList + 'static,+ {self.with_shared_suffix_list(std::sync::Arc::new(psl))}- /// Specify an immutable shared public suffix list without copying its parsed rule table.+ /// Specify an immutable shared public suffix lookup without copying its rule table.#[cfg(feature = "public_suffix")]- pub fn with_shared_suffix_list(self, psl: std::sync::Arc<publicsuffix::List>) -> CookieStore {+ pub fn with_shared_suffix_list<L>(self, psl: std::sync::Arc<L>) -> CookieStore+ where+ L: crate::CookiePublicSuffixList + 'static,+ {CookieStore {cookies: self.cookies,next_creation_index: self.next_creation_index,diff --git a/moli-cookie-store/src/cookie_store/storage.rs b/moli-cookie-store/src/cookie_store/storage.rsindex 4db3b1d73..c9470ffa0 100644--- a/moli-cookie-store/src/cookie_store/storage.rs+++ b/moli-cookie-store/src/cookie_store/storage.rs@@ -149,17 +149,6 @@ impl CookieStore {}}- #[cfg(feature = "public_suffix")]- pub fn new_with_public_suffix(public_suffix_list: Option<publicsuffix::List>) -> Self {- Self {- cookies: DomainMap::new(),- next_creation_index: 0,- next_access_index: 0,- limits: CookieStoreLimits::default(),- public_suffix_list: public_suffix_list.map(std::sync::Arc::new),- }- }-pub(super) fn bump_creation_index(&mut self) -> u64 {let index = self.next_creation_index;self.next_creation_index = self.next_creation_index.saturating_add(1);diff --git a/moli-cookie-store/src/cookie_store/tests.rs b/moli-cookie-store/src/cookie_store/tests.rsindex 4ccba68bc..d7f1585f2 100644--- a/moli-cookie-store/src/cookie_store/tests.rs+++ b/moli-cookie-store/src/cookie_store/tests.rs@@ -84,11 +84,19 @@ fn add_cookie(}#[cfg(feature = "public_suffix")]-fn make_public_suffix_list() -> publicsuffix::List {- publicsuffix::List::from_bytes(- b"// BEGIN ICANN DOMAINS\nco.uk\ngov.uk\ngithub.io\n// BEGIN PRIVATE DOMAINS\n",- )- .expect("test PSL must parse")+#[derive(Debug)]+struct TestPublicSuffixList;++#[cfg(feature = "public_suffix")]+impl crate::CookiePublicSuffixList for TestPublicSuffixList {+ fn is_public_suffix(&self, domain: &[u8]) -> bool {+ matches!(domain, b"co.uk" | b"gov.uk" | b"github.io")+ }+}++#[cfg(feature = "public_suffix")]+fn make_public_suffix_list() -> TestPublicSuffixList {+ TestPublicSuffixList}fn make_match_store() -> CookieStore {diff --git a/moli-cookie-store/src/cookie_store/tests/context.rs b/moli-cookie-store/src/cookie_store/tests/context.rsindex c80790595..bf2651c46 100644--- a/moli-cookie-store/src/cookie_store/tests/context.rs+++ b/moli-cookie-store/src/cookie_store/tests/context.rs@@ -56,20 +56,19 @@ fn shared_public_suffix_list_is_retained_across_store_clones() {let store = CookieStore::default().with_shared_suffix_list(std::sync::Arc::clone(&suffix_list));let cloned = store.clone();- assert!(std::sync::Arc::ptr_eq(- store- .public_suffix_list- .as_ref()- .expect("store should retain the shared PSL"),- &suffix_list,- ));- assert!(std::sync::Arc::ptr_eq(- cloned- .public_suffix_list- .as_ref()- .expect("cloned store should retain the shared PSL"),- &suffix_list,- ));+ let stored = store+ .public_suffix_list+ .as_ref()+ .expect("store should retain the shared PSL");+ let cloned = cloned+ .public_suffix_list+ .as_ref()+ .expect("cloned store should retain the shared PSL");+ assert!(std::sync::Arc::ptr_eq(stored, cloned));+ assert_eq!(+ std::sync::Arc::as_ptr(stored) as *const (),+ std::sync::Arc::as_ptr(&suffix_list) as *const (),+ );}#[cfg(feature = "public_suffix")]diff --git a/moli-cookie-store/src/cookie_store/write_policy.rs b/moli-cookie-store/src/cookie_store/write_policy.rsindex 78361cf65..f56c60add 100644--- a/moli-cookie-store/src/cookie_store/write_policy.rs+++ b/moli-cookie-store/src/cookie_store/write_policy.rs@@ -26,7 +26,7 @@ pub(super) fn canonicalize_cookie_for_store_checks<'a>() -> Result<Cookie<'a>, CookieError> {#[cfg(feature = "public_suffix")]if let Some(ref psl) = store.public_suffix_list {- if cookie.domain.is_public_suffix(psl) {+ if cookie.domain.is_public_suffix(psl.as_ref()) {if cookie.domain.host_is_identical(request_url) {cookie.domain = crate::cookie_domain::CookieDomain::host_only(request_url)?;} else {diff --git a/moli-cookie-store/src/lib.rs b/moli-cookie-store/src/lib.rsindex 294073a70..28d46a260 100644--- a/moli-cookie-store/src/lib.rs+++ b/moli-cookie-store/src/lib.rs@@ -35,6 +35,10 @@ pub use crate::cookie_store::{SameSiteContextHttpMethod, SameSiteContextMetadata, SameSiteContextRedirectType,SameSiteContextTrackMetadata, SameSiteRequestContext, StorageAccessStatus, StoreAction,};+#[cfg(feature = "public_suffix")]+mod public_suffix;+#[cfg(feature = "public_suffix")]+pub use crate::public_suffix::CookiePublicSuffixList;mod utils;#[derive(Debug)]diff --git a/moli-cookie-store/src/public_suffix.rs b/moli-cookie-store/src/public_suffix.rsnew file mode 100644index 000000000..5046ac8b5--- /dev/null+++ b/moli-cookie-store/src/public_suffix.rs@@ -0,0 +1,22 @@+use psl_types::{List as Psl, Suffix};++/// Public-suffix lookup needed by cookie Domain-attribute validation.+///+/// This object-safe boundary lets browser embedders provide either a static+/// generated table or another implementation of [`psl_types::List`].+pub trait CookiePublicSuffixList: std::fmt::Debug + Send + Sync {+ /// Returns whether `domain` exactly matches an explicitly listed suffix.+ fn is_public_suffix(&self, domain: &[u8]) -> bool;+}++impl<T> CookiePublicSuffixList for T+where+ T: Psl + std::fmt::Debug + Send + Sync,+{+ fn is_public_suffix(&self, domain: &[u8]) -> bool {+ self.suffix(domain)+ // The implicit wildcard must not reject arbitrary top-level names.+ .filter(Suffix::is_known)+ .is_some_and(|suffix| suffix == domain)+ }+}diff --git a/moli-renderer-v8/Cargo.toml b/moli-renderer-v8/Cargo.tomlindex f4b24bdfc..cfbe5d384 100644--- a/moli-renderer-v8/Cargo.toml+++ b/moli-renderer-v8/Cargo.toml@@ -82,7 +82,6 @@ moli-webidl = { path = "../moli-webidl" }moli-webidl-callback = { path = "../moli-webidl-callback" }moli-websocket = { path = "../moli-websocket" }moli-xpath = { path = "../moli-xpath" }-publicsuffix = "2.3.0"curl = { version = "0.4.49", default-features = false, features = ["brotli", "http2", "poll_7_68_0", "ssl", "static-curl"] }http = "1"httpdate = "1.0.3"diff --git a/moli-site/Cargo.toml b/moli-site/Cargo.tomlindex 85c05b66f..b4f504bb1 100644--- a/moli-site/Cargo.toml+++ b/moli-site/Cargo.toml@@ -7,7 +7,7 @@ description = "Moli registrable-site and schemeful-site helpers"[dependencies]moli-url = { path = "../moli-url" }-publicsuffix = "2.3.0"+psl-types = "2.0.11"url = "2.5.7"[lints]diff --git a/moli-site/build.rs b/moli-site/build.rsnew file mode 100644index 000000000..3811f44e1--- /dev/null+++ b/moli-site/build.rs@@ -0,0 +1,188 @@+use std::{+ collections::{BTreeMap, HashMap},+ env,+ fmt::Write as _,+ fs,+ path::PathBuf,+};++const LEAF_FLAG: u8 = 1;+const EXCEPTION_FLAG: u8 = 2;+const MAX_LABEL_LEN: usize = u8::MAX as usize;+const MAX_EDGE_COUNT: usize = u16::MAX as usize;+const MAX_LABEL_OFFSET: usize = 0x00ff_ffff;++#[derive(Default)]+struct TrieNode {+ flags: u8,+ children: BTreeMap<String, TrieNode>,+}++fn main() {+ const SOURCE_PATH: &str = "src/data/public_domains.txt";+ println!("cargo:rerun-if-changed={SOURCE_PATH}");++ let source = fs::read_to_string(SOURCE_PATH).expect("read vendored public suffix list");+ let (root, rule_count) = parse_rules(&source);+ let generated = generate_tables(&root, rule_count);+ let output = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR is set by Cargo"))+ .join("public_suffix_tables.rs");+ fs::write(output, generated).expect("write generated public suffix tables");+}++fn parse_rules(source: &str) -> (TrieNode, usize) {+ let mut root = TrieNode::default();+ let mut rule_count = 0;++ for raw_line in source.lines() {+ let line = raw_line.trim();+ if line.is_empty() || line.starts_with("//") {+ continue;+ }++ let (rule, is_exception) = match line.strip_prefix('!') {+ Some(rule) => (rule, true),+ None => (line, false),+ };+ assert!(!rule.is_empty(), "public suffix rule must not be empty");+ assert!(+ rule.is_ascii() && rule.bytes().all(|byte| !byte.is_ascii_uppercase()),+ "public suffix rule must be lowercase ASCII: {line}"+ );+ assert!(+ !rule.contains('*') || (rule.starts_with("*.") && rule[2..].find('*').is_none()),+ "public suffix wildcard must be the leftmost label: {line}"+ );++ let mut node = &mut root;+ let mut label_count = 0;+ for label in rule.rsplit('.') {+ assert!(+ !label.is_empty(),+ "public suffix label must not be empty: {line}"+ );+ assert!(+ label.len() <= MAX_LABEL_LEN,+ "public suffix label is too long: {label}"+ );+ node = node.children.entry(label.to_owned()).or_default();+ label_count += 1;+ }+ assert!(+ !is_exception || label_count > 1,+ "public suffix exception must contain more than one label: {line}"+ );+ assert_eq!(node.flags, 0, "duplicate public suffix rule: {line}");+ node.flags = LEAF_FLAG | if is_exception { EXCEPTION_FLAG } else { 0 };+ rule_count += 1;+ }++ assert!(+ rule_count > 0,+ "vendored public suffix list must not be empty"+ );+ (root, rule_count)+}++fn generate_tables(root: &TrieNode, rule_count: usize) -> String {+ // Nodes are visited breadth-first. Every edge appends exactly one child to+ // `nodes`, so an edge at index N always points to node N + 1. That removes+ // the child-index field from the generated edge table.+ let mut nodes = vec![root];+ let mut node_metadata = Vec::new();+ let mut edges = Vec::new();+ let mut label_offsets = HashMap::<&str, u32>::new();+ let mut labels = Vec::new();+ let mut node_index = 0;++ while node_index < nodes.len() {+ let node = nodes[node_index];+ let edge_start = edges.len();+ assert!(+ node.children.len() <= MAX_EDGE_COUNT,+ "a public suffix trie node has too many children"+ );++ for (label, child) in &node.children {+ let label_start = match label_offsets.get(label.as_str()) {+ Some(offset) => *offset,+ None => {+ assert!(+ labels.len() <= MAX_LABEL_OFFSET,+ "public suffix label blob exceeds the 24-bit edge encoding"+ );+ let offset = labels.len() as u32;+ labels.extend_from_slice(label.as_bytes());+ label_offsets.insert(label, offset);+ offset+ }+ };+ let encoded_edge = (label_start << 8) | label.len() as u32;+ edges.push(encoded_edge);+ nodes.push(child);+ assert_eq!(+ nodes.len() - 1,+ edges.len(),+ "breadth-first edge index must identify its child node"+ );+ }++ let metadata =+ edge_start as u64 | ((node.children.len() as u64) << 32) | ((node.flags as u64) << 48);+ node_metadata.push(metadata);+ node_index += 1;+ }++ assert_eq!(nodes.len(), node_metadata.len());+ assert_eq!(nodes.len(), edges.len() + 1);++ let mut output = String::new();+ writeln!(output, "#[cfg(test)]").unwrap();+ writeln!(output, "pub(super) const RULE_COUNT: usize = {rule_count};").unwrap();+ writeln!(+ output,+ "pub(super) static NODE_METADATA: [u64; {}] = [",+ node_metadata.len()+ )+ .unwrap();+ for chunk in node_metadata.chunks(8) {+ output.push_str(" ");+ for value in chunk {+ write!(output, "0x{value:016x}, ").unwrap();+ }+ output.push('\n');+ }+ output.push_str("];\n");++ writeln!(+ output,+ "pub(super) static EDGES: [u32; {}] = [",+ edges.len()+ )+ .unwrap();+ for chunk in edges.chunks(12) {+ output.push_str(" ");+ for value in chunk {+ write!(output, "0x{value:08x}, ").unwrap();+ }+ output.push('\n');+ }+ output.push_str("];\n");++ writeln!(+ output,+ "pub(super) static LABELS: [u8; {}] = [",+ labels.len()+ )+ .unwrap();+ for chunk in labels.chunks(24) {+ output.push_str(" ");+ for byte in chunk {+ write!(output, "0x{byte:02x}, ").unwrap();+ }+ output.push('\n');+ }+ output.push_str("];\n");++ output+}diff --git a/moli-site/src/lib.rs b/moli-site/src/lib.rsindex bed096b54..f30f3679a 100644--- a/moli-site/src/lib.rs+++ b/moli-site/src/lib.rs@@ -5,8 +5,8 @@ mod same_site;mod schemeful;pub use public_suffix::{- host_is_public_suffix, public_suffix_list, registrable_site_host, same_site_hosts,- site_key_for_host,+ StaticPublicSuffixList, host_is_public_suffix, public_suffix_list, registrable_site_host,+ same_site_hosts, site_key_for_host,};pub use same_site::same_site_urls;pub use schemeful::schemeful_site_for_url;diff --git a/moli-site/src/public_suffix.rs b/moli-site/src/public_suffix.rsindex e7238925e..db8df0db1 100644--- a/moli-site/src/public_suffix.rs+++ b/moli-site/src/public_suffix.rs@@ -1,82 +1,105 @@-use std::{- collections::HashSet,- net::IpAddr,- sync::{Arc, LazyLock},-};--use publicsuffix::{List as PublicSuffixList, Psl};--#[derive(Clone, Debug, Default)]-struct PublicDomainRules {- rules: HashSet<String>,- wildcards: HashSet<String>,- exceptions: HashSet<String>,+use std::{net::IpAddr, sync::Arc};++use psl_types::{Info, List as Psl, Type};++const LEAF_FLAG: u8 = 1;+const EXCEPTION_FLAG: u8 = 2;+const EDGE_START_MASK: u64 = u32::MAX as u64;+const EDGE_COUNT_MASK: u64 = u16::MAX as u64;++mod tables {+ include!(concat!(env!("OUT_DIR"), "/public_suffix_tables.rs"));}-static PUBLIC_DOMAIN_RULES: LazyLock<PublicDomainRules> = LazyLock::new(|| {- // Vendored PSL snapshot copied from Servo's local resource list. Keep the- // site-context model self-contained inside Moli instead of depending- // on runtime fetches or cargo-registry paths.- PublicDomainRules::parse(include_str!("data/public_domains.txt"))-});--static PUBLIC_SUFFIX_LIST: LazyLock<Arc<PublicSuffixList>> = LazyLock::new(|| {- // Servo's vendored snapshot is a raw rules file, while the `publicsuffix`- // crate expects the official list sections. Wrap the same bytes instead of- // carrying a second PSL snapshot just for fork-side rejection.- let wrapped_list = format!(- "// BEGIN ICANN DOMAINS\n{}\n// BEGIN PRIVATE DOMAINS\n",- include_str!("data/public_domains.txt")- );- Arc::new(- PublicSuffixList::from_bytes(wrapped_list.as_bytes())- .expect("vendored public suffix list must parse"),- )-});--impl PublicDomainRules {- fn parse(content: &str) -> PublicDomainRules {- let mut result = PublicDomainRules::default();- for item in content- .lines()- .map(str::trim)- .filter(|line| !line.is_empty() && !line.starts_with("//"))- {- if let Some(stripped) = item.strip_prefix('!') {- result.exceptions.insert(stripped.to_owned());- } else if let Some(stripped) = item.strip_prefix("*.") {- result.wildcards.insert(stripped.to_owned());- } else {- result.rules.insert(item.to_owned());- }- }- result+/// Allocation-free view of Moli's vendored Public Suffix List snapshot.+///+/// The rule trie is generated into compact read-only tables at build time. An+/// instance is zero-sized; clones share the executable's static table pages.+#[derive(Clone, Copy, Debug, Default)]+pub struct StaticPublicSuffixList;++static PUBLIC_SUFFIX_LIST: std::sync::LazyLock<Arc<StaticPublicSuffixList>> =+ std::sync::LazyLock::new(|| Arc::new(StaticPublicSuffixList));++impl StaticPublicSuffixList {+ fn child(node_index: usize, label: &[u8]) -> Option<usize> {+ let metadata = *tables::NODE_METADATA.get(node_index)?;+ let edge_start = (metadata & EDGE_START_MASK) as usize;+ let edge_count = ((metadata >> 32) & EDGE_COUNT_MASK) as usize;+ let edges = tables::EDGES.get(edge_start..edge_start + edge_count)?;++ edges+ .binary_search_by(|encoded| edge_label(*encoded).cmp(label))+ .ok()+ .map(|relative_index| edge_start + relative_index + 1)+ }++ fn flags(node_index: usize) -> u8 {+ (tables::NODE_METADATA[node_index] >> 48) as u8}+}- fn suffix_pair<'a>(&self, domain: &'a str) -> (&'a str, &'a str) {- let domain = domain.trim_start_matches('.');- let mut suffix = domain;- let mut prev_suffix = domain;+impl Psl for StaticPublicSuffixList {+ fn find<'a, T>(&self, mut labels: T) -> Info+ where+ T: Iterator<Item = &'a [u8]>,+ {+ let Some(first_label) = labels.next() else {+ return Info { len: 0, typ: None };+ };+ let mut info = Info {+ len: first_label.len(),+ typ: None,+ };+ let Some(mut node_index) = Self::child(0, first_label) else {+ return info;+ };+ if Self::flags(node_index) & LEAF_FLAG != 0 {+ info.typ = Some(Type::Icann);+ }- for (index, _) in domain.match_indices('.') {- let next_suffix = &domain[index + 1..];- if self.exceptions.contains(suffix) {- return (next_suffix, suffix);+ let mut len_so_far = info.len;+ for label in labels {+ let label_plus_dot = label.len() + 1;++ // A wildcard at this level remains a valid candidate even when an+ // exact child exists only to hold a deeper rule. Keep the wildcard+ // result, then continue down the exact branch in search of a+ // longer match. Choosing the exact branch up front would make a+ // deeper rule incorrectly shadow its parent's wildcard.+ if let Some(wildcard_node) = Self::child(node_index, b"*") {+ let flags = Self::flags(wildcard_node);+ if flags & LEAF_FLAG != 0 {+ info.typ = Some(Type::Icann);+ info.len = len_so_far + label_plus_dot;+ }}- if self.wildcards.contains(next_suffix) || self.rules.contains(suffix) {- return (suffix, prev_suffix);++ let Some(next_node) = Self::child(node_index, label) else {+ break;+ };+ node_index = next_node;++ let flags = Self::flags(node_index);+ if flags & LEAF_FLAG != 0 {+ info.typ = Some(Type::Icann);+ if flags & EXCEPTION_FLAG != 0 {+ info.len = len_so_far;+ break;+ }+ info.len = len_so_far + label_plus_dot;}- prev_suffix = suffix;- suffix = next_suffix;+ len_so_far += label_plus_dot;}- (suffix, prev_suffix)+ info}+}- fn registrable_suffix<'a>(&self, domain: &'a str) -> &'a str {- let (_, registrable) = self.suffix_pair(domain);- registrable- }+fn edge_label(encoded: u32) -> &'static [u8] {+ let start = (encoded >> 8) as usize;+ let len = (encoded & u8::MAX as u32) as usize;+ &tables::LABELS[start..start + len]}pub(crate) fn is_ip_host(host: &str) -> bool {@@ -102,7 +125,7 @@ pub fn host_is_public_suffix(host: &str) -> bool {if host.is_empty() || is_ip_host(&host) {return false;}- PUBLIC_SUFFIX_LIST+ StaticPublicSuffixList.suffix(host.as_bytes()).is_some_and(|suffix| suffix.is_known() && suffix == host.as_bytes())}@@ -111,10 +134,14 @@ pub fn host_is_public_suffix(host: &str) -> bool {/// would not find an eTLD+1, such as IP literals, localhost, and public suffixes.pub fn registrable_site_host(host: &str) -> &str {if is_ip_host(host) {- host- } else {- PUBLIC_DOMAIN_RULES.registrable_suffix(host)+ return host;}++ let host = host.trim_start_matches('.');+ StaticPublicSuffixList+ .domain(host.as_bytes())+ .and_then(|domain| std::str::from_utf8(domain.as_bytes()).ok())+ .unwrap_or(host)}/// Compares two hosts using registrable-domain site semantics.@@ -127,6 +154,16 @@ pub fn same_site_hosts(host_a: &str, host_b: &str) -> bool {}/// Returns the shared vendored public suffix list used by cookie stores.-pub fn public_suffix_list() -> Arc<PublicSuffixList> {+pub fn public_suffix_list() -> Arc<StaticPublicSuffixList> {Arc::clone(&PUBLIC_SUFFIX_LIST)}++#[cfg(test)]+pub(super) fn static_table_size_bytes() -> usize {+ std::mem::size_of_val(&tables::NODE_METADATA)+ + std::mem::size_of_val(&tables::EDGES)+ + std::mem::size_of_val(&tables::LABELS)+}++#[cfg(test)]+pub(super) const STATIC_RULE_COUNT: usize = tables::RULE_COUNT;diff --git a/moli-site/src/tests.rs b/moli-site/src/tests.rsindex 99f92ec22..725fb9cd4 100644--- a/moli-site/src/tests.rs+++ b/moli-site/src/tests.rs@@ -1,7 +1,10 @@use super::{- host_is_public_suffix, public_suffix_list, registrable_site_host, same_site_hosts,- same_site_urls, schemeful_site_for_url, site_key_for_host,+ StaticPublicSuffixList, host_is_public_suffix, public_suffix_list, registrable_site_host,+ same_site_hosts, same_site_urls, schemeful_site_for_url, site_key_for_host,};+use crate::public_suffix::{STATIC_RULE_COUNT, static_table_size_bytes};+use psl_types::List as _;+use std::collections::HashSet;use std::sync::Arc;use url::Url;@@ -196,3 +199,109 @@ fn public_suffix_list_returns_the_shared_snapshot() {assert!(Arc::ptr_eq(&first, &second));}++#[test]+fn static_public_suffix_tables_are_compact_and_allocation_free() {+ assert_eq!(std::mem::size_of::<StaticPublicSuffixList>(), 0);+ assert_eq!(STATIC_RULE_COUNT, 8_818);+ assert!(static_table_size_bytes() < 256 * 1024);+}++#[test]+fn deeper_exact_branch_does_not_shadow_a_parent_wildcard() {+ assert!(host_is_public_suffix("oci.customer-oci.com"));+ assert_eq!(+ registrable_site_host("oci.customer-oci.com"),+ "oci.customer-oci.com"+ );+ assert_eq!(+ registrable_site_host("tenant.oci.customer-oci.com"),+ "tenant.oci.customer-oci.com"+ );+}++#[test]+fn static_public_suffix_lookup_preserves_the_previous_site_semantics() {+ let source = include_str!("data/public_domains.txt");+ let static_list = StaticPublicSuffixList;+ let mut legacy_rules = HashSet::new();+ let mut legacy_wildcards = HashSet::new();+ let mut legacy_exceptions = HashSet::new();+ for rule in source+ .lines()+ .map(str::trim)+ .filter(|line| !line.is_empty() && !line.starts_with("//"))+ {+ if let Some(rule) = rule.strip_prefix('!') {+ legacy_exceptions.insert(rule);+ } else if let Some(rule) = rule.strip_prefix("*.") {+ legacy_wildcards.insert(rule);+ } else {+ legacy_rules.insert(rule);+ }+ }++ for raw_rule in source+ .lines()+ .map(str::trim)+ .filter(|line| !line.is_empty() && !line.starts_with("//"))+ {+ let rule = raw_rule.strip_prefix('!').unwrap_or(raw_rule);+ let base = rule.strip_prefix("*.").unwrap_or(rule);+ for domain in [+ base.to_owned(),+ format!("probe.{base}"),+ format!("deep.probe.{base}"),+ ] {+ let (legacy_suffix, legacy_domain) = legacy_suffix_pair(+ &domain,+ &legacy_rules,+ &legacy_wildcards,+ &legacy_exceptions,+ );+ let static_suffix = static_list+ .suffix(domain.as_bytes())+ .and_then(|suffix| std::str::from_utf8(suffix.as_bytes()).ok());+ assert_eq!(+ static_suffix,+ Some(legacy_suffix),+ "suffix differs for {domain} while checking rule {raw_rule}"+ );++ let static_domain = static_list+ .domain(domain.as_bytes())+ .and_then(|domain| std::str::from_utf8(domain.as_bytes()).ok())+ .unwrap_or(&domain);+ assert_eq!(+ static_domain, legacy_domain,+ "site key differs for {domain} while checking rule {raw_rule}"+ );+ assert_eq!(registrable_site_host(&domain), legacy_domain);+ }+ }+}++fn legacy_suffix_pair<'a>(+ domain: &'a str,+ rules: &HashSet<&str>,+ wildcards: &HashSet<&str>,+ exceptions: &HashSet<&str>,+) -> (&'a str, &'a str) {+ let domain = domain.trim_start_matches('.');+ let mut suffix = domain;+ let mut previous_suffix = domain;++ for (index, _) in domain.match_indices('.') {+ let next_suffix = &domain[index + 1..];+ if exceptions.contains(suffix) {+ return (next_suffix, suffix);+ }+ if wildcards.contains(next_suffix) || rules.contains(suffix) {+ return (suffix, previous_suffix);+ }+ previous_suffix = suffix;+ suffix = next_suffix;+ }++ (suffix, previous_suffix)+}