The design language that makes your AI harness better at design.
JavaScript
Consent walls count the page's shadow text and see under hidden roots
Paul Bakaus5b51ddd6b861
diff --git a/crates/browser/src/consent.rs b/crates/browser/src/consent.rsindex 4a076b053..87ee1657e 100644--- a/crates/browser/src/consent.rs+++ b/crates/browser/src/consent.rs@@ -219,8 +219,11 @@ const SHOWING_JS: &str = r#"const q = s => { try { return Array.from(document.qu};const showing = el => {if (!el || !el.isConnected) return false;+ // A hidden or transparent root hides everything under it, but a root at+ // `visibility: hidden` does not: a child at `visibility: visible`+ // paints, so the walk below still looks at it.try {- if (el.checkVisibility && !el.checkVisibility({ opacityProperty: true, visibilityProperty: true })) return false;+ if (el.checkVisibility && !el.checkVisibility({ opacityProperty: true })) return false;} catch (e) {}if (boxShows(el)) return true;for (const scope of [el, el.shadowRoot]) {@@ -235,7 +238,9 @@ const SHOWING_JS: &str = r#"const q = s => { try { return Array.from(document.qu/// are showing), `consentChars` (the visible text inside those roots, shadow/// roots included), `consentShadowChars` (the part of it `body.innerText`/// does not see: shadow-root text, and a root outside `<body>`),-/// `consentFrames` (sizable frames inside those roots) and+/// `consentPageShadowChars` (visible text in the page's own open shadow+/// roots, outside those roots), `consentFrames` (sizable frames inside+/// those roots) and/// `consentOutside` (visible form controls and sizable media outside them,/// counted up to 20).pub fn probe_fragment() -> String {@@ -292,6 +297,18 @@ pub fn probe_fragment() -> String {// {media_px} square pixels. A sign-in form or an image-first page has// little text of its own and is still a page.let consentOutside = 0;+ // `consentPageShadowChars` is the visible text in the page's own open+ // shadow roots (a web-component app), which `body.innerText` does not see+ // either: it counts as text outside the managers.+ let consentPageShadowChars = 0;+ if (consentRoots.length && document.body) {{+ const hosts = document.body.querySelectorAll('*');+ for (let i = 0; i < hosts.length && i < 30000; i++) {{+ const host = hosts[i];+ if (!host.shadowRoot || consentRoots.some(r => r.contains(host))) continue;+ for (const child of host.shadowRoot.children) if (boxShows(child)) consentPageShadowChars += visibleText(child);+ }}+ }}if (consentRoots.length) {{const inside = el => consentRoots.some(r => r.contains(el));for (const el of q({controls})) {{diff --git a/crates/browser/src/validity.rs b/crates/browser/src/validity.rsindex ab4af8a1d..bef99b9c2 100644--- a/crates/browser/src/validity.rs+++ b/crates/browser/src/validity.rs@@ -107,6 +107,7 @@ pub fn probe_js() -> String {consent,consentChars,consentShadowChars,+ consentPageShadowChars,consentFrames,consentOutside,}};@@ -133,6 +134,9 @@ pub struct PageProbe {/// `body.innerText`) does not see: shadow-root text, and a root outside/// `<body>`.pub consent_shadow_chars: u64,+ /// Visible text in the page's own open shadow roots, outside the+ /// managers' roots: page text `text_chars` does not see.+ pub consent_page_shadow_chars: u64,/// Sizable frames inside those roots: a message drawn in an iframe,/// whose text the page cannot read.pub consent_frames: u64,@@ -157,6 +161,11 @@ impl PageProbe {consent_chars: v.get("consentChars").and_then(Value::as_f64).unwrap_or(0.0).max(0.0) as u64,consent_outside: v.get("consentOutside").and_then(Value::as_f64).unwrap_or(0.0).max(0.0) as u64,consent_shadow_chars: v.get("consentShadowChars").and_then(Value::as_f64).unwrap_or(0.0).max(0.0) as u64,+ consent_page_shadow_chars: v+ .get("consentPageShadowChars")+ .and_then(Value::as_f64)+ .unwrap_or(0.0)+ .max(0.0) as u64,consent_frames: v.get("consentFrames").and_then(Value::as_f64).unwrap_or(0.0).max(0.0) as u64,}}@@ -179,6 +188,9 @@ impl PageProbe {if self.consent_shadow_chars > 0 {v["consentShadowChars"] = json!(self.consent_shadow_chars);}+ if self.consent_page_shadow_chars > 0 {+ v["consentPageShadowChars"] = json!(self.consent_page_shadow_chars);+ }v}@@ -373,7 +385,8 @@ pub fn classify_with(response: Option<&DocumentResponse>, probe: &PageProbe, con// shadow root, and the manager's count includes it: both are read// over the same text here, while the challenge checks above keep the// page's own count.- let total = probe.text_chars + probe.consent_shadow_chars;+ // The page's own shadow-root text is outside the managers too.+ let total = probe.text_chars + probe.consent_shadow_chars + probe.consent_page_shadow_chars;let outside = total.saturating_sub(probe.consent_chars);if outside < CONSENT_WALL_OUTSIDE_CHARS&& probe.consent_outside == 0@@ -493,6 +506,21 @@ mod tests {));}+ #[test]+ fn the_pages_own_shadow_text_is_outside_the_banner() {+ // A web-component app: its text lives in its own shadow roots, under+ // a light-DOM OneTrust banner that holds all of body.innerText.+ let mut p = probe("Example", 400, &[]);+ p.consent = vec!["OneTrust".into()];+ p.consent_chars = 400;+ assert!(matches!(+ classify(Some(&resp(200, &[])), &p),+ PageValidity::Blocked { kind: BlockKind::ConsentWall, .. }+ ));+ p.consent_page_shadow_chars = 600;+ assert_eq!(classify(Some(&resp(200, &[])), &p), PageValidity::Ok);+ }+#[test]fn a_short_page_under_a_banner_is_the_page() {// The review's sign-in page: an h1, two fields, a hint and a buttondiff --git a/crates/browser/tests/consent_hiding.rs b/crates/browser/tests/consent_hiding.rsindex d120fdb8b..2cc07c560 100644--- a/crates/browser/tests/consent_hiding.rs+++ b/crates/browser/tests/consent_hiding.rs@@ -359,3 +359,22 @@ fn a_root_matched_by_two_selectors_is_counted_once() {let url = format!("http://127.0.0.1:{port}/quantcast-short-page.html");engine.detect_url_scan(&url, &ScanOptions::default()).expect("a short page is a page");}++#[test]+fn a_web_component_page_under_a_banner_is_scanned() {+ let Some(engine) = engine() else { return };+ let port = serve();+ let url = format!("http://127.0.0.1:{port}/shadow-app-page.html");+ engine+ .detect_url_scan(&url, &ScanOptions::default())+ .expect("text in the page's own shadow root is the page");+}++#[test]+fn a_wall_under_a_visibility_hidden_root_is_refused() {+ let Some(engine) = engine() else { return };+ let port = serve();+ let url = format!("http://127.0.0.1:{port}/hidden-host-wall.html");+ let err = engine.detect_url(&url, &ScanOptions::default()).expect_err("a visible dialog under a hidden root is a wall");+ assert!(err.message.starts_with("the page is a consent wall, not the site (consent manager OneTrust, "), "{}", err.message);+}diff --git a/tests/fixtures/consent/hidden-host-wall.html b/tests/fixtures/consent/hidden-host-wall.htmlnew file mode 100644index 000000000..742a208a3--- /dev/null+++ b/tests/fixtures/consent/hidden-host-wall.html@@ -0,0 +1,26 @@+<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8">+ <meta name="viewport" content="width=device-width, initial-scale=1.0">+ <title>Consent hiding: a wall under a visibility-hidden root</title>+ <style>+ /* A OneTrust root at visibility: hidden whose unnamed dialog sets+ visibility: visible, so the dialog paints. The page is only that+ dialog, so it is refused as a consent wall. Pinned by+ crates/browser/tests/consent_hiding.rs. */+ body { font: 16px/1.5 system-ui, sans-serif; margin: 0; background: #f3f4f6; }+ #onetrust-consent-sdk { visibility: hidden; }+ .ot-dialog { visibility: visible; position: fixed; top: 80px; left: 50%; width: 640px; margin-left: -320px; padding: 32px; background: #fff; }+ </style>+</head>+<body>+ <div id="onetrust-consent-sdk">+ <div class="ot-dialog" role="dialog" aria-label="Privacy preferences">+ <h2>Your privacy choices</h2>+ <p>We and our partners store and access information on your device, such as cookies, and process personal data such as unique identifiers and browsing data, to personalise content and advertising, to measure the performance of content and advertising, and to develop and improve our products. You can accept all of these purposes, reject them, or choose which ones you allow.</p>+ <button type="button">Accept all</button> <button type="button">Reject all</button>+ </div>+ </div>+</body>+</html>diff --git a/tests/fixtures/consent/shadow-app-page.html b/tests/fixtures/consent/shadow-app-page.htmlnew file mode 100644index 000000000..b5b7c908d--- /dev/null+++ b/tests/fixtures/consent/shadow-app-page.html@@ -0,0 +1,29 @@+<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8">+ <meta name="viewport" content="width=device-width, initial-scale=1.0">+ <title>Consent hiding: a web-component page under a banner</title>+ <style>+ /* A page whose own text lives in an open shadow root, under a+ light-DOM OneTrust dialog that holds all of body.innerText. The+ shadow text is the page's, so it is scanned, not refused as a consent+ wall. Pinned by crates/browser/tests/consent_hiding.rs. */+ body { font: 16px/1.5 system-ui, sans-serif; margin: 0; background: #fff; }+ #onetrust-pc-sdk { position: fixed; top: 80px; left: 50%; width: 640px; margin-left: -320px; padding: 32px; background: #fff; }+ </style>+</head>+<body>+ <x-app></x-app>+ <div id="onetrust-consent-sdk">+ <div id="onetrust-pc-sdk" role="dialog" aria-label="Privacy preferences">+ <h2>Your privacy choices</h2>+ <p>We and our partners store and access information on your device, such as cookies, and process personal data such as unique identifiers and browsing data, to personalise content and advertising and to measure its performance.</p>+ </div>+ </div>+ <script>+ const root = document.querySelector('x-app').attachShadow({ mode: 'open' });+ root.innerHTML = '<main style="display:block;padding:24px"><h1>Field notes</h1><p>The ferry leaves the harbour at seven, crosses the strait in a little under two hours and reaches the island before the morning market closes. Most visitors walk from the pier to the old town, stop at the bakery by the church and take the coast path north towards the lighthouse.</p></main>';+ </script>+</body>+</html>