Skip to content

The design language that makes your AI harness better at design.

JavaScript

Consent wall gate refuses only pages hiding would leave empty

Paul Bakaus84755b48ab54
diff --git a/crates/browser/src/consent.rs b/crates/browser/src/consent.rs
index e053a290c..bc572d0dc 100644
--- a/crates/browser/src/consent.rs
+++ b/crates/browser/src/consent.rs
@@ -146,7 +146,7 @@ pub const CONSENT_MANAGERS: &[ConsentManager] = &[
},
ConsentManager {
name: "Borlabs Cookie",
- roots: &["#BorlabsCookieBox", "#BorlabsCookieBoxWrap", ".brlbs-cmpnt-container"],
+ roots: &["#BorlabsCookieBox", "#BorlabsCookieBoxWrap"],
backdrops: &[],
html_lock_classes: &[],
body_lock_classes: &[],
@@ -226,8 +226,10 @@ const SHOWING_JS: &str = r#"const q = s => { try { return Array.from(document.qu
return false;
};"#;
-/// A probe fragment: `consentShowing(managers)` returns the names of the
-/// managers whose roots are showing, and the visible text inside those roots.
+/// A probe fragment. Defines `consent` (the names of the managers whose roots
+/// are showing), `consentChars` (the visible text inside those roots) and
+/// `consentOutside` (visible form controls and sizable media outside them,
+/// counted up to 20).
pub fn probe_fragment() -> String {
format!(
r#"const consentManagers = {managers};
@@ -243,23 +245,53 @@ pub fn probe_fragment() -> String {
for (const el of consentRoots) {{
if (consentRoots.some(o => o !== el && o.contains(el))) continue;
consentChars += (el.innerText || '').replace(/\s+/g, ' ').trim().length;
+ }}
+ // Content outside the managers that is not text: a form control of any
+ // size, or an image, video, canvas, SVG or frame of at least
+ // {media_px} square pixels. A sign-in form or an image-first page has
+ // little text of its own and is still a page.
+ let consentOutside = 0;
+ if (consentRoots.length) {{
+ const inside = el => consentRoots.some(r => r.contains(el));
+ for (const el of q({controls})) {{
+ if (consentOutside >= 20) break;
+ if (!inside(el) && boxShows(el)) consentOutside++;
+ }}
+ for (const el of q({media})) {{
+ if (consentOutside >= 20) break;
+ if (inside(el) || !boxShows(el)) continue;
+ const r = el.getBoundingClientRect();
+ if (r.width * r.height >= {media_px}) consentOutside++;
+ }}
}}"#,
managers = managers_json(),
showing = SHOWING_JS,
+ controls = json!(OUTSIDE_CONTROLS),
+ media = json!(OUTSIDE_MEDIA),
+ media_px = OUTSIDE_MEDIA_MIN_AREA,
)
}
+/// Form controls that make a page more than its consent wall, wherever they
+/// sit outside a manager's roots.
+pub const OUTSIDE_CONTROLS: &str = "input:not([type='hidden']), select, textarea, button";
+/// Media that make a page more than its consent wall, when at least
+/// [`OUTSIDE_MEDIA_MIN_AREA`] square pixels (a logo is smaller).
+pub const OUTSIDE_MEDIA: &str = "img, svg, video, canvas, iframe, object, embed";
+pub const OUTSIDE_MEDIA_MIN_AREA: u32 = 10_000;
+
/// The hide step. Idempotent: run it once before the reveal sweep and again
/// right before the capture, to catch a manager that injects its banner late.
-/// Returns `{ hidden: [name], matched: { name: [selector] }, unlocked: [what] }`
-/// for the managers it found showing this time.
+/// Returns `{ hidden: [name], matched: { name: [selector] }, unlocked: [what],
+/// changed }` for the managers it found showing this time; `changed` is
+/// whether this run altered the page (a new rule, an inline hide, an unlock).
pub fn hide_js() -> String {
format!(
r#"(() => {{
const managers = {managers};
const STYLE_ID = {style_id};
{showing}
- const out = {{ hidden: [], matched: {{}}, unlocked: [] }};
+ const out = {{ hidden: [], matched: {{}}, unlocked: [], changed: false }};
const present = [];
for (const m of managers) {{
const sels = m.roots.concat(m.backdrops).filter(s => q(s).length > 0);
@@ -289,17 +321,18 @@ pub fn hide_js() -> String {
const rules = [];
for (const {{ sels }} of present) for (const s of sels) rules.push(s + ' {{ display: none !important; }}');
const text = rules.join('\n');
- if (style.textContent !== text) style.textContent = text;
+ if (style.textContent !== text) {{ style.textContent = text; out.changed = true; }}
// A manager that sets `display` inline with !important outranks the rule.
for (const {{ sels }} of present) for (const s of sels) for (const el of q(s)) {{
try {{
- if (getComputedStyle(el).display !== 'none') el.style.setProperty('display', 'none', 'important');
+ if (getComputedStyle(el).display !== 'none') {{ el.style.setProperty('display', 'none', 'important'); out.changed = true; }}
}} catch (e) {{}}
}}
// Undo the scroll lock the manager applied, and only that: its own classes
// on <html> and <body>, and an inline overflow: hidden on either while its
- // backdrop was up (a modal consent layer locks the page that way). A
- // site's own overflow is left alone.
+ // backdrop was up (a modal consent layer locks the page that way) and the
+ // page is not an app shell that scrolls inside itself. A site's own
+ // overflow is left alone.
const html = document.documentElement;
const body = document.body;
for (const {{ m }} of present) {{
@@ -307,7 +340,22 @@ pub fn hide_js() -> String {
for (const c of m.htmlLock) if (html.classList.contains(c)) {{ html.classList.remove(c); out.unlocked.push('html.' + c); }}
if (body) for (const c of m.bodyLock) if (body.classList.contains(c)) {{ body.classList.remove(c); out.unlocked.push('body.' + c); }}
}}
- if (showingBackdrop.length) {{
+ // An app shell locks <body> itself and scrolls an inner container that
+ // fills the viewport. That lock is the site's, whatever the manager did,
+ // so it stays.
+ const appShell = () => {{
+ const all = body ? body.querySelectorAll('*') : [];
+ const vh = window.innerHeight || 0;
+ for (let i = 0; i < all.length && i < 5000; i++) {{
+ const el = all[i];
+ if (el.clientHeight < vh * 0.5 || el.scrollHeight <= el.clientHeight + 1) continue;
+ if (present.some(({{ sels }}) => sels.some(s => {{ try {{ return !!el.closest(s); }} catch (e) {{ return false; }} }}))) continue;
+ const oy = getComputedStyle(el).overflowY;
+ if (oy === 'auto' || oy === 'scroll') return true;
+ }}
+ return false;
+ }};
+ if (showingBackdrop.length && !appShell()) {{
for (const [el, tag] of [[html, 'html'], [body, 'body']]) {{
if (!el) continue;
for (const p of ['overflow', 'overflow-y']) {{
@@ -318,6 +366,7 @@ pub fn hide_js() -> String {
}}
}}
}}
+ if (out.unlocked.length) out.changed = true;
return out;
}})()"#,
managers = managers_json(),
diff --git a/crates/browser/src/lib.rs b/crates/browser/src/lib.rs
index cdb63f9e8..26cc76e0b 100644
--- a/crates/browser/src/lib.rs
+++ b/crates/browser/src/lib.rs
@@ -782,7 +782,10 @@ fn scan_page(
/// Probe the loaded page and classify it. A probe that fails because the page
/// navigated underneath it (a challenge redirecting on its own) is retried
/// once after a short wait.
-fn check_validity(page: &mut Page<'_>) -> Result<(Option<DocumentResponse>, PageProbe, PageValidity), CdpError> {
+fn check_validity(
+ page: &mut Page<'_>,
+ consent_wall: bool,
+) -> Result<(Option<DocumentResponse>, PageProbe, PageValidity), CdpError> {
let js = validity::probe_js();
let raw = match page.evaluate_value(&js) {
Ok(v) => v,
@@ -795,7 +798,7 @@ fn check_validity(page: &mut Page<'_>) -> Result<(Option<DocumentResponse>, Page
let response = page
.main_document_response()
.and_then(|r| DocumentResponse::from_cdp(&r));
- let verdict = validity::classify(response.as_ref(), &probe);
+ let verdict = validity::classify_with(response.as_ref(), &probe, consent_wall);
Ok((response, probe, verdict))
}
@@ -832,9 +835,12 @@ fn scan_page_inner(
});
}
- // A challenge or error page is not the site: refuse to report on it.
+ // A challenge or error page is not the site: refuse to report on it. Nor
+ // is a consent wall, unless the scan keeps the banners: then the banner
+ // is what the caller asked to read.
+ let hide_consent = !options.keep_consent_banners;
let (response, probe, verdict) =
- step(profile, "load", "validity", url, || check_validity(page)).map_err(cdp_err)?;
+ step(profile, "load", "validity", url, || check_validity(page, hide_consent)).map_err(cdp_err)?;
let blocked = verdict.error_message();
if let Some((ev, _)) = evidence.as_mut() {
ev.response = response;
@@ -855,11 +861,11 @@ fn scan_page_inner(
// lock they applied, before anything is swept or measured. Never a click
// and never a consent choice: an injected style paints the page without
// the manager's layer. The validity probe above ran first, so a page that
- // is only a consent wall was refused rather than scanned empty.
- let hide_consent = !options.keep_consent_banners;
+ // is only a consent wall was refused rather than scanned empty. Every hide
+ // pass is best-effort: a failed one leaves the banner showing, the way a
+ // scan with `--no-consent-hiding` reads the page.
if hide_consent {
- step(profile, "load", "hide-consent", url, || hide_consent_banners(page, consent))
- .map_err(cdp_err)?;
+ step(profile, "load", "hide-consent", url, || hide_consent_banners(page, consent));
}
// Inject the plain-JS snapshot producer (no WebAssembly runs in the page).
@@ -883,8 +889,7 @@ fn scan_page_inner(
// A manager that injects its banner late (after load, or on the first
// scroll) is hidden here, before the capture every pass reads.
if hide_consent {
- step(profile, "scan", "hide-consent", url, || hide_consent_banners(page, consent))
- .map_err(cdp_err)?;
+ step(profile, "scan", "hide-consent", url, || hide_consent_banners(page, consent));
}
// The one capture every pass below reads: the rule pass, the hidden-text
@@ -892,7 +897,13 @@ fn scan_page_inner(
// answers is answered for all three. The sweep leaves the page revealed and
// scrolled to the top, which is the scroll-0 snapshot the in-page path
// measured and analyzed.
- let base_json = snapshot_engine::capture_snapshot_json(page).map_err(cdp_err)?;
+ let mut base_json = snapshot_engine::capture_snapshot_json(page).map_err(cdp_err)?;
+ // A banner that arrived around the capture is hidden, and the page
+ // captured again, so the capture, the live hit tests and the pixel reads
+ // all see the page without it.
+ if hide_consent && step(profile, "scan", "hide-consent", url, || hide_consent_banners(page, consent)) {
+ base_json = snapshot_engine::capture_snapshot_json(page).map_err(cdp_err)?;
+ }
let base = snapshot_engine::parse_snapshot(&base_json).map_err(cdp_err)?;
if let Some((ev, _)) = evidence.as_mut() {
ev.scan_snapshot = Some(base_json);
@@ -1203,11 +1214,17 @@ fn capture_post_scan(
}
/// Run the consent hide step ([`consent::hide_js`]) and fold what it hid into
-/// `report`. Idempotent; each run also catches a banner injected since the last.
-fn hide_consent_banners(page: &mut Page<'_>, report: &mut ConsentReport) -> Result<(), CdpError> {
- let v = page.evaluate_value(&consent::hide_js())?;
- report.merge(&v);
- Ok(())
+/// `report`. Idempotent; each run also catches a banner injected since the
+/// last. Best-effort: a failed evaluation hides nothing and is not an error.
+/// Returns whether this run changed the page.
+fn hide_consent_banners(page: &mut Page<'_>, report: &mut ConsentReport) -> bool {
+ match page.evaluate_value(&consent::hide_js()) {
+ Ok(v) => {
+ report.merge(&v);
+ v.get("changed").and_then(Value::as_bool).unwrap_or(false)
+ }
+ Err(_) => false,
+ }
}
/// The `measureContentHiddenAfterReveal` reveal sweep: scroll the page top to
diff --git a/crates/browser/src/validity.rs b/crates/browser/src/validity.rs
index 43ae5e3b0..7912c55ca 100644
--- a/crates/browser/src/validity.rs
+++ b/crates/browser/src/validity.rs
@@ -14,11 +14,15 @@
//! form or be titled "Access denied" in a CMS without being a block page.
//!
//! The probe also names the known consent managers ([`crate::consent`]) whose
-//! banner is showing, and how much visible text sits inside them. It runs
-//! before the scan hides those banners, so a page that is nothing but a
-//! consent wall (a small page whose readable text is almost all the
-//! manager's) is refused as one, instead of being scanned empty and reported
-//! clean.
+//! banner is showing, how much visible text sits inside them, and what content
+//! lies outside them. It runs before the scan hides those banners, so a page
+//! that is nothing but a consent wall (hiding the manager would leave next to
+//! nothing: no more than a line of text, no form control, no sizable image)
+//! is refused as one, instead of being scanned empty and reported clean. A
+//! short page that carries an ordinary banner (a sign-in form, a splash page,
+//! an image-first portfolio) is a page, and is scanned. With the banners kept
+//! (`--no-consent-hiding`) the gate is off: that scan reads the banner, which
+//! is what the flag asks for.
use serde_json::{json, Value};
@@ -28,9 +32,11 @@ pub const SMALL_PAGE_CHARS: u64 = 3000;
/// A small page with a known consent manager showing is a consent wall when
/// fewer than this many visible characters lie outside the manager's roots,
-/// and the manager holds at least [`CONSENT_WALL_RATIO`] times as many.
-pub const CONSENT_WALL_OUTSIDE_CHARS: u64 = 100;
-pub const CONSENT_WALL_RATIO: u64 = 3;
+/// nothing else shows outside them (no form control and no image, video or
+/// frame of 10,000 square pixels or more, see [`crate::consent`]), and the
+/// manager holds at least [`CONSENT_WALL_MIN_CHARS`] characters itself.
+pub const CONSENT_WALL_OUTSIDE_CHARS: u64 = 20;
+pub const CONSENT_WALL_MIN_CHARS: u64 = 100;
/// Titles challenge and block pages use, lowercased and trimmed. Matched by
/// prefix, so "Access Denied" also covers "Access Denied - Reference #18...".
@@ -98,6 +104,7 @@ pub fn probe_js() -> String {
markers: found,
consent,
consentChars,
+ consentOutside,
}};
}})()"#,
markers = Value::Array(markers),
@@ -116,6 +123,8 @@ pub struct PageProbe {
pub consent: Vec<String>,
/// Visible characters inside those managers' roots.
pub consent_chars: u64,
+ /// Visible form controls and sizable media outside those roots (up to 20).
+ pub consent_outside: u64,
}
impl PageProbe {
@@ -135,6 +144,7 @@ impl PageProbe {
.map(|a| a.iter().filter_map(Value::as_str).map(String::from).collect())
.unwrap_or_default(),
consent_chars: v.get("consentChars").and_then(Value::as_f64).unwrap_or(0.0).max(0.0) as u64,
+ consent_outside: v.get("consentOutside").and_then(Value::as_f64).unwrap_or(0.0).max(0.0) as u64,
}
}
@@ -146,6 +156,7 @@ impl PageProbe {
"markers": self.markers,
"consent": self.consent,
"consentChars": self.consent_chars,
+ "consentOutside": self.consent_outside,
})
}
}
@@ -267,7 +278,9 @@ impl PageValidity {
/// Classify a loaded page. `response` is `None` when no main-document
/// response was seen (a `file://` URL, or a same-document navigation).
-pub fn classify(response: Option<&DocumentResponse>, probe: &PageProbe) -> PageValidity {
+/// `consent_wall` turns the consent-wall gate on; a scan that keeps the
+/// banners turns it off.
+pub fn classify_with(response: Option<&DocumentResponse>, probe: &PageProbe, consent_wall: bool) -> PageValidity {
let mut challenge: Vec<String> = Vec::new();
if let Some(r) = response {
if let Some(action) = r.header("x-amzn-waf-action") {
@@ -316,14 +329,18 @@ pub fn classify(response: Option<&DocumentResponse>, probe: &PageProbe) -> PageV
evidence: vec![format!("HTTP {status}")],
};
}
- if small && !probe.consent.is_empty() {
+ if consent_wall && small && !probe.consent.is_empty() {
let outside = probe.text_chars.saturating_sub(probe.consent_chars);
- if outside < CONSENT_WALL_OUTSIDE_CHARS && probe.consent_chars >= CONSENT_WALL_RATIO * outside {
+ if outside < CONSENT_WALL_OUTSIDE_CHARS
+ && probe.consent_outside == 0
+ && probe.consent_chars >= CONSENT_WALL_MIN_CHARS
+ {
return PageValidity::Blocked {
kind: BlockKind::ConsentWall,
evidence: vec![
format!("consent manager {}", probe.consent.join(", ")),
format!("{outside} of {} visible characters outside it", probe.text_chars),
+ "no control or image outside it".to_string(),
],
};
}
@@ -331,6 +348,11 @@ pub fn classify(response: Option<&DocumentResponse>, probe: &PageProbe) -> PageV
PageValidity::Ok
}
+/// [`classify_with`] with every gate on, the consent wall included.
+pub fn classify(response: Option<&DocumentResponse>, probe: &PageProbe) -> PageValidity {
+ classify_with(response, probe, true)
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -355,7 +377,7 @@ mod tests {
#[test]
fn a_page_that_is_only_a_consent_wall_is_refused() {
- let mut p = probe("Example", 900, &[]);
+ let mut p = probe("Example", 872, &[]);
p.consent = vec!["OneTrust".into()];
p.consent_chars = 860;
let v = classify(Some(&resp(200, &[])), &p);
@@ -363,10 +385,43 @@ mod tests {
v,
PageValidity::Blocked {
kind: BlockKind::ConsentWall,
- evidence: vec!["consent manager OneTrust".into(), "40 of 900 visible characters outside it".into()],
+ evidence: vec![
+ "consent manager OneTrust".into(),
+ "12 of 872 visible characters outside it".into(),
+ "no control or image outside it".into(),
+ ],
}
);
assert!(v.error_message().unwrap().starts_with("the page is a consent wall"));
+ // A scan that keeps the banners reads the wall instead.
+ assert_eq!(classify_with(Some(&resp(200, &[])), &p, false), PageValidity::Ok);
+ }
+
+ #[test]
+ fn a_short_page_under_a_banner_is_the_page() {
+ // The review's sign-in page: an h1, two fields, a hint and a button
+ // under a OneTrust bar. 48 characters of its own and four controls.
+ let mut p = probe("Sign in", 411, &[]);
+ p.consent = vec!["OneTrust".into()];
+ p.consent_chars = 363;
+ p.consent_outside = 4;
+ assert_eq!(classify(Some(&resp(200, &[])), &p), PageValidity::Ok);
+ // An image-first page: a caption's worth of text and a large image.
+ let mut p = probe("Portfolio", 380, &[]);
+ p.consent = vec!["Cookiebot".into()];
+ p.consent_chars = 372;
+ p.consent_outside = 1;
+ assert_eq!(classify(Some(&resp(200, &[])), &p), PageValidity::Ok);
+ // A line of its own text and nothing else is still more than a wall.
+ let mut p = probe("Splash", 400, &[]);
+ p.consent = vec!["OneTrust".into()];
+ p.consent_chars = 370;
+ assert_eq!(classify(Some(&resp(200, &[])), &p), PageValidity::Ok);
+ // An empty page under a two-word banner is not a wall either.
+ let mut p = probe("Example", 60, &[]);
+ p.consent = vec!["OneTrust".into()];
+ p.consent_chars = 60;
+ assert_eq!(classify(Some(&resp(200, &[])), &p), PageValidity::Ok);
}
#[test]
diff --git a/crates/browser/tests/consent_hiding.rs b/crates/browser/tests/consent_hiding.rs
index c8710ff29..510366bae 100644
--- a/crates/browser/tests/consent_hiding.rs
+++ b/crates/browser/tests/consent_hiding.rs
@@ -10,7 +10,10 @@
//! underlay and an inline scroll lock on `<body>`: caught by the hide pass
//! after the reveal sweep, recorded in the evidence, and the lock undone.
//! - A notice the site wrote itself (generic names, no vendor) stays.
-//! - A page that is only a consent wall is refused as one.
+//! - A page that is only a consent wall is refused as one; with the banners
+//! kept it is scanned, since the banner is what that scan asked for.
+//! - A short page under an ordinary banner (a sign-in form) is not a wall.
+//! - An app shell's own body lock stays when a manager's backdrop shows.
use std::collections::HashMap;
use std::io::{Read, Write};
@@ -181,10 +184,10 @@ fn a_page_that_is_only_a_consent_wall_is_refused() {
let Some(engine) = engine() else { return };
let port = serve();
let url = format!("http://127.0.0.1:{port}/wall.html");
- for options in [ScanOptions::default(), KEEP] {
- let err = engine.detect_url(&url, &options).expect_err("a consent wall must not scan");
- assert!(err.message.starts_with("the page is a consent wall, not the site (consent manager OneTrust, "), "{}", err.message);
- }
+ let err = engine.detect_url(&url, &ScanOptions::default()).expect_err("a consent wall must not scan");
+ assert!(err.message.starts_with("the page is a consent wall, not the site (consent manager OneTrust, "), "{}", err.message);
+ // With the banners kept, the gate is off and the dialog is what is read.
+ engine.detect_url(&url, &KEEP).expect("a kept consent wall scans");
let mut browser = engine.launch().expect("launch");
let (findings, evidence) =
detect_url_evidence(&mut browser, &url, &ScanOptions::default(), "load", 100, &EvidenceRequest::default())
@@ -195,3 +198,36 @@ fn a_page_that_is_only_a_consent_wall_is_refused() {
assert_eq!(evidence.probe.as_ref().unwrap().consent, vec!["OneTrust"]);
assert!(evidence.consent.is_none());
}
+
+#[test]
+fn a_short_page_under_a_banner_is_scanned_not_refused() {
+ let Some(engine) = engine() else { return };
+ let port = serve();
+ let url = format!("http://127.0.0.1:{port}/short-page.html");
+ let scan = engine.detect_url_scan(&url, &ScanOptions::default()).expect("a sign-in page is a page");
+ let found = flagged(&scan.findings);
+ assert!(has(&found, "low-contrast", "#hint"), "{found:#?}");
+ assert!(!has(&found, "low-contrast", "#onetrust-policy-text"), "{found:#?}");
+ let kept = engine.detect_url_scan(&url, &KEEP).expect("kept, it scans too");
+ let found = flagged(&kept.findings);
+ assert!(has(&found, "low-contrast", "#onetrust-policy-text"), "{found:#?}");
+}
+
+#[test]
+fn an_app_shells_own_body_lock_stays() {
+ let Some(engine) = engine() else { return };
+ let port = serve();
+ let url = format!("http://127.0.0.1:{port}/app-shell.html");
+ let mut browser = engine.launch().expect("launch");
+ let (_, evidence) =
+ detect_url_evidence(&mut browser, &url, &ScanOptions::default(), "load", 100, &EvidenceRequest::default())
+ .expect("scan");
+ browser.close();
+ let consent = evidence.consent.as_ref().expect("consent report");
+ assert_eq!(consent.hidden, vec!["OneTrust"]);
+ // The backdrop was showing, so only the app-shell check kept the lock.
+ assert!(consent.matched[0].1.iter().any(|s| s == ".onetrust-pc-dark-filter"), "{:?}", consent.matched);
+ assert!(consent.unlocked.is_empty(), "{:?}", consent.unlocked);
+ let snapshot = evidence.scan_snapshot.as_deref().unwrap();
+ assert!(snapshot.contains("overflow: hidden"), "the body keeps its own lock");
+}
diff --git a/docs/CLI-CONTRACT.md b/docs/CLI-CONTRACT.md
index 9234da819..7c500753d 100644
--- a/docs/CLI-CONTRACT.md
+++ b/docs/CLI-CONTRACT.md
@@ -349,8 +349,8 @@ Optional keys added later by engines (appended after the above): `ignoreValue` (
- **Purple headings in the brand hue** (corpus decision r3-23-ai-color-palette-brand-hue). The heading forms of `ai-color-palette` (a snippet ending ` on heading`) do not report when the heading's computed `color` is within 12 degrees of hue of a brand surface on the same page: the solid `background-color` (alpha at least 0.9, chroma at least 50, no gradient painted over it) of the logo (an element whose `id`, `class`, `alt` or `aria-label` contains `logo`; its text colour too when it has text of its own), the nav bar (`nav`, `header`, `role=navigation|banner`, at least half the viewport wide), or a band across the page (at least 90% of the viewport's width) covering a fifth of the viewport's area or more. Headings are never brand surfaces, and the gradient and neon forms keep reporting: purple that shows up only in headings and gradients is the tell. URL engine only (the static engine has no rendered page to read surfaces from).
- **Contrast ratios in snippets.** Every `low-contrast` producer (the element text path shared by both engines, which covers links, spans and placeholders, the static engine's `:hover state`, and the URL engine's `browser contrast` sampling and `pixel contrast` passes, verdict and median) prints a ratio against its threshold: one decimal; two when one decimal would round a failing ratio up to the threshold; and those two cut, not rounded, when rounding them would reach it too. A ratio under the bar never prints as the bar: `4.49:1 (need 4.5:1)`, never `4.5:1` or `4.50:1`. A ratio at or above the threshold, such as a median over it, prints one decimal.
- **Page validity gate.** After `goto` (and settle), before injecting anything, the engine classifies the page (`crates/browser/src/validity.rs`) from the main document's final CDP response and an in-page probe (title, visible text length, final URL, challenge-widget markers). Challenge: an `x-amzn-waf-action` header, `cf-mitigated: challenge`, a final URL on a challenge-only host (`perfdrive.com`, `captcha-delivery.com`), or, on a page with at most 3000 characters of visible text, a challenge title prefix ("Just a moment", "Attention Required! | Cloudflare", "Human Verification", "Access Denied", "Radware Bot Manager", "Captcha", ...) or a full-page widget marker (Cloudflare challenge form, PerimeterX, DataDome, Imperva, Akamai, AWS WAF, slider captchas). HTTP error: a final status >= 400 with no challenge signal. A blocked page throws `the page is a bot challenge, not the site (<signals>). Findings would describe the challenge page, so none are reported.` or `the page returned an error (HTTP <status>). Findings would describe the error page, so none are reported.`, which the CLI prints as `Error: ...` and counts as an operational failure (exit 1). A `file://` load has no response, so only the probe signals apply.
- Consent wall: a page with at most 3000 characters of visible text where a known consent manager (next item) is showing, fewer than 100 visible characters lie outside its roots, and the manager holds at least three times as many. It throws `the page is a consent wall, not the site (consent manager <names>, <n> of <total> visible characters outside it). Findings would describe the consent dialog, so none are reported.` The probe runs before any banner is hidden, and the gate applies with `--no-consent-hiding` too. The probe object (evidence) carries `consent` (the names of the managers showing) and `consentChars`.
- - **Consent banners** (Rust engine). A consent banner covers copy a visitor reads once they dismiss it: text under it was skipped as covered or scored against the banner, and the banner's own text was scored as the site's. After the validity gate and again after the reveal sweep (so a banner injected after load or on the first scroll is caught before the capture), the engine hides the roots and backdrops of the consent managers in `crates/browser/src/consent.rs`: OneTrust, Cookiebot, Usercentrics, TrustArc, Didomi, Quantcast Choice, Sourcepoint, Osano, the open-source Cookie Consent library, CookieYes, Complianz, iubenda, Termly, Axeptio, Borlabs Cookie, Klaro, Cookie Notice, Shopify's privacy banner, and Google Funding Choices. Selectors are the vendor's own ids and classes, never a generic word, so a notice a site wrote for itself stays on the page and is scanned. Hiding is one injected `<style id="impeccable-consent-hide">` of `display: none !important` rules (plus an inline `display: none !important` on a matched element whose own inline style outranks the rule); nothing is clicked and no cookie or consent state is set. The scroll lock is undone only where the manager applied it: its own lock classes on `<html>` / `<body>` (Didomi `didomi-popup-open`, Sourcepoint `sp-message-open`) are removed, and an inline `overflow` / `overflow-y: hidden` on `<html>` or `<body>` is removed only while that manager's backdrop was showing. Every pass, the hit tests, and the evidence screenshot see the page without the banner (a last hide runs before the screenshot). Findings from a scan that hid a showing banner carry `consentHidden: ["<name>", ...]` (after `thirdParty`), and text mode prints the note above. `--no-consent-hiding` turns all of it off. The evidence API (`detect_url_evidence`) leaves the findings untagged, so they replay equal, and records `Evidence.consent`: `{ hidden, matched: { <name>: [selectors] }, unlocked }`. The static (file) engine is unaffected.
+ Consent wall: a page with at most 3000 characters of visible text where a known consent manager (next item) is showing, hiding it would leave next to nothing (fewer than 20 visible characters outside its roots, and no visible form control (`input` other than hidden, `select`, `textarea`, `button`) and no `img`, `svg`, `video`, `canvas`, `iframe`, `object` or `embed` of 10,000 square pixels or more outside them), and the manager holds at least 100 characters. It throws `the page is a consent wall, not the site (consent manager <names>, <n> of <total> visible characters outside it, no control or image outside it). Findings would describe the consent dialog, so none are reported.` A short page under an ordinary banner (a sign-in form, a splash page with a button, an image-first page) is scanned. The probe runs before any banner is hidden. With `--no-consent-hiding` the gate is off and the page is scanned with its banner, which is what that flag asks for. The probe object (evidence) carries `consent` (the names of the managers showing), `consentChars` (visible characters inside them) and `consentOutside` (visible controls and sizable media outside them, counted up to 20).
+ - **Consent banners** (Rust engine). A consent banner covers copy a visitor reads once they dismiss it: text under it was skipped as covered or scored against the banner, and the banner's own text was scored as the site's. After the validity gate, again after the reveal sweep (so a banner injected after load or on the first scroll is caught before the capture), and once more right after the capture (when that pass changes the page, the page is captured again, so the capture, the live hit tests and the pixel reads agree), the engine hides the roots and backdrops of the consent managers in `crates/browser/src/consent.rs`: OneTrust, Cookiebot, Usercentrics, TrustArc, Didomi, Quantcast Choice, Sourcepoint, Osano, the open-source Cookie Consent library, CookieYes, Complianz, iubenda, Termly, Axeptio, Borlabs Cookie, Klaro, Cookie Notice, Shopify's privacy banner, and Google Funding Choices. Selectors are the vendor's own ids and classes, never a generic word, so a notice a site wrote for itself stays on the page and is scanned. Hiding is one injected `<style id="impeccable-consent-hide">` of `display: none !important` rules (plus an inline `display: none !important` on a matched element whose own inline style outranks the rule); nothing is clicked and no cookie or consent state is set. The scroll lock is undone only where the manager applied it: its own lock classes on `<html>` / `<body>` (Didomi `didomi-popup-open`, Sourcepoint `sp-message-open`) are removed, and an inline `overflow` / `overflow-y: hidden` on `<html>` or `<body>` is removed only while that manager's backdrop was showing and the page is not an app shell (an element outside the managers at least half the viewport tall that scrolls itself with `overflow-y: auto` / `scroll`, whose body lock is the site's own). Every hide pass is best-effort: one that fails hides nothing and does not fail the scan. Every pass, the hit tests, and the evidence screenshot see the page without the banner (a last hide runs before the screenshot). Findings from a scan that hid a showing banner carry `consentHidden: ["<name>", ...]` (after `thirdParty`), and text mode prints the note above. `--no-consent-hiding` turns all of it off. The evidence API (`detect_url_evidence`) leaves the findings untagged, so they replay equal, and records `Evidence.consent`: `{ hidden, matched: { <name>: [selectors] }, unlocked }`. The static (file) engine is unaffected.
- **What the visual pass refuses to read.** The JS reported whatever its sampling produced; the engine says nothing where the reading is not the text's own background. The candidate collector applies the element pass's text gates before a candidate takes one of the `maxCandidates` slots, is sampled, or is scrolled into view: text with no letter and no digit (`◯`, `{ }`, `/`) as well as emoji-only text; a disabled control (`[disabled]`, `[aria-disabled="true"]`); a computed font size under 1px; a `color` at alpha 0.02 or less, or a transparent `-webkit-text-fill-color`, unless the element clips its own background to its text; and an element the painted-at-capture predicate (`crates/core/src/browser/painted.rs`, text gate) says is not painted, such as a tab-strip cell past its clipping edge or a carousel slide parked beside the page. A background image read as a surface has to be under the text: an image drawn `no-repeat` (from the computed `background` shorthand's first layer) whose painted rect, at a `background-size` the capture can place (pixels, percentages, `cover`, or any size once the image's intrinsic size is loaded), covers neither the candidate's text box nor its own box (`body` and `html` are asked only about the text) ends the walk `background image does not cover the text` (`stop`), whether or not the pass could load the file. An icon sprite framing a count and a mark beside a button label are refused this way; a repeating tile and a placement the capture cannot state are read as before. In the stack walk (`impeccable_core::browser::visual`): a translucent surface no longer restarts the walk at its own parent (which re-entered the stack above the node it came from and composited that surface over itself until its alpha read as opaque, and which skipped siblings painted between the surface and that parent) — the walk keeps descending the one `elementsFromPoint` stack, collecting translucent surfaces until an opaque one grounds them, then composites them back in order (topmost surface still names the method, `...+alpha`); a walk that reaches the end of the stack with nothing opaque is unresolved rather than reporting the last translucent color as a surface. A gradient answers analytically only when its stops agree: any stop below 0.95 alpha (a scrim, a glow, a `from-x/10` wash composites differently at every point of the box) or two opaque stops more than 2× apart in contrast make it `gradient stops disagree`, and an `svg` in the stack makes it `<tag> paint`; both end the walk (`stop`) rather than letting the nodes under them answer for the text, so the candidate falls to the screenshot pass. In the screenshot flow: a candidate whose reasons include `background-clip text`, `filter`, or `backdrop filter` is refused before the two screenshots are taken; the verdict is the median over the *glyph cores* (channel delta ≥ 90% of the strongest delta in the box, ≥ 8 of them; when fewer than 8 pixels reach 90%, as on small or thin text, the pixels from 75% up) rather than the tenth percentile over every changed pixel, which antialiased edges drive to 1:1; and the pass reports nothing when the diff is not a glyph over its background — more than 55% of the clip changed (the page repainted), more than a quarter of the changed pixels moved the opposite way from the rest (text mid-animation leaves its old position and its new one in the same diff), most fully painted pixels are not the color the text declares (`preferRenderedForeground` aside, where the painted value *is* the claim), the surface under the glyphs stands 2:1 or further from the surface beside them in the same box (the page paints its own text twice, so hiding one copy leaves the other where the glyphs were), or the median of every measured pixel is more than 3× the verdict (the sample is two populations, not one surface). The snippet shape is unchanged. `median` in it is the median over the same glyph cores the verdict reads, so it prints the verdict's number and never reads below it (it used to be the median over every changed pixel, edges included, which printed verdicts above their own median such as `pixel contrast 3.5:1 median 1.7:1`). The median over every changed pixel still decides the bimodal refusal above, but it is not printed.
- **Viewport edges and overflow.** `body-text-viewport-edge` reports a gutter under 16px, or under 12px when `innerWidth` is 480px or less. A paragraph whose measured text runs past a side of the viewport (the distance rounds below 0) reports no gutter on that side; it still reports a gutter from 0px to the floor on the other side (`(left 8px)`). Those paragraphs, through the same gates plus the Text paint gate and inline ignores, report once per page instead: one `body-text-viewport-edge` finding attributed to the page (`selector` `body`, in `pageLevel`) on the side the furthest text reaches. The message follows what the page does. It reads `text runs past the <side> edge of the <vw>px viewport and is cut off: <name> reaches <d>px past it, with text in <n> block(s)` when the text is cut off: the viewport clips x overflow (the root's `overflow-x`, or body's when the root's is `visible`, at `hidden` or `clip`), the side is not the one the page scrolls to (right, or left under `direction: rtl`), the root's recorded `scrollWidth` does not exceed its `clientWidth` by more than 1px, or for every paragraph on that side a box at `overflow-x: hidden` or `clip` (the paragraph or an ancestor below the body, with its own edge on that side inside the viewport) is passed by the text. `<name>` is then the box reaching furthest among the paragraphs and their ancestors. Otherwise it reads `page scrolls sideways at <vw>px: <name> reaches <d>px past the <side> edge, and text in <n> block(s) runs past it`, where `<name>` is the box reaching furthest past that side among those no ancestor below the root clips or scrolls on x and not `position: fixed`, and no further than the page scrolls (ties go to the outer box). When the engine cannot tell (nothing it can see clips the text and the root recorded no `scrollWidth`) the sideways wording stands and `<name>` is the box reaching furthest among the paragraphs and their ancestors. In both wordings the paragraph itself is named when its text reaches further than any such box. `<name>` is the element's `class_selector`, except that a `div` with no class is named `div#<id>`, else `div in <tag>#<id>` or `div in <tag>.<class>` (at most three classes) after its nearest ancestor below the body with an id or a class, else `div ("<text>")` with up to 40 characters of its text, cut at a word and ended with `…` when cut, double quotes in the text written as `'`. `text-overflow` reports a spill only when it does harm: a box at `overflow-x: hidden` or `clip` (the element itself or an ancestor below the body, with area) is passed by the spill's reach, the spill reaches within a quarter em of the viewport's side or past it, or the spilled part comes within a quarter em of another element's text or painted box on the element's lines (not an ancestor or descendant, not a box that holds the element's whole box, and not a decoration layer: an absolutely or fixed positioned box with no text, other than a replaced element or an `svg`, that is 2px thick or less or paints only a plain fill, with no background image or border, at alpha times opacity under 0.25). The reach is the painted extent, or the scroll extent where that is taken as read; text that runs past its box into free space reports nothing.
- `createBrowserDetector(options)`: shared browser, defaults `waitUntil:'load'`, `settleMs:100`, `viewport 1280x800`; `close()` closes only an owned browser.
diff --git a/tests/fixtures/consent/app-shell.html b/tests/fixtures/consent/app-shell.html
new file mode 100644
index 000000000..f39c3f119
--- /dev/null
+++ b/tests/fixtures/consent/app-shell.html
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>Consent hiding: an app shell under a OneTrust backdrop</title>
+ <style>
+ /* An app shell locks <body> itself (inline overflow: hidden) and scrolls
+ inside <main>. A OneTrust banner with its dark filter shows over it.
+ The banner and filter are hidden, and the body lock stays: it is the
+ site's, not the manager's. Pinned by
+ crates/browser/tests/consent_hiding.rs. */
+ body { font: 16px/1.5 system-ui, sans-serif; margin: 0; color: #111; background: #fff; height: 100vh; }
+ main { box-sizing: border-box; height: 100vh; overflow-y: auto; padding: 40px; }
+ .tall { height: 2000px; }
+ .onetrust-pc-dark-filter { position: fixed; inset: 0; background: rgba(0, 0, 0, 0.5); z-index: 999; }
+ #onetrust-banner-sdk { position: fixed; left: 0; right: 0; bottom: 0; height: 120px; background: #fff; border-top: 1px solid #ccc; z-index: 1000; padding: 20px; }
+ </style>
+</head>
+<body style="overflow: hidden;">
+ <main><h1>App shell</h1><p>The body does not scroll; main does.</p><div class="tall"></div></main>
+ <div id="onetrust-consent-sdk">
+ <div class="onetrust-pc-dark-filter"></div>
+ <div id="onetrust-banner-sdk"><p id="onetrust-policy-text">We use cookies to measure visits.</p></div>
+ </div>
+</body>
+</html>
diff --git a/tests/fixtures/consent/short-page.html b/tests/fixtures/consent/short-page.html
new file mode 100644
index 000000000..371a8d0c8
--- /dev/null
+++ b/tests/fixtures/consent/short-page.html
@@ -0,0 +1,36 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>Sign in</title>
+ <style>
+ /* A short page that is still a page: a sign-in form with a faint hint,
+ under a standard OneTrust bottom bar whose text outweighs the page's
+ own. Almost all its visible text is the banner's, but the form is
+ content, so the validity gate scans it rather than refusing it as a
+ consent wall, and the hint reports. Pinned by
+ crates/browser/tests/consent_hiding.rs. */
+ body { font: 16px/1.5 system-ui, sans-serif; margin: 0; padding: 40px; color: #111; background: #fff; }
+ label { display: block; margin: 0 0 12px; }
+ .hint { color: #aaaaaa; font-size: 13px; }
+ #onetrust-banner-sdk { position: fixed; left: 0; right: 0; bottom: 0; background: #fff; z-index: 1000; padding: 20px; border-top: 1px solid #ccc; }
+ #onetrust-policy-text { margin: 0 0 12px; font-size: 14px; color: #b0b0b0; }
+ </style>
+</head>
+<body>
+ <h1>Sign in</h1>
+ <form>
+ <label>Email <input type="email"></label>
+ <label>Password <input type="password"></label>
+ <p class="hint" id="hint">Forgot password?</p>
+ <button type="submit">Sign in</button>
+ </form>
+ <div id="onetrust-consent-sdk">
+ <div id="onetrust-banner-sdk">
+ <p id="onetrust-policy-text">By clicking "Accept All Cookies", you agree to the storing of cookies on your device to enhance site navigation, analyze site usage, and assist in our marketing efforts. Some cookies are strictly necessary for the site to work; others help us understand how you use it so we can make it better. Read our Cookie Policy.</p>
+ <button type="button">Cookies Settings</button> <button type="button">Reject All</button> <button type="button">Accept All Cookies</button>
+ </div>
+ </div>
+</body>
+</html>
diff --git a/tests/oracle/DELTAS.md b/tests/oracle/DELTAS.md
index 240917ca1..bd05e02b0 100644
--- a/tests/oracle/DELTAS.md
+++ b/tests/oracle/DELTAS.md
@@ -4592,17 +4592,35 @@ behavior is pinned by `crates/browser/tests/consent_hiding.rs` over
library (centene.com) and Google Funding Choices (ynet.co.il) on 1 each.
The other listed vendors have no corpus capture yet.
- **When.** After the validity gate, after the reveal sweep (a banner that
- arrives late or on the first scroll), and before the evidence screenshot.
+ arrives late or on the first scroll), right after the capture (a pass that
+ changes the page triggers a second capture, so the capture, the live hit
+ tests and the pixel reads agree), and before the evidence screenshot.
+ Every pass is best-effort: a failed one hides nothing and the scan goes on.
An injected `display: none !important` style; nothing is clicked and no
consent state is set. Scroll locks are undone only where the manager put
them: its own lock classes, and an inline `overflow: hidden` on html/body
- while its backdrop was up.
+ while its backdrop was up, unless the page is an app shell (an element at
+ least half the viewport tall that scrolls itself), whose body lock is the
+ site's.
+- **Borlabs.** Only `#BorlabsCookieBox` and `#BorlabsCookieBoxWrap`. The
+ shared `.brlbs-cmpnt-container` class was dropped in review: Borlabs
+ Cookie 3 also puts it on its content blockers (the YouTube and Maps
+ placeholders), which are page content.
- **Consent wall.** There was no consent-wall detection before this branch
(the round-4 note that the gate "already recognizes OneTrust, Cookiebot and
- Usercentrics" was wrong). The probe now names the managers showing and the
- text inside them, before hiding, and a small page with under 100 visible
- characters outside a showing manager (and at least three times as many
- inside it) is refused as `consent-wall`.
+ Usercentrics" was wrong). The probe now names the managers showing, the
+ text inside them, and the visible form controls and media of 10,000 square
+ pixels or more outside them, before hiding. A small page is refused as
+ `consent-wall` only when hiding would leave next to nothing: under 20
+ visible characters outside a showing manager, no control or sizable image
+ outside it, and at least 100 characters inside it. The first version
+ (under 100 characters outside, three times as many inside) refused a
+ sign-in page under a OneTrust bar in review, 48 characters of its own plus
+ a form, which base scanned and reported correctly; that page is now
+ `tests/fixtures/consent/short-page.html`. With `--no-consent-hiding` the
+ gate is off and the wall is scanned with its banner, which is what the
+ flag asks for. No capture in runs 26 and 27 comes near the gate: the two
+ mckesson.com pages under 3000 characters keep about 2400 of their own.
Measure, live (the ratchet cannot replay this: the recorded captures have
the banners in them). Run 26 recaptured hrsimple.app, theagenticdatacompany.com,
@@ -4632,6 +4650,20 @@ scanned twice at once with the branch binary, with and without
icon, Tailwind classes, no vendor), so this branch would not have hidden it:
the ten findings under it stay covered by design.
+After review (run 27, same six sites, same command): the counts against run
+26 are unchanged on every site but otto.de (low-contrast 158 to 156,
+undersized-ui-text 198 to 206, text-occlusion 0 and 0; otto.de's deal tiles
+rotate), and all 36 captures are `ok`. Ten run-25 crops that showed a
+consent layer were opened next to their run-27 twins: mckesson.com
+line-length x2 (the OneTrust bar over the paragraph, now the paragraph),
+centene.com body text x2 (the cc-window box, now the italic copy),
+otto.de mobile filter chips x3 (112172 to 112174: "Cookies und andere
+Technologien erlauben?", now the "von OTTO", "Damen" and "schwarz" chips),
+otto.de mobile cramped-padding (the OneTrust heading, now the promo module),
+and theagenticdatacompany.com undersized-ui-text at both widths (Cookiebot's
+"Preferences" toggle, now the "NEW" chip). All ten now show the flagged
+page element.
+
### Known limits at merge
1. **Site-made banners still cover text.** hrsimple.app and veeza.ai are the
@@ -4639,7 +4671,9 @@ scanned twice at once with the branch binary, with and without
decision ruled out.
2. **Scroll-lock data is thin.** No corpus capture shows a vendor lock class
on html/body; the two listed (Didomi, Sourcepoint) come from the vendors'
- own stylesheets, and the inline rule is inferred.
+ own stylesheets, and the inline rule is inferred. A site that locks body
+ scroll for its own modal (not an app shell) while a vendor backdrop is
+ also up is unlocked too.
3. **The corpus harness does not record `Evidence.consent`.** Its
`evidence.json` carries the probe's `consent` list (showing before
hiding), which names the same managers; recording the matched selectors