Skip to content

The design language that makes your AI harness better at design.

JavaScript

Keep verdicts the pixels can replace out of the replayed scan

Paul Bakausb6d51dfb1b4c
diff --git a/crates/browser/src/lib.rs b/crates/browser/src/lib.rs
index a0bbe93d0..9b8ab0658 100644
--- a/crates/browser/src/lib.rs
+++ b/crates/browser/src/lib.rs
@@ -264,8 +264,10 @@ pub mod origin {
pub const CONTENT_HIDDEN: &str = "content-hidden";
/// Uncaught page errors. Recorded, not replayable.
pub const SCRIPT_ERROR: &str = "script-error";
- /// The visual-contrast pass (image and pixel reads). Recorded, not
- /// replayable.
+ /// The visual-contrast pass (image and pixel reads), and the rule pass's
+ /// `low-contrast` verdicts on text it hands to that pass
+ /// (`visual::routed_reason`), which the pixels replace where they give a
+ /// verdict and which stand where they do not. Recorded, not replayable.
pub const VISUAL_CONTRAST: &str = "visual-contrast";
}
@@ -616,9 +618,14 @@ pub fn replay_url_scan(
let dom = snapshot_engine::parse_snapshot(snapshot).map_err(cdp_err)?;
dom.add_facts(facts);
let collected = collect_browser_findings(&dom, &config);
+ let handed = handed_over(&dom, &collected.groups);
let mut unanswered = dom.take_needs().hit_tests.len();
let groups = serialize_findings(&dom, &collected.groups);
let mut results = results_from_groups(groups.as_array().map(Vec::as_slice).unwrap_or(&[]));
+ // The verdicts the live scan hands to the pixels are reported under the
+ // visual-contrast origin, which a replay does not reproduce.
+ hand_over(&mut results, &handed);
+ results.retain(|r| r.origin == origin::SCAN);
let measured = measure_hidden_text_dom(&dom);
unanswered += dom.take_needs().hit_tests.len();
results.extend(content_hidden_results(
@@ -706,6 +713,40 @@ fn results_from_groups(groups: &[Value]) -> Vec<RawResult> {
out
}
+/// Per result of `results_from_groups(serialize_findings(dom, groups))`, in
+/// order: whether it is a `low-contrast` verdict on text the element pass
+/// hands to the pixels ([`impeccable_core::browser::visual::routed_reason`]).
+/// The live scan lets a pixel verdict replace such a verdict, so it is the
+/// visual-contrast pass's to report, and a replay leaves it out.
+fn handed_over(dom: &SnapshotDom, groups: &[impeccable_core::browser::FindingGroup]) -> Vec<bool> {
+ let mut out = Vec::new();
+ for g in groups {
+ let mut routed: Option<bool> = None;
+ for f in &g.findings {
+ let handed = f.type_ == "low-contrast"
+ && *routed.get_or_insert_with(|| {
+ impeccable_core::browser::visual::routed_reason(dom, g.el).is_some()
+ });
+ out.push(handed);
+ }
+ }
+ out
+}
+
+/// Move the verdicts [`handed_over`] marks to [`origin::VISUAL_CONTRAST`].
+/// Results that do not line up with the marks (never expected) stay in the
+/// scan origin, the same way live and in a replay.
+fn hand_over(results: &mut [RawResult], handed: &[bool]) {
+ if handed.len() != results.len() {
+ return;
+ }
+ for (r, handed) in results.iter_mut().zip(handed) {
+ if *handed {
+ r.origin = origin::VISUAL_CONTRAST;
+ }
+ }
+}
+
fn content_hidden_results(
total_chars: f64,
hidden_chars: f64,
@@ -917,10 +958,17 @@ fn scan_page_inner(
let mut serialized_groups: Vec<Value> = Vec::new();
let mut results = step_findings(profile, "scan", "browser-scan", url, || {
let facts = evidence.as_mut().map(|(ev, _)| &mut ev.scan_facts);
- let collected = snapshot_engine::resolve_needs_recording(
+ // Which verdicts the element pass hands to the pixels is decided here,
+ // inside the recorded rounds, so a replay reads it off the same
+ // capture and the same hit-test answers.
+ let (collected, handed) = snapshot_engine::resolve_needs_recording(
&base,
page,
- |d| collect_browser_findings(d, &config),
+ |d| {
+ let collected = collect_browser_findings(d, &config);
+ let handed = handed_over(d, &collected.groups);
+ (collected, handed)
+ },
facts,
)
.map_err(cdp_err)?;
@@ -929,6 +977,7 @@ fn scan_page_inner(
.cloned()
.unwrap_or_default();
let mut results = results_from_groups(&serialized_groups);
+ hand_over(&mut results, &handed);
// One serialized group per finding group, one result per finding, in
// order. Anything else leaves the elements unset, and the evidence
// resolves those selectors as before.
@@ -981,10 +1030,12 @@ fn scan_page_inner(
let (mut visual, superseded) =
run_visual_contrast_fallback(page, &analyses, &serialized_groups, viewport, profile, url)?;
// The element pass's verdict on text it hands over (paint the walk never
- // read under it, an outline) is replaced where the pixels gave one.
+ // read under it, an outline) is replaced where the pixels gave one. Only
+ // the verdicts `hand_over` moved to the visual-contrast origin can go: the
+ // scan origin is what a replay reproduces, and a replay has no pixels.
if !superseded.is_empty() {
results.retain(|r| {
- !(r.origin == origin::SCAN
+ !(r.origin == origin::VISUAL_CONTRAST
&& r.id == "low-contrast"
&& r.selector.as_deref().is_some_and(|s| superseded.iter().any(|x| x == s)))
});
diff --git a/crates/browser/tests/evidence.rs b/crates/browser/tests/evidence.rs
index d2b53a073..c383263f5 100644
--- a/crates/browser/tests/evidence.rs
+++ b/crates/browser/tests/evidence.rs
@@ -40,6 +40,7 @@ const REPLAY_FIXTURES: &[&str] = &[
// Copies part way past a carousel clip and the page edge, measured on
// their text rects.
"on-screen.html",
+ "unread-surface-contrast.html",
];
fn fixtures_dir() -> PathBuf {
diff --git a/crates/browser/tests/unread_surfaces.rs b/crates/browser/tests/unread_surfaces.rs
index f311ad6ec..1bd01cb38 100644
--- a/crates/browser/tests/unread_surfaces.rs
+++ b/crates/browser/tests/unread_surfaces.rs
@@ -18,13 +18,16 @@
//! - A glow over a light panel laid on a dark section is not on a dark
//! background; one under a translucent vignette still is.
//! - A page laid out wider than the phone scores its right column.
+//! - The element pass's verdict on text it hands to the pixels (the outlined
+//! sticker) is the visual-contrast pass's to keep or replace, so a replay
+//! of the recording reproduces the scan origin exactly.
use std::collections::HashMap;
use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};
use std::path::{Path, PathBuf};
-use impeccable_browser::BrowserEngine;
+use impeccable_browser::{detect_url_evidence, origin, replay_url_scan, BrowserEngine, EvidenceRequest};
use impeccable_detect::engines::{ScanOptions, UrlEngine};
fn fixtures_dir() -> PathBuf {
@@ -176,3 +179,55 @@ fn a_page_wider_than_the_phone_scores_its_right_column() {
"{flagged:#?}"
);
}
+
+#[test]
+fn verdicts_the_pixels_replace_stay_out_of_the_replayed_scan() {
+ let Some(engine) = engine() else { return };
+ let port = serve();
+ let url = format!("http://127.0.0.1:{port}/unread-surface-contrast.html");
+ let options = ScanOptions::default();
+ let mut browser = engine.launch().expect("launch");
+ let (live, evidence) =
+ detect_url_evidence(&mut browser, &url, &options, "networkidle0", 0, &EvidenceRequest::default())
+ .expect("evidence scan");
+ let selector = |f: &impeccable_core::findings::Finding| {
+ f.extras.get("selector").and_then(|v| v.as_str()).unwrap_or("").to_string()
+ };
+ // The pixels read the outlined sticker and pass it, which drops the
+ // element pass's verdict on it. That verdict was never the scan
+ // origin's: a replay has no pixels to drop it with.
+ let sticker: Vec<(&str, String)> = live
+ .iter()
+ .zip(&evidence.origins)
+ .filter(|(f, _)| f.antipattern == "low-contrast" && selector(f) == "#pass-outline-sticker")
+ .map(|(f, o)| (*o, f.snippet.clone()))
+ .collect();
+ assert!(sticker.is_empty(), "{sticker:?}");
+
+ let replay = replay_url_scan(
+ &url,
+ evidence.scan_snapshot.as_deref().expect("scan snapshot"),
+ &evidence.scan_facts,
+ None,
+ &options,
+ )
+ .expect("replay");
+ assert_eq!(replay.unanswered_hit_tests, 0);
+ let replayable: Vec<&impeccable_core::findings::Finding> = live
+ .iter()
+ .zip(&evidence.origins)
+ .filter(|(_, o)| **o == origin::SCAN || **o == origin::CONTENT_HIDDEN)
+ .map(|(f, _)| f)
+ .collect();
+ let replayed: Vec<&impeccable_core::findings::Finding> = replay.findings.iter().collect();
+ assert_eq!(replayed, replayable, "replay differs from the live scan");
+ assert!(
+ !replayed.iter().any(|f| f.antipattern == "low-contrast" && selector(f) == "#pass-outline-sticker"),
+ "{replayed:#?}"
+ );
+ // Verdicts the pass reads from the capture alone are still the scan's.
+ assert!(
+ replayed.iter().any(|f| f.antipattern == "low-contrast" && selector(f) == "#flag-dim-on-dark-panel"),
+ "{replayed:#?}"
+ );
+}
diff --git a/docs/CLI-CONTRACT.md b/docs/CLI-CONTRACT.md
index 2395ac2e0..8ff4993ec 100644
--- a/docs/CLI-CONTRACT.md
+++ b/docs/CLI-CONTRACT.md
@@ -340,7 +340,7 @@ Optional keys added later by engines (appended after the above): `ignoreValue` (
- **Covered text and unread surfaces.** The element pass asks `elementsFromPoint` over a failing `low-contrast` run, at the points the `text-occlusion` grid asks for the same box, in the same hit-test rounds. Where every deciding point has an image, video, canvas, or opaque fill at full opacity above the text (not the element, a descendant or an ancestor), the text is covered at capture: no verdict, and the visual-contrast collector skips it too. Where every deciding point has paint beneath the text that is nobody's ancestor, lies above the surface the background walk used, and could change what the text sits on, no element-pass verdict is printed; a visual-contrast candidate is still measured. That paint is a picture (an image, video, canvas or SVG `image`, or a `url()` background drawn `no-repeat`, `cover`, `contain`, at a percentage or larger than 256px), a gradient or fill that moves the named colour by more than 24 summed over the channels once composited over it, or, where the walk named no single colour or only reached the page ground (`html`, `body`, the canvas), a gradient, fill or SVG shape it cannot compare. Texture is read past and the verdict stands: a gradient drawn in cells of at most 64px on both axes, a `repeating-*` gradient, a gradient whose stops are mostly transparent or that pairs a transparent stop with a stop within 3px, a `url()` repeated at a drawn size of at most 256px, anything with a `mask-image`, and a picture, gradient or shape at an effective opacity of 0.3 or less. Paint at alpha times opacity of 0.1 or less is a wash and is read past too. A layer that cannot be placed (a remote image tiled at its own size, an `image-set`) keeps the verdict. The snapshot records `maskImage` and `webkitMaskImage` for this. The run must lie wholly inside the viewport, every point must be answered, and at least half must find the text; otherwise, and for a run only partly under a layer, the verdict stands. The SAFE_TAGS path asks before the page claims a colour pair. `text-occlusion` counts a decorated box only where its captured rect holds the probe point, so an answer from a page that moved after the capture names no occluder.
- **What is not painted at capture reports nothing.** The URL engine's element pass drops findings on an element the painted-at-capture predicate (`crates/core/src/browser/painted.rs`) says a visitor cannot see: `display: none` or `content-visibility: hidden` above it, `visibility: hidden` or `checkVisibility()` false, an effective opacity of 0.02 or less, a clipping ancestor with no area or no overlap, a box wholly outside the document, a 1px visually hidden box. The text measurements, `italic-serif-display` and `gradient-text` also need text and area; `layout-transition`, `bounce-easing`, `dark-glow` and `ai-color-palette` need area. `blinking-cursor` needs area, and its own opacity and a `visibility: hidden` its parent does not share do not hide it, so a cursor caught between blinks still reports. `kicker-above-heading` asks the same about the heading and the label above it. The page-level forms of `bounce-easing`, `dark-glow` and `pulsing-dot` report only when their selector matches at least one painted element (no area test, since the selector may name a pseudo-element); a form without a selector is unchanged. `text-occlusion` asks it about the covered text, every hit-test answer that covers it, each card a headline overhangs and each inline leak host, in addition to its own ancestor test; it also skips text under a `filter: blur()` of 4px or more on the element or an ancestor and text with fewer than two characters on screen (a lone emoji), and names what covers the text: `overlapping text`, `an SVG graphic` (an SVG element other than `text`, `tspan` or `textPath`), `an opaque element` (a box with an opaque fill), or `a bordered element` (a box counted through its borders alone). `content-hidden-at-rest` is not gated. The static HTML engine measures no boxes and is unchanged.
- **Surfaces off the ancestor chain.** Every `low-contrast` verdict, not only the SAFE_TAGS path's, also asks the structural layer climb (`impeccable_core::browser::visual::layer_under_text`), which answers below the fold and through layers with `pointer-events: none`, which `elementsFromPoint` never lists. Besides pictures and fills, the climb reads a gradient that paints a surface on a box that is nobody's ancestor (not a dot grid, hairline, small cell, mask, a wash at alpha times opacity of 0.1 or less, or a box at opacity 0.3 or less) and an SVG shape drawn around the text (at most three times the text box plus 16px on each axis) as paint the walk never read; an ancestor painting an opaque gradient ends the climb as its fill does. A positioned `::before` or `::after` is the surface only where its box, placed from its containing block, pixel offsets, size and transform (a rotation, scale or skew as the bounds of the transformed box about its centre), covers the text; one it cannot place keeps the size test. A pseudo-element or box at a negative `z-index` is skipped where an opaque fill or gradient lies between it and its stacking context (the host's own fill included, unless the host opens that context): an offset shadow behind a white card, a `-z-10` layer inside a white section that is not `isolate`. Paint the capture cannot place for certain (an unplaced pseudo, a pseudo with no recorded `z-index` or a negative layer with no recorded `isolation` over such a fill) is read as before but decides no verdict of its own: the verdict stands outside the SAFE_TAGS path, nothing is reprinted against it, and `dark-glow` ignores it. The snapshot records `isolation`, and `zIndex` and `translate` for pseudo-elements. Outside the SAFE_TAGS path the climb's own finding counts where the stacks cannot answer (below the fold, a replay that never asked), already see unread paint, or confirm the walk only because the layer ignores pointer events or is an SVG shape. That paint is read for the surfaces it can make over the ground the walk read (`unread_verdict`): the climb is asked again past each translucent layer, up to four, until it reaches paint that hides what is beneath or the walk's ground; the walk's own translucent layers are placed over or under that paint by the box each hangs from. A verdict that fails on every surface in that span stands, also where the stacks see unread paint, and prints `N:1 (need T:1) — text #ink on #surface (layer on <box>)` against the surface it reads best on when the walk named a flat colour more than 24 away from the span and the span's colours are known (fills and gradients); a verdict that passes on every one is dropped; one that depends on what the paint shows (a photo, unknown SVG fill) stands outside the SAFE_TAGS path, and the SAFE_TAGS path waives it as before. Outlined text (a `-webkit-text-stroke` wider than 0 in a colour of its own, or two or more opaque text-shadow layers blurred at most 2px and offset in opposing directions) keeps its element verdict. The snapshot records `webkitTextStrokeColor` and `webkitTextStrokeWidth`. `dark-glow`'s "on dark background" form reads an opaque detached fill or gradient under the element's box as its surface; a picture or translucent layer leaves the claim as the walk made it.
- - **Text the element pass hands over.** `collect_visual_contrast_candidates` takes `maxRoutedCandidates` (0 unless given; the URL engine passes 12, after its 12 `maxCandidates`, whose candidates are unchanged): text over unread paint whose verdict depends on the paint, ink in exactly the walk's surface colour over unread paint, links and spans with no fill among them, and outlined text, in document order. They carry `routed` (`unread layer` or `text outline`) and those reasons, which the sampled pass refuses; a first-budget candidate the element pass hands over carries `routed` only (and `preferRenderedForeground: true` for an outline). The URL engine takes a routed candidate to the screenshot pass even where the element pass reported on its selector, and where the pixels give a verdict, pass or fail, drops the element pass's `low-contrast` finding on that selector. Those extra reads stop after 4 seconds or 4 slow reads in a row with no verdict. The screenshot flow's hide style also clears `-webkit-text-stroke-color`, and `fill` and `stroke` on SVG `text` and `tspan`. In the stack walk a picture faded by its own box or a wrapper (below the nearest box that also holds the text) is composited at that opacity, and a picture that cannot be read (a tainted image, a refused video frame) ends the walk. `overlaps_page_width` measures against the root's scroll width when the document is wider than the window.
+ - **Text the element pass hands over.** `collect_visual_contrast_candidates` takes `maxRoutedCandidates` (0 unless given; the URL engine passes 12, after its 12 `maxCandidates`, whose candidates are unchanged): text over unread paint whose verdict depends on the paint, ink in exactly the walk's surface colour over unread paint, links and spans with no fill among them, and outlined text, in document order. They carry `routed` (`unread layer` or `text outline`) and those reasons, which the sampled pass refuses; a first-budget candidate the element pass hands over carries `routed` only (and `preferRenderedForeground: true` for an outline). The URL engine takes a routed candidate to the screenshot pass even where the element pass reported on its selector, and where the pixels give a verdict, pass or fail, drops the element pass's `low-contrast` finding on that selector. Those extra reads stop after 4 seconds or 4 slow reads in a row with no verdict. Because the pixels can drop it, every element-pass `low-contrast` verdict on text `routed_reason` hands over (decided in the rule pass's recorded hit-test rounds) carries the `visual-contrast` origin in the evidence, where it stands when the pixels give none, and `replay_url_scan` leaves it out: the `scan` origin stays what a replay of the recorded capture reproduces. The CLI's findings and their order are unchanged. The screenshot flow's hide style also clears `-webkit-text-stroke-color`, and `fill` and `stroke` on SVG `text` and `tspan`. In the stack walk a picture faded by its own box or a wrapper (below the nearest box that also holds the text) is composited at that opacity, and a picture that cannot be read (a tainted image, a refused video frame) ends the walk. `overlaps_page_width` measures against the root's scroll width when the document is wider than the window.
- **Element identity.** A finding from the rule pass or the visual-contrast pass carries `selector` (after `ignoreValue`): the flagged group's generated selector (`body` / `html` for page-level groups), or the visual-contrast candidate's selector. `content-hidden-at-rest` and `script-error` findings have none.
- **Where a script error was thrown.** A `script-error` snippet is the message followed by ` (at <source>)`: `<function>, <url>:<line>:<column>` from the top stack frame, or `<url>:<line>:<column>` for an anonymous frame or a script with no stack (a syntax error carries only the script position). A frame in a script with no URL (code from `eval` or `new Function`, a script element given text rather than a `src`) prints V8's name for it, `<anonymous>`, in the URL's place: `(at eval, <anonymous>:3:1)`. Line and column are 1-based (CDP's are 0-based); a URL past 160 characters is cut to 160 plus `...`; a throw with neither a named script nor a frame gets no suffix. The suffix is added after deduping, from the first occurrence. The JSON shape is unchanged: the source travels in `snippet`.
- **One message shape.** Every script-error message reads the way the console prints it, `Uncaught <Type>: <message>`, whatever path the throw took, where the JS printed puppeteer's `err.message` with the type stripped on one path and CDP's typed text on another. CDP's marker comes first: `Uncaught`, or `Uncaught (in promise)` for an unhandled rejection. An `Error` reads the first line of its description, which names its type (`TypeError: ...`, or the name a library gave it, `ChunkLoadError: ...`); a description that names no type gets the class in front, and an empty message is the type alone (`Uncaught AbortError`). A thrown value that is not an `Error` (CDP `type: object` with no `subtype`: a plain object, a class instance, a rejected object) reads its class and V8's property preview, `Uncaught Object {code: "E_CONFIG", message: "config missing"}` (strings JSON-quoted, other values as previewed, `...` when the preview overflowed). A primitive reads as `String(value)`, `Uncaught boom`. A throw that arrives with no exception object (a cross-origin script) keeps CDP's text, which is typed already (`Uncaught TypeError: ...`, or `Uncaught [object Object]`). The body after the marker is its first line, trimmed, and the message after its type (`TypeError: `) is cut at 160 characters, the cut the JS applied to the untyped message, so naming the type costs the message nothing it showed before; a throw whose body is empty reports nothing.