Skip to content

Write HTML. Render video. Built for agents.

perf(cli): check and inspect skip paint waits and screenshots nothing reads (#5868)

Miguel Ángel8434945784b5
diff --git a/packages/cli/src/capture/captureCompositionFrame.lateSubresources.browser.test.ts b/packages/cli/src/capture/captureCompositionFrame.lateSubresources.browser.test.ts
new file mode 100644
index 0000000000..e1dd3e3c74
--- /dev/null
+++ b/packages/cli/src/capture/captureCompositionFrame.lateSubresources.browser.test.ts
@@ -0,0 +1,63 @@
+import { createServer, type Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import { afterAll, beforeAll, expect, it } from "vitest";
+import { openSettledCompositionPage } from "./captureCompositionFrame.js";
+
+// An image, then a fetched asset after the load event, both inside the old 1.5 s settle.
+const IMAGE_MS = 800;
+const FETCH_MS = 1_100;
+const PNG = Buffer.from(
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==",
+ "base64",
+);
+const HTML = `<!doctype html><html><body>
+<div data-composition-id="main" data-width="320" data-height="180" data-duration="1"><img id="late" src="/late.png" /></div>
+<script>
+ window.__renderReady = true;
+ fetch("/late.json").then((r) => r.text()).then((t) => { document.body.dataset.late = t; });
+</script></body></html>`;
+
+let server: Server;
+let url: string;
+
+const ROUTES: Record<string, { ms: number; type: string; body: string | Buffer }> = {
+ "/": { ms: 0, type: "text/html; charset=utf-8", body: HTML },
+ "/late.png": { ms: IMAGE_MS, type: "image/png", body: PNG },
+ "/late.json": { ms: FETCH_MS, type: "application/json", body: '"arrived"' },
+};
+
+beforeAll(async () => {
+ server = createServer((req, res) => {
+ const route = ROUTES[req.url ?? "/"] ?? ROUTES["/"]!;
+ setTimeout(() => res.writeHead(200, { "Content-Type": route.type }).end(route.body), route.ms);
+ });
+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+ url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/`;
+});
+
+afterAll(() => {
+ server.closeAllConnections();
+ server.close();
+});
+
+it("returns only once images and fetched assets that land after DOMContentLoaded are in", async () => {
+ const { browser, page } = await openSettledCompositionPage(HTML, url, {
+ renderReadyTimeoutMs: 5_000,
+ renderReadyWarningSuffix: "test",
+ browserGpuMode: "software",
+ });
+ try {
+ expect(
+ await page.evaluate(() => {
+ const image = document.getElementById("late") as HTMLImageElement;
+ return {
+ complete: image.complete,
+ width: image.naturalWidth,
+ late: document.body.dataset.late,
+ };
+ }),
+ ).toEqual({ complete: true, width: 1, late: '"arrived"' });
+ } finally {
+ await browser.close();
+ }
+}, 90_000);
diff --git a/packages/cli/src/capture/captureCompositionFrame.test.ts b/packages/cli/src/capture/captureCompositionFrame.test.ts
index 7793f356bb..44f2c09c96 100644
--- a/packages/cli/src/capture/captureCompositionFrame.test.ts
+++ b/packages/cli/src/capture/captureCompositionFrame.test.ts
@@ -6,6 +6,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import {
AUDIT_SEEK_OPTIONS,
DENSE_GEOMETRY_SEEK_OPTIONS,
+ SCREENSHOT_SEEK_OPTIONS,
captureRegionCrop,
clampCropRegion,
installPageFunctionGuard,
@@ -112,7 +113,7 @@ describe("seekCompositionTimeline", () => {
expect(evaluate.mock.calls[2]?.[0]).toContain("window.setTimeout(finish, 100)");
// Post-seek font settle: a seek can reveal glyphs whose unicode-range
// subsets only start loading after the next layout (CJK snapshot reports).
- expect(evaluate).toHaveBeenNthCalledWith(4, expect.any(Function), 500);
+ expect(evaluate).toHaveBeenNthCalledWith(4, expect.any(Function), { ms: 500, frames: 1 });
});
it("waitForFontsMs: 0 disables the post-seek font wait", async () => {
@@ -215,7 +216,7 @@ describe("seekCompositionTimeline", () => {
expect(evaluate).toHaveBeenCalledWith(expect.any(Function), 3, true, false, false);
});
- it("supports layout's ordered double-frame, bounded font, and sleep settles", async () => {
+ it("checks fonts on the double settle's second frame instead of waiting a third", async () => {
vi.useFakeTimers();
const { page, evaluate } = fakeSeekPage();
@@ -228,11 +229,34 @@ describe("seekCompositionTimeline", () => {
await vi.advanceTimersByTimeAsync(120);
await pending;
- expect(evaluate).toHaveBeenCalledTimes(4);
+ expect(evaluate).toHaveBeenCalledTimes(3);
expect(evaluate).toHaveBeenNthCalledWith(1, expect.any(Function), 4, true, false, false);
expect(evaluate).toHaveBeenNthCalledWith(2, expect.any(Function), { videoFrames: true });
- expect(evaluate).toHaveBeenNthCalledWith(3, expect.any(Function));
- expect(evaluate).toHaveBeenNthCalledWith(4, expect.any(Function), 500);
+ expect(evaluate).toHaveBeenNthCalledWith(3, expect.any(Function), { ms: 500, frames: 2 });
+
+ const frames: (() => void)[] = [];
+ const statusReadAt: number[] = [];
+ let frame = 0;
+ vi.stubGlobal("requestAnimationFrame", (callback: () => void) => frames.push(callback));
+ vi.stubGlobal("document", {
+ body: { offsetHeight: 0 },
+ fonts: {
+ get status() {
+ statusReadAt.push(frame);
+ return "loaded";
+ },
+ ready: Promise.resolve(),
+ },
+ });
+ const fontWait = evaluate.mock.calls[2]?.[0];
+ if (typeof fontWait !== "function") throw new Error("Expected the font wait page function");
+ const waited = Reflect.apply(fontWait, undefined, [{ ms: 500, frames: 2 }]);
+ while (frames.length > 0) {
+ frame += 1;
+ frames.shift()?.();
+ }
+ await waited;
+ expect(statusReadAt).toEqual([2]);
});
it("awaits GPU completion registered by the seek event before settling", async () => {
@@ -470,6 +494,24 @@ describe("installPageFunctionGuard", () => {
});
});
+describe("SCREENSHOT_SEEK_OPTIONS", () => {
+ it("waits for image decodes before a screenshot, not for a fixed sleep", async () => {
+ vi.useFakeTimers();
+ const { page, evaluate } = fakeSeekPage();
+ let done = false;
+ void seekCompositionTimeline(page, 2, SCREENSHOT_SEEK_OPTIONS).then(() => {
+ done = true;
+ });
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(done).toBe(true);
+ expect(evaluate).toHaveBeenLastCalledWith(
+ expect.any(Function),
+ SCREENSHOT_SEEK_OPTIONS.waitForImagesMs,
+ );
+ });
+});
+
describe("DENSE_GEOMETRY_SEEK_OPTIONS", () => {
// A 79 s project of 1080p H.264 clips: each of the 600 dense seeks waited ~185 ms for a video frame it never reads,
// so `check` ran past its deadline. Frame sources and hf-seek work can move the DOM, so those still hold it.
@@ -496,8 +538,10 @@ describe("DENSE_GEOMETRY_SEEK_OPTIONS", () => {
expect(DENSE_GEOMETRY_SEEK_OPTIONS.animationFrameSettle).toBe("none");
expect(DENSE_GEOMETRY_SEEK_OPTIONS.waitForFontsMs).toBe(0);
expect(DENSE_GEOMETRY_SEEK_OPTIONS.settleMs).toBe(0);
- // AUDIT (the full-settle path used by the base grid) must still carry the waits.
+ // AUDIT (the base grid) keeps the frame and font waits; only a screenshot seek adds the paint sleep.
expect(AUDIT_SEEK_OPTIONS.animationFrameSettle).toBe("double");
- expect(AUDIT_SEEK_OPTIONS.settleMs).toBeGreaterThan(0);
+ expect("settleMs" in AUDIT_SEEK_OPTIONS).toBe(false);
+ expect(SCREENSHOT_SEEK_OPTIONS.waitForImagesMs).toBeGreaterThan(0);
+ expect("settleMs" in SCREENSHOT_SEEK_OPTIONS).toBe(false);
});
});
diff --git a/packages/cli/src/capture/captureCompositionFrame.ts b/packages/cli/src/capture/captureCompositionFrame.ts
index 6a63a58f4a..d068007fd6 100644
--- a/packages/cli/src/capture/captureCompositionFrame.ts
+++ b/packages/cli/src/capture/captureCompositionFrame.ts
@@ -23,16 +23,21 @@ import { gotoWhenSent } from "./navigateForCapture.js";
const SHADER_TRANSITIONS_TIMEOUT_MS = 90_000;
const CAPTURE_SETTLE_MS = 1500;
+const NETWORK_IDLE_MS = 100;
const PREFERRED_SEEK_TARGET_WAIT_MS = 500;
const DEFAULT_POST_SEEK_FONT_WAIT_MS = 500;
// The audit-grade seek tuning shared by check and the deprecated inspect/layout:
-// bridge+timeline fallback, ordered double-rAF settle, bounded font wait, sleep.
+// bridge+timeline fallback, ordered double-rAF settle, bounded font wait.
export const AUDIT_SEEK_OPTIONS = {
fallbackToBridgeAndTimelines: true,
animationFrameSettle: "double",
waitForFontsMs: 500,
- settleMs: 120,
+} as const;
+
+export const SCREENSHOT_SEEK_OPTIONS = {
+ ...AUDIT_SEEK_OPTIONS,
+ waitForImagesMs: 120,
} as const;
// Geometry-only seek for the dense content_overlap grid: getBoundingClientRect is valid synchronously after setTime, so drop all post-seek waits (rAF/font/sleep) that would multiply across the dense grid.
@@ -52,18 +57,21 @@ export interface SeekCompositionTimelineOptions {
animationFrameSettle?: "race" | "double" | "none";
waitForFontsMs?: number;
settleMs?: number;
+ waitForImagesMs?: number;
waitForVideoFrames?: boolean;
// Box-only reads: software GL spends about a second per frame on draws nothing here looks at.
skipWebglDraws?: boolean;
}
type SeekBarrierOptions = { videoFrames: boolean };
+type FontWaitOptions = { ms: number; frames: number };
type CompositionPageFunction =
| string
| (() => unknown)
| ((value: number) => unknown)
| ((barrier: SeekBarrierOptions) => unknown)
+ | ((wait: FontWaitOptions) => unknown)
| ((
value: number,
fallbackToBridgeAndTimelines: boolean,
@@ -74,7 +82,7 @@ type CompositionPageFunction =
export interface CompositionEvaluationPage {
evaluate(
pageFunction: CompositionPageFunction,
- value?: number | SeekBarrierOptions,
+ value?: number | SeekBarrierOptions | FontWaitOptions,
fallbackToBridgeAndTimelines?: boolean,
exactTime?: boolean,
skipWebglDraws?: boolean,
@@ -188,11 +196,41 @@ async function waitForCompositionSettle(
console.warn(` ${c.warn("⚠")} Shader transitions did not finish pre-rendering`);
});
- await page.evaluate(() => document.fonts.ready).catch(() => {});
- await new Promise((resolveSettle) => setTimeout(resolveSettle, CAPTURE_SETTLE_MS));
+ await waitForLateSubresources(page);
return runtimeReady;
}
+async function waitForLateSubresources(page: Page): Promise<void> {
+ let cap: ReturnType<typeof setTimeout> | undefined;
+ const deadline = Date.now() + CAPTURE_SETTLE_MS;
+ const ready = (async () => {
+ await page.waitForFunction(() => document.readyState === "complete", {
+ timeout: CAPTURE_SETTLE_MS,
+ });
+ await page.waitForNetworkIdle({
+ idleTime: NETWORK_IDLE_MS,
+ timeout: Math.max(1, deadline - Date.now()),
+ });
+ await page.evaluate(async () => {
+ await Promise.all(Array.from(document.images, (image) => image.decode().catch(() => {})));
+ await document.fonts.ready;
+ });
+ })();
+ try {
+ await Promise.race([
+ ready,
+ new Promise((resolveCap) => {
+ cap = setTimeout(resolveCap, CAPTURE_SETTLE_MS);
+ }),
+ ]);
+ } catch {
+ await new Promise((resolveCap) => setTimeout(resolveCap, Math.max(0, deadline - Date.now())));
+ } finally {
+ clearTimeout(cap);
+ ready.catch(() => {});
+ }
+}
+
// tsx/esbuild-style dev transpilers run with keepNames, which rewrites named
// inner functions in serialized page closures into __name(...) calls — a
// helper that exists in the Node bundle but not in the browser realm, so any
@@ -289,6 +327,110 @@ export async function openSettledCompositionPage(
}
}
+// Serialized into the page; the seek-target cascade must stay one function.
+// fallow-ignore-next-line complexity
+export function seekInPage(
+ t: number,
+ fallbackToBridgeAndTimelines: boolean,
+ exactTime: boolean,
+ skipDraws: boolean,
+): void {
+ const getProperty = (target: unknown, key: string): unknown => {
+ if ((typeof target !== "object" || target === null) && typeof target !== "function") {
+ return undefined;
+ }
+ return Reflect.get(target, key);
+ };
+ const call = (fn: unknown, receiver: unknown, args: unknown[]): boolean => {
+ if (typeof fn !== "function") return false;
+ Reflect.apply(fn, receiver, args);
+ return true;
+ };
+
+ const player = Reflect.get(window, "__player");
+ if (!player && !fallbackToBridgeAndTimelines) return;
+
+ const safe = Math.max(0, Number(t) || 0);
+ const renderSeek = getProperty(player, "renderSeek");
+ const playerSeek = getProperty(player, "seek");
+ const hf = Reflect.get(window, "__hf");
+ const bridgeSeek = getProperty(hf, "seek");
+ const gsap = Reflect.get(window, "gsap");
+ const ticker = getProperty(gsap, "ticker");
+
+ // fallow-ignore-next-line complexity
+ const seekTo = (time: number): void => {
+ // Prefer renderSeek because it also runs the runtime's data-start/data-duration
+ // visibility sync; raw timeline seeks leave off-window clips visible to audits.
+ if (call(renderSeek, player, exactTime ? [time, { exact: true }] : [time])) {
+ // Preferred runtime target handled the seek.
+ } else if (fallbackToBridgeAndTimelines && call(bridgeSeek, hf, [time])) {
+ // Producer bridge handled the seek.
+ } else if (call(playerSeek, player, [time])) {
+ // Legacy player target handled the seek.
+ } else if (fallbackToBridgeAndTimelines) {
+ const timelines = Reflect.get(window, "__timelines");
+ if (typeof timelines === "object" && timelines !== null) {
+ for (const key of Object.keys(timelines)) {
+ const timeline = Reflect.get(timelines, key);
+ call(getProperty(timeline, "pause"), timeline, []);
+ call(getProperty(timeline, "seek"), timeline, [time]);
+ }
+ }
+ }
+ call(getProperty(ticker, "tick"), ticker, []);
+ };
+
+ const gateSlot = Symbol.for("hyperframes.webglDrawGate");
+ let gate = Reflect.get(window, gateSlot) as { skip: boolean; skipped: boolean } | undefined;
+ if (skipDraws && !gate) {
+ const installed = { skip: false, skipped: false };
+ for (const name of ["WebGLRenderingContext", "WebGL2RenderingContext"]) {
+ const proto = getProperty(Reflect.get(window, name), "prototype");
+ if (!proto) continue;
+ for (const method of [
+ "clear",
+ "drawArrays",
+ "drawElements",
+ "drawArraysInstanced",
+ "drawElementsInstanced",
+ "drawRangeElements",
+ "blitFramebuffer",
+ ]) {
+ const draw = getProperty(proto, method);
+ if (typeof draw !== "function") continue;
+ Reflect.set(proto, method, function (this: unknown, ...args: unknown[]) {
+ if (!installed.skip) return Reflect.apply(draw, this, args);
+ installed.skipped = true;
+ return undefined;
+ });
+ }
+ }
+ Reflect.set(window, gateSlot, installed);
+ gate = installed;
+ }
+ const seekWithoutDraws = (time: number): void => {
+ if (!gate) return seekTo(time);
+ gate.skip = true;
+ try {
+ seekTo(time);
+ } finally {
+ gate.skip = false;
+ }
+ };
+
+ if (skipDraws) {
+ seekWithoutDraws(safe);
+ return;
+ }
+ // A skipped draw left the canvas stale and a same-time seek redraws nothing: step off first.
+ if (gate?.skipped) {
+ seekWithoutDraws(safe >= 0.001 ? safe - 0.001 : safe + 0.001);
+ gate.skipped = false;
+ }
+ seekTo(safe);
+}
+
export async function seekCompositionTimeline(
page: CompositionSeekPage,
timeSeconds: number,
@@ -299,104 +441,7 @@ export async function seekCompositionTimeline(
}
await page.evaluate(
- // Serialized into the page; the seek-target cascade must stay one function.
- // fallow-ignore-next-line complexity
- (t: number, fallbackToBridgeAndTimelines: boolean, exactTime: boolean, skipDraws: boolean) => {
- const getProperty = (target: unknown, key: string): unknown => {
- if ((typeof target !== "object" || target === null) && typeof target !== "function") {
- return undefined;
- }
- return Reflect.get(target, key);
- };
- const call = (fn: unknown, receiver: unknown, args: unknown[]): boolean => {
- if (typeof fn !== "function") return false;
- Reflect.apply(fn, receiver, args);
- return true;
- };
-
- const player = Reflect.get(window, "__player");
- if (!player && !fallbackToBridgeAndTimelines) return;
-
- const safe = Math.max(0, Number(t) || 0);
- const renderSeek = getProperty(player, "renderSeek");
- const playerSeek = getProperty(player, "seek");
- const hf = Reflect.get(window, "__hf");
- const bridgeSeek = getProperty(hf, "seek");
- const gsap = Reflect.get(window, "gsap");
- const ticker = getProperty(gsap, "ticker");
-
- // fallow-ignore-next-line complexity
- const seekTo = (time: number): void => {
- // Prefer renderSeek because it also runs the runtime's data-start/data-duration
- // visibility sync; raw timeline seeks leave off-window clips visible to audits.
- if (call(renderSeek, player, exactTime ? [time, { exact: true }] : [time])) {
- // Preferred runtime target handled the seek.
- } else if (fallbackToBridgeAndTimelines && call(bridgeSeek, hf, [time])) {
- // Producer bridge handled the seek.
- } else if (call(playerSeek, player, [time])) {
- // Legacy player target handled the seek.
- } else if (fallbackToBridgeAndTimelines) {
- const timelines = Reflect.get(window, "__timelines");
- if (typeof timelines === "object" && timelines !== null) {
- for (const key of Object.keys(timelines)) {
- const timeline = Reflect.get(timelines, key);
- call(getProperty(timeline, "pause"), timeline, []);
- call(getProperty(timeline, "seek"), timeline, [time]);
- }
- }
- }
- call(getProperty(ticker, "tick"), ticker, []);
- };
-
- const gateSlot = Symbol.for("hyperframes.webglDrawGate");
- let gate = Reflect.get(window, gateSlot) as { skip: boolean; skipped: boolean } | undefined;
- if (skipDraws && !gate) {
- const installed = { skip: false, skipped: false };
- for (const name of ["WebGLRenderingContext", "WebGL2RenderingContext"]) {
- const proto = getProperty(Reflect.get(window, name), "prototype");
- if (!proto) continue;
- for (const method of [
- "clear",
- "drawArrays",
- "drawElements",
- "drawArraysInstanced",
- "drawElementsInstanced",
- "drawRangeElements",
- "blitFramebuffer",
- ]) {
- const draw = getProperty(proto, method);
- if (typeof draw !== "function") continue;
- Reflect.set(proto, method, function (this: unknown, ...args: unknown[]) {
- if (!installed.skip) return Reflect.apply(draw, this, args);
- installed.skipped = true;
- return undefined;
- });
- }
- }
- Reflect.set(window, gateSlot, installed);
- gate = installed;
- }
- const seekWithoutDraws = (time: number): void => {
- if (!gate) return seekTo(time);
- gate.skip = true;
- try {
- seekTo(time);
- } finally {
- gate.skip = false;
- }
- };
-
- if (skipDraws) {
- seekWithoutDraws(safe);
- return;
- }
- // A skipped draw left the canvas stale and a same-time seek redraws nothing: step off first.
- if (gate?.skipped) {
- seekWithoutDraws(safe >= 0.001 ? safe - 0.001 : safe + 0.001);
- gate.skipped = false;
- }
- seekTo(safe);
- },
+ seekInPage,
timeSeconds,
options.fallbackToBridgeAndTimelines === true,
options.exactTime === true,
@@ -414,6 +459,7 @@ export async function seekCompositionTimeline(
);
const animationFrameSettle = options.animationFrameSettle ?? "race";
+ const waitForFontsMs = options.waitForFontsMs ?? DEFAULT_POST_SEEK_FONT_WAIT_MS;
if (animationFrameSettle === "race") {
await page.evaluate(`new Promise(function(r) {
var settled = false;
@@ -421,7 +467,7 @@ export async function seekCompositionTimeline(
window.setTimeout(finish, 100);
requestAnimationFrame(function() { requestAnimationFrame(finish); });
})`);
- } else if (animationFrameSettle === "double") {
+ } else if (animationFrameSettle === "double" && !(waitForFontsMs > 0)) {
await page.evaluate(
() =>
new Promise<void>((resolveFrame) =>
@@ -434,10 +480,10 @@ export async function seekCompositionTimeline(
// check, compare, validate, layout) and a seek that reveals unrequested
// glyphs must not screenshot before their font subsets load. Costs one
// evaluate + one rAF when nothing is loading. Pass 0 to disable.
- const waitForFontsMs = options.waitForFontsMs ?? DEFAULT_POST_SEEK_FONT_WAIT_MS;
if (waitForFontsMs > 0) {
- await waitForCompositionFonts(page, waitForFontsMs);
+ await waitForCompositionFonts(page, waitForFontsMs, animationFrameSettle === "double" ? 2 : 1);
}
+ if (options.waitForImagesMs) await waitForCompositionImages(page, options.waitForImagesMs);
if (options.settleMs !== undefined) {
const settleMs = Math.max(0, options.settleMs);
await new Promise((resolveSettle) => setTimeout(resolveSettle, settleMs));
@@ -469,40 +515,60 @@ export async function waitForPreferredSeekTarget(
}
}
+async function waitForCompositionImages(
+ page: CompositionEvaluationPage,
+ timeoutMs: number,
+): Promise<void> {
+ await page.evaluate(
+ (ms: number) =>
+ new Promise<void>((resolveWait) => {
+ const deadline = setTimeout(resolveWait, ms);
+ void Promise.all(
+ Array.from(document.images, (image) => image.decode().catch(() => {})),
+ ).then(() => {
+ clearTimeout(deadline);
+ resolveWait();
+ });
+ }),
+ timeoutMs,
+ );
+}
+
export async function waitForCompositionFonts(
page: CompositionEvaluationPage,
timeoutMs: number,
+ frames = 1,
): Promise<void> {
await page
- .evaluate((ms: number) => {
- const fonts = Reflect.get(document, "fonts");
- if (typeof fonts !== "object" || fonts === null) return Promise.resolve();
- // A seek can reveal glyphs whose font faces were never requested —
- // CJK @font-face splits into unicode-range subsets that only load when
- // first laid out. `fonts.ready` is already-resolved at that moment, so
- // awaiting it immediately races the load request and screenshots blank
- // glyphs (wild reports: Traditional Chinese / Microsoft YaHei check
- // snapshots). Force a synchronous layout so pending subsets actually
- // start loading, give the loader one frame to flip `fonts.status`,
- // THEN await readiness.
- void document.body?.offsetHeight;
- return new Promise<void>((resolveWait) => {
- const deadline = setTimeout(resolveWait, ms);
- requestAnimationFrame(() => {
- const status = Reflect.get(fonts, "status");
- const ready = Reflect.get(fonts, "ready");
- if (status !== "loading" || !ready) {
- clearTimeout(deadline);
- resolveWait();
- return;
- }
- Promise.resolve(ready).then(() => {
- clearTimeout(deadline);
- resolveWait();
+ .evaluate(
+ ({ ms, frames: count }: FontWaitOptions) => {
+ const fonts = Reflect.get(document, "fonts");
+ // A seek can reveal glyphs whose unicode-range font subsets never loaded, and `fonts.ready`
+ // resolves before their load starts (blank CJK glyphs in snapshots). Force layout so they start,
+ // let the loader flip `fonts.status` over the given frames, then await readiness.
+ void document.body?.offsetHeight;
+ const afterFrames = (left: number, then: () => void): void => {
+ requestAnimationFrame(() => (left > 1 ? afterFrames(left - 1, then) : then()));
+ };
+ return new Promise<void>((resolveWait) => {
+ const deadline = setTimeout(resolveWait, ms);
+ afterFrames(count, () => {
+ const status = fonts ? Reflect.get(fonts, "status") : undefined;
+ const ready = fonts ? Reflect.get(fonts, "ready") : undefined;
+ if (status !== "loading" || !ready) {
+ clearTimeout(deadline);
+ resolveWait();
+ return;
+ }
+ Promise.resolve(ready).then(() => {
+ clearTimeout(deadline);
+ resolveWait();
+ });
});
});
- });
- }, timeoutMs)
+ },
+ { ms: timeoutMs, frames },
+ )
.catch(() => {});
}
diff --git a/packages/cli/src/commands/check.test.ts b/packages/cli/src/commands/check.test.ts
index 06f211f508..93690b13ad 100644
--- a/packages/cli/src/commands/check.test.ts
+++ b/packages/cli/src/commands/check.test.ts
@@ -158,7 +158,7 @@ function fakeDriver(overrides: Partial<CheckAuditDriver> = {}): CheckAuditDriver
// by accident in unrelated scenarios — tests that want it force a constant
// via `collectLayoutGeometry: vi.fn(async () => "same")`.
let geometryCallCount = 0;
- return {
+ const driver: CheckAuditDriver = {
initialize: vi.fn(async (_contrast: boolean) => undefined),
getDuration: vi.fn(async () => 9),
hasNoTimelineDeclaration: vi.fn(async () => false),
@@ -183,8 +183,17 @@ function fakeDriver(overrides: Partial<CheckAuditDriver> = {}): CheckAuditDriver
})),
),
collectContrast: vi.fn(async (_time: number) => ({ entries: [], pngBase64: PNG_BASE64 })),
+ collectOverlapGrid: vi.fn(async (times: number[]) => {
+ const issues: AnchoredLayoutIssue[][] = [];
+ for (const time of times) {
+ await driver.seekGeometry(time);
+ issues.push(await driver.collectOverlap(time));
+ }
+ return issues;
+ }),
...overrides,
};
+ return driver;
}
interface GeometryFixture {
@@ -1008,6 +1017,16 @@ it("keeps contrast and snapshot sampling on the pre-gate layout grid", async ()
expect(collectContrast).toHaveBeenCalledWith(5);
});
+it("waits for paint only on the grid seeks a contrast screenshot follows", async () => {
+ const driver = fakeDriver({ getDuration: vi.fn(async () => 10) });
+ await runScenario(driver, { samples: 1, captionZone: { x0: 0, y0: 0.8, x1: 1, y1: 1 } });
+
+ expect(vi.mocked(driver.seek).mock.calls).toEqual([
+ [5, true],
+ [10, false],
+ ]);
+});
+
function layoutFindingOf(
code: LayoutIssueCode,
severity: "error" | "warning" | "info",
diff --git a/packages/cli/src/commands/contrast-audit.browser.js b/packages/cli/src/commands/contrast-audit.browser.js
index 4889dc2d26..fbed24163d 100644
--- a/packages/cli/src/commands/contrast-audit.browser.js
+++ b/packages/cli/src/commands/contrast-audit.browser.js
@@ -1,49 +1,14 @@
-// Browser-side WCAG contrast audit.
-// Loaded as a raw string and injected via page.addScriptTag to avoid
-// esbuild mangling (page.evaluate serializes functions; __name helpers break).
-//
-// Two-phase API — see packages/cli/src/commands/validate.ts's
-// runContrastAudit() for the calling contract:
-//
-// 1. window.__contrastAuditPrepare() walks the DOM for text-bearing
-// elements, computes each one's foreground paint (CSS `color`, or SVG
-// `fill` for SVG text — fill and color are independent CSS properties
-// in SVG), and HIDES that element's own text paint (color/fill set to
-// transparent). Hiding is layout-neutral — it doesn't reflow anything —
-// so the caller can screenshot the frame right after with the glyphs
-// invisible but everything else unchanged. Returns the candidate list
-// (selector, text, fg, bbox, font metrics).
-// 2. Caller takes ONE screenshot (page.screenshot()) — same number of
-// screenshots as before, just moved to after prepare() instead of
-// before it.
-// 3. window.__contrastAuditFinish(imgBase64, time, candidates) restores
-// the original paint FIRST (so a slow/failed decode can't leave the
-// page's text stuck invisible), then decodes the screenshot and, for
-// each candidate, samples the real composited pixels directly INSIDE
-// its own bounding box for the true background.
-//
-// Why sample inside the box instead of the 4px ring just outside it (the
-// previous approach): the ring is a proximity heuristic that's wrong
-// whenever what's immediately outside the text differs from what's actually
-// behind it — a neighboring panel/component just past the text's edge, a
-// rounded pill/button whose corner falls inside the ring, a
-// backdrop-filter-blurred glass panel sized only a couple pixels larger
-// than the text, or a translucent decoration that only partially overlaps
-// the ring (or sits entirely inside the bbox, never touching the ring at
-// all). Hiding the glyphs and sampling their own box side-steps all of
-// that: it reads the exact pixels that were behind them.
-//
-// window.__contrastAuditRestoreIfPending() is a safety net: if the caller's
-// screenshot or finish() call throws between prepare() and finish(), calling
-// this restores any still-hidden paint so the next sample in the loop
-// doesn't audit a page with stale invisible text. It's a no-op after a
-// normal finish() call.
-//
-// NOTE: this logic (DOM-walk, foreground/paint-hide, background sampling)
-// plus the pure WCAG math (relLum, wcagRatio, median) is duplicated in
-// skills/hyperframes-creative/scripts/contrast-report.mjs — keep in sync.
-// The pure "which rect to sample" decision is also mirrored in
-// contrast-sample.ts (unit-tested there since this file can't import).
+// Browser-side WCAG contrast audit, injected as a raw string so esbuild can't mangle it.
+// __contrastAuditPrepare() hides each text element's own paint (layout-neutral) and returns candidates;
+// the caller takes one screenshot, full frame or clipped to the candidates (collectContrast in checkBrowser.ts);
+// __contrastAuditFinish(png, time, candidates, region) restores paint first, then samples inside each box.
+
+// Sampling inside the box reads the exact pixels behind the glyphs; a ring outside the text misreads
+// neighbouring panels, rounded pills, blurred glass and partial decorations.
+// __contrastAuditRestoreIfPending() restores hidden paint if the screenshot or finish() throws.
+
+// Duplicated in skills/hyperframes-creative/scripts/contrast-report.mjs (keep in sync); the
+// "which rect to sample" decision is mirrored and unit-tested in contrast-sample.ts.
/* eslint-disable */
window.__contrastAuditPrepare = function () {
@@ -394,7 +359,7 @@ window.__contrastAuditRestoreIfPending = function () {
__contrastAuditRestoreAll();
};
-window.__contrastAuditFinish = async function (imgBase64, time, candidates) {
+window.__contrastAuditFinish = async function (imgBase64, time, candidates, region) {
function relLum(r, g, b) {
function ch(v) {
var s = v / 255;
@@ -438,8 +403,11 @@ window.__contrastAuditFinish = async function (imgBase64, time, candidates) {
if (!ctx) return [];
ctx.drawImage(img, 0, 0);
var px = ctx.getImageData(0, 0, canvas.width, canvas.height).data;
- var w = canvas.width;
- var h = canvas.height;
+ // A clipped shot holds `region.clip` of the frame: clamp to the frame, read at the offset.
+ var w = region ? region.frameWidth : canvas.width;
+ var h = region ? region.frameHeight : canvas.height;
+ var ox = region ? region.clip.x : 0;
+ var oy = region ? region.clip.y : 0;
var out = [];
for (var ci = 0; ci < candidates.length; ci++) {
@@ -467,7 +435,7 @@ window.__contrastAuditFinish = async function (imgBase64, time, candidates) {
aa = [];
for (var y = y0; y <= y1; y += stepY) {
for (var x = x0; x <= x1; x += stepX) {
- var idx = (y * w + x) * 4;
+ var idx = ((y - oy) * canvas.width + (x - ox)) * 4;
rr.push(px[idx]);
gg.push(px[idx + 1]);
bb.push(px[idx + 2]);
diff --git a/packages/cli/src/commands/contrast-clip.browser.test.ts b/packages/cli/src/commands/contrast-clip.browser.test.ts
new file mode 100644
index 0000000000..f0c4299d3d
--- /dev/null
+++ b/packages/cli/src/commands/contrast-clip.browser.test.ts
@@ -0,0 +1,74 @@
+import { readFileSync } from "node:fs";
+import { createServer, type Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import { join } from "node:path";
+import { afterAll, beforeAll, expect, it } from "vitest";
+import { openSettledCompositionPage } from "../capture/captureCompositionFrame.js";
+
+// Text over a two-tone backdrop, away from the origin: reading a clipped shot at the wrong offset picks the other tone.
+const HTML = `<!doctype html><html><body style="margin:0">
+<div data-composition-id="main" data-width="640" data-height="360" data-duration="1"
+ style="position:relative;width:640px;height:360px;background:linear-gradient(90deg,#102040 50%,#f0e0c0 50%)">
+ <p id="left" style="position:absolute;left:180px;top:150px;margin:0;font:32px sans-serif;color:#8090a0">Left copy</p>
+ <p id="right" style="position:absolute;left:420px;top:230px;margin:0;font:32px sans-serif;color:#8090a0">Right copy</p>
+</div><script>window.__renderReady = true;</script></body></html>`;
+const SCRIPT = readFileSync(join(import.meta.dirname, "contrast-audit.browser.js"), "utf8");
+
+let server: Server;
+let url: string;
+
+beforeAll(async () => {
+ server = createServer((_req, res) =>
+ res.writeHead(200, { "Content-Type": "text/html" }).end(HTML),
+ );
+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+ url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/`;
+});
+
+afterAll(() => {
+ server.closeAllConnections();
+ server.close();
+});
+
+type Box = { x: number; y: number; w: number; h: number };
+
+it("measures the same contrast from a shot clipped to the text as from the full frame", async () => {
+ const { browser, page } = await openSettledCompositionPage(HTML, url, {
+ renderReadyTimeoutMs: 5_000,
+ renderReadyWarningSuffix: "test",
+ browserGpuMode: "software",
+ });
+ try {
+ await page.addScriptTag({ content: SCRIPT });
+ const measure = async (clipped: boolean) => {
+ const candidates = (await page.evaluate("window.__contrastAuditPrepare()")) as {
+ bbox: Box;
+ }[];
+ const x = Math.floor(Math.min(...candidates.map((c) => c.bbox.x)));
+ const y = Math.floor(Math.min(...candidates.map((c) => c.bbox.y)));
+ const right = Math.ceil(Math.max(...candidates.map((c) => c.bbox.x + c.bbox.w)));
+ const bottom = Math.ceil(Math.max(...candidates.map((c) => c.bbox.y + c.bbox.h)));
+ const clip = { x, y, width: right - x, height: bottom - y };
+ const shot = await page.screenshot({
+ encoding: "base64",
+ omitBackground: true,
+ ...(clipped ? { clip, captureBeyondViewport: false } : {}),
+ });
+ const region = clipped ? { clip, frameWidth: 640, frameHeight: 360 } : null;
+ return page.evaluate(
+ (png: string, items: unknown[], at: unknown) =>
+ Reflect.apply(Reflect.get(window, "__contrastAuditFinish"), window, [png, 0, items, at]),
+ shot,
+ candidates,
+ region,
+ );
+ };
+
+ const full = (await measure(false)) as { bg: number[] }[];
+ expect(full).toHaveLength(2);
+ expect(full[0]?.bg).not.toEqual(full[1]?.bg);
+ expect(await measure(true)).toEqual(full);
+ } finally {
+ await browser.close();
+ }
+}, 90_000);
diff --git a/packages/cli/src/utils/checkBrowser.test.ts b/packages/cli/src/utils/checkBrowser.test.ts
index e57a599ab5..4bdc755fa9 100644
--- a/packages/cli/src/utils/checkBrowser.test.ts
+++ b/packages/cli/src/utils/checkBrowser.test.ts
@@ -1,12 +1,14 @@
// @vitest-environment happy-dom
// fallow-ignore-file code-duplication
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
openSettledCompositionPage,
type OpenSettledCompositionPageOptions,
+ SCREENSHOT_SEEK_OPTIONS,
+ seekCompositionTimeline,
} from "../capture/captureCompositionFrame.js";
import { DEFAULT_CHECK_OPTIONS, runAuditGrid } from "./checkPipeline.js";
import {
@@ -92,6 +94,11 @@ const PROJECT: ProjectDir = {
indexPath: "/project/index.html",
};
+// The overlap grid logs one marker per sample; keep them out of test output.
+beforeEach(() => {
+ vi.spyOn(console, "debug").mockImplementation(() => undefined);
+});
+
afterEach(() => {
vi.useRealTimers();
vi.clearAllMocks();
@@ -170,6 +177,100 @@ it("carries raw browser geometry through the page driver and pipeline", async ()
expect(mocks.serverClose).toHaveBeenCalledOnce();
});
+it("runs the dense overlap grid in one page task, keeping each sample's time on what it logs", async () => {
+ mountCanvasFixture(
+ `<section data-composition-file="scenes/copy.html"><p id="copy" data-layout-name="copy">Hi</p></section>`,
+ );
+ const page = fakePage();
+ const consoleHandlers: ((message: ReturnType<typeof fakeConsoleMessage>) => void)[] = [];
+ const onConsole = (message: ReturnType<typeof fakeConsoleMessage>) =>
+ consoleHandlers.forEach((handler) => handler(message));
+ page.on = vi.fn((event: string, handler: (typeof consoleHandlers)[number]) => {
+ if (event === "console") consoleHandlers.push(handler);
+ });
+ // CDP delivers the page's console calls in order; forward jsdom's the same way.
+ vi.spyOn(console, "debug").mockImplementation((text: string) => {
+ onConsole(fakeConsoleMessage("debug", text));
+ });
+ let evaluateCalls = 0;
+ let afterGrid = false;
+ const evaluate = page.evaluate;
+ page.evaluate = vi.fn(async (...args: unknown[]) => {
+ // A timer a sample queued fires once the grid's task ends, before Node's next command.
+ if (afterGrid) onConsole(fakeConsoleMessage("error", "queued during the grid"));
+ afterGrid = typeof args[0] === "string" && args[0].includes("__hfCheckSample");
+ evaluateCalls += 1;
+ return Reflect.apply(evaluate, page, args);
+ });
+ const denseAudits: number[] = [];
+ const injectScript = page.addScriptTag;
+ page.addScriptTag = vi.fn(async (arg: { content: string }) => {
+ await injectScript(arg);
+ Reflect.set(window, "__hyperframesOverlapAudit", ({ time }: { time: number }) => {
+ if (time !== 5) denseAudits.push(evaluateCalls);
+ if (time !== 2.5) return [];
+ onConsole(fakeConsoleMessage("error", "thrown at 2.5"));
+ const rect = { left: 10, top: 20, right: 110, bottom: 60, width: 100, height: 40 };
+ return [
+ {
+ code: "content_overlap",
+ severity: "warning",
+ time,
+ selector: "#copy",
+ message: "x",
+ rect,
+ },
+ ];
+ });
+ });
+ installSessionMock(page);
+
+ const result = await runBrowserCheck(
+ PROJECT,
+ { ...DEFAULT_CHECK_OPTIONS, samples: 1, contrast: false },
+ { kind: "none" },
+ runAuditGrid,
+ );
+
+ expect(denseAudits.length).toBeGreaterThan(10);
+ expect(new Set(denseAudits).size).toBe(1);
+ expect(result.runtimeFindings).toContainEqual(
+ expect.objectContaining({ code: "console_error", message: "thrown at 2.5", time: 2.5 }),
+ );
+ expect(result.runtimeFindings).toContainEqual(
+ expect.objectContaining({ message: "queued during the grid", time: 0, timeRange: [0, 10] }),
+ );
+ expect(
+ result.runtimeFindings.find((finding) => finding.message === "thrown at 2.5"),
+ ).not.toHaveProperty("timeRange");
+ expect(result.layoutIssues).toContainEqual(
+ expect.objectContaining({
+ code: "content_overlap",
+ selector: "#copy",
+ sourceFile: "scenes/copy.html",
+ dataAttributes: { "data-layout-name": "copy" },
+ bbox: { x: 10, y: 20, width: 100, height: 40 },
+ time: 2.5,
+ }),
+ );
+});
+
+it("takes no contrast screenshot at a sample with no text to measure", async () => {
+ mountCanvasFixture();
+ const page = fakePage();
+ page.screenshot = vi.fn(async () => "c3R1Yg==");
+ installSessionMock(page);
+
+ await runBrowserCheck(
+ PROJECT,
+ { ...DEFAULT_CHECK_OPTIONS, samples: 1, contrast: true },
+ { kind: "none" },
+ runAuditGrid,
+ );
+
+ expect(page.screenshot).not.toHaveBeenCalled();
+});
+
it("uses check --timeout for both navigation and render-ready settling", async () => {
mountCanvasFixture();
const page = fakePage();
@@ -288,60 +389,74 @@ it("round-trips the browser script's raw contrast candidates back into finish",
}
});
-it("captures check overview snapshots after contrast restores hidden text", async () => {
- vi.spyOn(Date, "now").mockReturnValue(100);
- mountCanvasFixture(`<div id="headline">Readable copy</div>`);
- const root = document.querySelector("[data-composition-id]");
- const headline = document.querySelector<HTMLElement>("#headline");
- if (!root || !headline) throw new Error("Contrast fixture failed to mount");
- vi.spyOn(root, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, 640, 360));
- vi.spyOn(headline, "getBoundingClientRect").mockReturnValue(new DOMRect(50, 50, 300, 40));
- vi.spyOn(window, "getComputedStyle").mockImplementation(
- () =>
- ({
- display: "block",
- visibility: "visible",
- opacity: "1",
- color: "rgb(255,255,255)",
- fill: "",
- clipPath: "none",
- fontSize: "32px",
- fontWeight: "700",
- }) as unknown as CSSStyleDeclaration,
- );
+it.each([
+ { snapshots: true, shots: 2, saved: ["text-visible-base64"] },
+ { snapshots: false, shots: 1, saved: [] },
+])(
+ "takes $shots contrast screenshot(s) when snapshots is $snapshots, overview after text restores",
+ async ({ snapshots, shots, saved }) => {
+ vi.spyOn(Date, "now").mockReturnValue(100);
+ mountCanvasFixture(`<div id="headline">Readable copy</div>`);
+ const root = document.querySelector("[data-composition-id]");
+ const headline = document.querySelector<HTMLElement>("#headline");
+ if (!root || !headline) throw new Error("Contrast fixture failed to mount");
+ vi.spyOn(root, "getBoundingClientRect").mockReturnValue(new DOMRect(0, 0, 640, 360));
+ vi.spyOn(headline, "getBoundingClientRect").mockReturnValue(new DOMRect(50, 50, 300, 40));
+ vi.spyOn(window, "getComputedStyle").mockImplementation(
+ () =>
+ ({
+ display: "block",
+ visibility: "visible",
+ opacity: "1",
+ color: "rgb(255,255,255)",
+ fill: "",
+ clipPath: "none",
+ fontSize: "32px",
+ fontWeight: "700",
+ }) as unknown as CSSStyleDeclaration,
+ );
- const page = fakePage();
- const injectScript = page.addScriptTag;
- page.addScriptTag = vi.fn(async (arg: { content: string }) => {
- await injectScript(arg);
- const w = window as unknown as {
- __contrastAuditFinish?: (...args: unknown[]) => Promise<unknown>;
- __contrastAuditRestoreIfPending?: () => void;
- };
- if (w.__contrastAuditFinish) {
- w.__contrastAuditFinish = async () => {
- w.__contrastAuditRestoreIfPending?.();
- return [];
+ const page = fakePage();
+ const injectScript = page.addScriptTag;
+ page.addScriptTag = vi.fn(async (arg: { content: string }) => {
+ await injectScript(arg);
+ const w = window as unknown as {
+ __contrastAuditFinish?: (...args: unknown[]) => Promise<unknown>;
+ __contrastAuditRestoreIfPending?: () => void;
};
- }
- });
- page.screenshot = vi.fn(async () =>
- headline.style.getPropertyValue("color") === "transparent"
- ? "text-hidden-base64"
- : "text-visible-base64",
- );
- installSessionMock(page);
+ if (w.__contrastAuditFinish) {
+ w.__contrastAuditFinish = async () => {
+ w.__contrastAuditRestoreIfPending?.();
+ return [];
+ };
+ }
+ });
+ page.screenshot = vi.fn(async () =>
+ headline.style.getPropertyValue("color") === "transparent"
+ ? "text-hidden-base64"
+ : "text-visible-base64",
+ );
+ installSessionMock(page);
- const result = await runBrowserCheck(
- PROJECT,
- { ...DEFAULT_CHECK_OPTIONS, samples: 1, contrast: true, snapshots: true },
- { kind: "none" },
- runAuditGrid,
- );
+ const result = await runBrowserCheck(
+ PROJECT,
+ { ...DEFAULT_CHECK_OPTIONS, samples: 1, contrast: true, snapshots },
+ { kind: "none" },
+ runAuditGrid,
+ );
- expect(page.screenshot).toHaveBeenCalledTimes(2);
- expect(result.screenshots[0]?.pngBase64).toBe("text-visible-base64");
-});
+ expect(page.screenshot).toHaveBeenCalledTimes(shots);
+ expect(page.screenshot).toHaveBeenNthCalledWith(
+ 1,
+ expect.objectContaining({
+ clip: { x: 50, y: 50, width: 300, height: 40 },
+ captureBeyondViewport: false,
+ }),
+ );
+ expect(seekCompositionTimeline).toHaveBeenCalledWith(page, 5, SCREENSHOT_SEEK_OPTIONS);
+ expect(result.screenshots.map((shot) => shot.pngBase64)).toEqual(saved);
+ },
+);
it("carries validate's clip-duration audit into the runtime findings", async () => {
vi.spyOn(Date, "now").mockReturnValue(100);
@@ -994,10 +1109,12 @@ function fakeConsoleMessage(type: string, text: string) {
function fakePage() {
return Object.assign(Object.create(null), {
on: vi.fn(),
+ viewport: () => ({ width: 640, height: 360 }),
addScriptTag: vi.fn(async ({ content }: { content: string }) => {
window.eval(content);
}),
evaluate: vi.fn(async (callback: unknown, ...args: unknown[]) => {
+ if (typeof callback === "string") return window.eval(callback);
if (typeof callback !== "function") throw new Error("Expected an evaluate callback");
return Reflect.apply(callback, window, args);
}),
diff --git a/packages/cli/src/utils/checkBrowser.ts b/packages/cli/src/utils/checkBrowser.ts
index 271d37213c..173c72734c 100644
--- a/packages/cli/src/utils/checkBrowser.ts
+++ b/packages/cli/src/utils/checkBrowser.ts
@@ -7,6 +7,7 @@ import type { Browser, Page } from "puppeteer-core";
import {
AUDIT_SEEK_OPTIONS,
DENSE_GEOMETRY_SEEK_OPTIONS,
+ SCREENSHOT_SEEK_OPTIONS,
DEFAULT_ZOOM_PADDING_PX,
DEFAULT_ZOOM_SCALE,
captureRegionCrop,
@@ -15,6 +16,7 @@ import {
resolveCliChromeGpuMode,
seekCompositionTimeline,
waitForPreferredSeekTarget,
+ seekInPage,
} from "../capture/captureCompositionFrame.js";
import {
auditClipDurations,
@@ -68,12 +70,21 @@ interface RuntimeDraft {
severity: CheckSeverity;
message: string;
time: number;
+ timeRange?: [number, number];
url?: string;
line?: number;
count?: number;
abortedImage?: boolean;
}
+type RuntimeStamp = Pick<RuntimeDraft, "time" | "timeRange">;
+
+interface RuntimeClock {
+ now(): RuntimeStamp;
+ at(time: number): void;
+ during(range: [number, number]): void;
+}
+
interface AnchorRequest {
selector: string;
time: number;
@@ -209,7 +220,16 @@ export async function runBrowserCheck(
[],
options.autoProxy,
);
- let currentTime = 0;
+ let stamp: RuntimeStamp = { time: 0 };
+ const clock: RuntimeClock = {
+ now: () => stamp,
+ at: (time) => {
+ stamp = { time };
+ },
+ during: (range) => {
+ stamp = { time: range[0], timeRange: range };
+ },
+ };
let failure: { kind: "none" } | { kind: "failed"; error: unknown } = { kind: "none" };
let chromeBrowser: import("puppeteer-core").Browser | undefined;
@@ -224,7 +244,7 @@ export async function runBrowserCheck(
onBrowser: deadline?.ownBrowser,
onBrowserReady: deadline?.start,
beforeNavigate: async (page) => {
- wireRuntimeListeners(page, drafts, () => currentTime);
+ wireRuntimeListeners(page, drafts, clock);
await deadline?.preparePage(page);
},
});
@@ -247,9 +267,7 @@ export async function runBrowserCheck(
const message = tweenTailMessage(tail);
drafts.push({ code: "gsap_tween_tail", severity: "warning", message, time: tail.later.end });
}
- const driver = createPageDriver(page, (time) => {
- currentTime = time;
- });
+ const driver = createPageDriver(page, clock);
const result = await runGrid(driver, options, motion);
const broken = await abortedImagesStillBroken(page, drafts);
return {
@@ -311,7 +329,7 @@ export async function captureFindingCrops(
const snapshotDir = checkSnapshotDirectory(project.dir);
mkdirSync(snapshotDir, { recursive: true });
for (const request of requests) {
- await seekCompositionTimeline(page, request.time, AUDIT_SEEK_OPTIONS);
+ await seekCompositionTimeline(page, request.time, SCREENSHOT_SEEK_OPTIONS);
const canvas = await page.evaluate(() => ({
width: window.innerWidth,
height: window.innerHeight,
@@ -400,7 +418,8 @@ function runtimeInfoFindingCode(text: string): string | null {
: "media_proxy_fallback";
}
-function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: () => number): void {
+function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], clock: RuntimeClock): void {
+ followSampleTimeMarkers(page, clock.at);
page.on("console", (message) => {
const type = message.type();
const text = message.text();
@@ -410,7 +429,7 @@ function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: (
code: "console_error",
severity: "error",
message: text,
- time: currentTime(),
+ ...clock.now(),
url: location.url,
line: location.lineNumber,
});
@@ -421,7 +440,7 @@ function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: (
code: webGpuFailure ? "webgpu_runtime_error" : "console_warning",
severity: webGpuFailure ? "error" : "warning",
message: text,
- time: currentTime(),
+ ...clock.now(),
url: location.url,
line: location.lineNumber,
});
@@ -433,7 +452,7 @@ function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: (
code,
severity: "info",
message: text,
- time: currentTime(),
+ ...clock.now(),
url: location.url,
line: location.lineNumber,
});
@@ -448,10 +467,10 @@ function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: (
code: "page_error",
severity: "error",
message,
- time: currentTime(),
+ ...clock.now(),
});
});
- wireNetworkListeners(page, drafts, currentTime);
+ wireNetworkListeners(page, drafts, clock);
}
/** Check's scrubs cancel image loads: an aborted image failed only if an `<img>` shows it and its decode fails. */
@@ -532,7 +551,7 @@ async function abortedImagesStillBroken(page: Page, drafts: RuntimeDraft[]): Pro
return new Set(broken);
}
-function wireNetworkListeners(page: Page, drafts: RuntimeDraft[], currentTime: () => number): void {
+function wireNetworkListeners(page: Page, drafts: RuntimeDraft[], clock: RuntimeClock): void {
page.on("requestfailed", (request) => {
const url = request.url();
if (url.includes("favicon") || url.startsWith("data:")) return;
@@ -542,7 +561,7 @@ function wireNetworkListeners(page: Page, drafts: RuntimeDraft[], currentTime: (
code: "request_failed",
severity: "error",
message: `Failed to load ${urlPath(url)}: ${failure ?? "net::ERR_FAILED"}`,
- time: currentTime(),
+ ...clock.now(),
url,
abortedImage: failure === "net::ERR_ABORTED" && request.resourceType() === "image",
});
@@ -556,13 +575,13 @@ function wireNetworkListeners(page: Page, drafts: RuntimeDraft[], currentTime: (
code: "http_error",
severity: "error",
message: `${response.status()} loading ${urlPath(url)}`,
- time: currentTime(),
+ ...clock.now(),
url,
});
});
}
-function createPageDriver(page: Page, setTime: (time: number) => void): CheckAuditDriver {
+function createPageDriver(page: Page, clock: RuntimeClock): CheckAuditDriver {
return {
initialize: (contrast) => injectAuditScripts(page, contrast),
matchComponents: (components, findings) =>
@@ -574,16 +593,26 @@ function createPageDriver(page: Page, setTime: (time: number) => void): CheckAud
getCanvas: () =>
page.evaluate(() => ({ width: window.innerWidth, height: window.innerHeight })),
findAmbiguousSelectors: (selectors) => findAmbiguousSelectors(page, selectors),
- seek: async (time) => {
- setTime(time);
- await seekCompositionTimeline(page, time, AUDIT_SEEK_OPTIONS);
+ seek: async (time, screenshotFollows) => {
+ clock.at(time);
+ await seekCompositionTimeline(
+ page,
+ time,
+ screenshotFollows ? SCREENSHOT_SEEK_OPTIONS : AUDIT_SEEK_OPTIONS,
+ );
},
seekGeometry: async (time) => {
- setTime(time);
+ clock.at(time);
await seekCompositionTimeline(page, time, DENSE_GEOMETRY_SEEK_OPTIONS);
},
collectLayout: (time, tolerance, layout) => collectLayout(page, time, tolerance, layout),
collectOverlap: (time) => collectOverlap(page, time),
+ collectOverlapGrid: async (times) => {
+ const issues = await collectOverlapGrid(page, times);
+ // What a grid sample queued for later lands after the grid, so it gets the grid's span, not its last time.
+ if (times.length > 0) clock.during([times[0] ?? 0, times.at(-1) ?? 0]);
+ return issues;
+ },
collectLayoutGeometry: () => collectLayoutGeometry(page),
collectSeekClock: () => collectSeekClock(page),
collectRotationSample: (time) => collectRotationSample(page, time),
@@ -738,6 +767,81 @@ async function collectOverlap(page: Page, time: number): Promise<AnchoredLayoutI
return anchorLayoutIssues(page, raw.flatMap(parseLayoutIssue));
}
+const SAMPLE_TIME_MARKER = "__hfCheckSample ";
+
+function followSampleTimeMarkers(page: Page, setTime: (time: number) => void): void {
+ page.on("console", (message) => {
+ const text = message.text();
+ if (message.type() === "debug" && text.startsWith(SAMPLE_TIME_MARKER)) {
+ setTime(Number(text.slice(SAMPLE_TIME_MARKER.length)));
+ }
+ });
+}
+
+interface OverlapGridSample {
+ raw: unknown[];
+ anchors: CheckAnchor[];
+}
+
+// Never yields to the renderer between samples, so it paints no frame for them.
+async function overlapGridInPage(
+ grid: {
+ times: number[];
+ marker: string;
+ fallback: boolean;
+ skipDraws: boolean;
+ videoFrames: boolean;
+ },
+ seek: typeof seekInPage,
+ anchors: typeof anchorsInPage,
+): Promise<OverlapGridSample[]> {
+ const audit = Reflect.get(window, "__hyperframesOverlapAudit");
+ const waitForCompletion = Reflect.get(window, "__hfWaitForSeekCompletion");
+ const samples: OverlapGridSample[] = [];
+ for (const time of grid.times) {
+ console.debug(`${grid.marker}${time}`);
+ seek(time, grid.fallback, false, grid.skipDraws);
+ if (typeof waitForCompletion === "function") {
+ await Reflect.apply(waitForCompletion, window, [{ videoFrames: grid.videoFrames }]);
+ }
+ const result: unknown =
+ typeof audit === "function" ? Reflect.apply(audit, window, [{ time }]) : [];
+ const raw = Array.isArray(result) ? result : [];
+ const zero = { x: 0, y: 0, width: 0, height: 0 };
+ const requests = raw.map((issue) => ({
+ selector: String(Reflect.get(Object(issue), "selector")),
+ time,
+ bbox: zero,
+ }));
+ samples.push({ raw, anchors: anchors(requests) });
+ }
+ return samples;
+}
+
+async function collectOverlapGrid(page: Page, times: number[]): Promise<AnchoredLayoutIssue[][]> {
+ if (times.length === 0) return [];
+ const grid = {
+ times,
+ marker: SAMPLE_TIME_MARKER,
+ fallback: DENSE_GEOMETRY_SEEK_OPTIONS.fallbackToBridgeAndTimelines,
+ skipDraws: DENSE_GEOMETRY_SEEK_OPTIONS.skipWebglDraws,
+ videoFrames: DENSE_GEOMETRY_SEEK_OPTIONS.waitForVideoFrames,
+ };
+ const samples = (await page.evaluate(
+ `(${overlapGridInPage})(${JSON.stringify(grid)}, ${seekInPage}, ${anchorsInPage})`,
+ )) as OverlapGridSample[];
+ return samples.map(({ raw, anchors }) =>
+ raw.flatMap((value, index) =>
+ parseLayoutIssue(value).map((issue) => ({
+ ...issue,
+ ...(anchors[index] ?? fallbackAnchor(undefined)),
+ bbox: rectToBbox(issue.rect),
+ time: issue.time,
+ })),
+ ),
+ );
+}
+
async function collectLayoutGeometry(page: Page): Promise<string> {
return page.evaluate(() => {
const geometry = Reflect.get(window, "__hyperframesLayoutGeometry");
@@ -944,36 +1048,38 @@ async function anchorLayoutIssues(
async function resolveAnchors(page: Page, requests: AnchorRequest[]): Promise<CheckAnchor[]> {
if (requests.length === 0) return [];
- return page.evaluate((values: AnchorRequest[]) => {
- const root = document.querySelector("[data-composition-id]");
- return values.map((request) => {
- let element: Element | null = null;
- try {
- element = document.querySelector(request.selector);
- } catch {
- element = null;
- }
- element ??= root;
- // Clones the anchor-extraction block in prepareContrast's evaluate() below;
- // both run inside separate serialized browser closures and can't share a
- // Node-side helper.
- // fallow-ignore-next-line code-duplication
- const dataAttributes: Record<string, string> = {};
- for (const attribute of Array.from(element?.attributes ?? [])) {
- if (attribute.name.startsWith("data-")) dataAttributes[attribute.name] = attribute.value;
- }
- const source = element
- ?.closest("[data-composition-file]")
- ?.getAttribute("data-composition-file");
- return {
- selector: element ? request.selector : "[data-composition-id]",
- dataAttributes,
- sourceFile: source || "index.html",
- bbox: request.bbox,
- time: request.time,
- };
- });
- }, requests);
+ return page.evaluate(anchorsInPage, requests);
+}
+
+function anchorsInPage(values: AnchorRequest[]): CheckAnchor[] {
+ const root = document.querySelector("[data-composition-id]");
+ return values.map((request) => {
+ let element: Element | null = null;
+ try {
+ element = document.querySelector(request.selector);
+ } catch {
+ element = null;
+ }
+ element ??= root;
+ // Clones the anchor-extraction block in prepareContrast's evaluate() below;
+ // both run inside separate serialized browser closures and can't share a
+ // Node-side helper.
+ // fallow-ignore-next-line code-duplication
+ const dataAttributes: Record<string, string> = {};
+ for (const attribute of Array.from(element?.attributes ?? [])) {
+ if (attribute.name.startsWith("data-")) dataAttributes[attribute.name] = attribute.value;
+ }
+ const source = element
+ ?.closest("[data-composition-file]")
+ ?.getAttribute("data-composition-file");
+ return {
+ selector: element ? request.selector : "[data-composition-id]",
+ dataAttributes,
+ sourceFile: source || "index.html",
+ bbox: request.bbox,
+ time: request.time,
+ };
+ });
}
async function resolveRootAnchor(page: Page): Promise<CheckAnchor> {
@@ -996,11 +1102,13 @@ async function compositionBbox(page: Page): Promise<CheckBbox> {
async function collectContrast(
page: Page,
time: number,
- layoutAnnotations: CheckAnnotationBox[] = [],
+ snapshotAnnotations?: CheckAnnotationBox[],
): Promise<ContrastCapture> {
let prepared: PreparedContrast[] = [];
try {
prepared = parsePreparedContrast(await prepareContrast(page, time));
+ if (prepared.length === 0 && !snapshotAnnotations) return { entries: [] };
+ const region = textRegion(prepared, page.viewport());
// This screenshot is the one contrast math is sampled from below — it must
// stay untouched by the annotation overlay (finishContrast reads real
// painted pixels), so annotation only ever happens on a SECOND shot.
@@ -1008,6 +1116,7 @@ async function collectContrast(
encoding: "base64",
type: "png",
omitBackground: true,
+ ...(region ? { clip: region.clip, captureBeyondViewport: false } : {}),
});
if (typeof measurementShot !== "string") throw new Error("Contrast screenshot was not base64");
const raw = await finishContrast(
@@ -1015,9 +1124,11 @@ async function collectContrast(
measurementShot,
time,
prepared.map((entry) => entry.raw),
+ region,
);
const finished = raw.flatMap(parseFinishedContrast);
const entries = joinContrastEntries(finished, prepared);
+ if (!snapshotAnnotations) return { entries };
// __contrastAuditFinish restores the text paint hidden by prepare. The
// measurement screenshot intentionally has glyphs removed so contrast
// can sample the pixels behind them; never persist that image as visual
@@ -1028,8 +1139,8 @@ async function collectContrast(
// they're appended to the layout-derived annotations passed in by the
// pipeline rather than being requested up front.
const annotations = [
- ...layoutAnnotations,
- ...contrastFailureAnnotations(entries, layoutAnnotations.length),
+ ...snapshotAnnotations,
+ ...contrastFailureAnnotations(entries, snapshotAnnotations.length),
];
const pngBase64 = await captureOverviewShot(page, annotations, overviewShot);
return { entries, pngBase64 };
@@ -1181,24 +1292,57 @@ async function prepareContrast(page: Page, time: number): Promise<unknown[]> {
}, time);
}
+interface TextRegion {
+ clip: { x: number; y: number; width: number; height: number };
+ frameWidth: number;
+ frameHeight: number;
+}
+
+// The smallest whole-pixel rect holding every box contrast samples; undefined means the full frame.
+function textRegion(
+ prepared: PreparedContrast[],
+ viewport: { width: number; height: number; deviceScaleFactor?: number } | null,
+): TextRegion | undefined {
+ if (!viewport || (viewport.deviceScaleFactor ?? 1) !== 1 || prepared.length === 0)
+ return undefined;
+ const boxes = prepared.map((entry) => entry.candidate.bbox);
+ const x = Math.max(0, Math.floor(Math.min(...boxes.map((box) => box.x))));
+ const y = Math.max(0, Math.floor(Math.min(...boxes.map((box) => box.y))));
+ const right = Math.min(viewport.width, Math.ceil(Math.max(...boxes.map((b) => b.x + b.width))));
+ const bottom = Math.min(
+ viewport.height,
+ Math.ceil(Math.max(...boxes.map((b) => b.y + b.height))),
+ );
+ if (right <= x || bottom <= y) return undefined;
+ const clip = { x, y, width: right - x, height: bottom - y };
+ return { clip, frameWidth: viewport.width, frameHeight: viewport.height };
+}
+
async function finishContrast(
page: Page,
screenshot: string,
time: number,
candidates: unknown[],
+ region: TextRegion | undefined,
): Promise<unknown[]> {
return page.evaluate(
- async (payload: { screenshot: string; time: number; candidates: unknown[] }) => {
+ async (payload: {
+ screenshot: string;
+ time: number;
+ candidates: unknown[];
+ region: TextRegion | null;
+ }) => {
const finish = Reflect.get(window, "__contrastAuditFinish");
if (typeof finish !== "function") return [];
const result = await Reflect.apply(finish, window, [
payload.screenshot,
payload.time,
payload.candidates,
+ payload.region,
]);
return Array.isArray(result) ? result : [];
},
- { screenshot, time, candidates },
+ { screenshot, time, candidates, region: region ?? null },
);
}
@@ -1390,6 +1534,7 @@ function runtimeFinding(draft: RuntimeDraft, root: CheckAnchor): CheckFinding {
sourceFile: root.sourceFile,
bbox: root.bbox,
time: draft.time,
+ ...(draft.timeRange ? { timeRange: draft.timeRange } : {}),
url: draft.url,
line: draft.line,
};
diff --git a/packages/cli/src/utils/checkPipeline.ts b/packages/cli/src/utils/checkPipeline.ts
index 9e880a6392..6771a7fef7 100644
--- a/packages/cli/src/utils/checkPipeline.ts
+++ b/packages/cli/src/utils/checkPipeline.ts
@@ -444,7 +444,7 @@ async function collectGridSamples(
indicatorFrames: [],
};
for (const time of mergeSampleTimes(grid.layoutSamples, motion.times)) {
- await driver.seek(time);
+ await driver.seek(time, contrastSet.has(time));
// Findings collected for THIS sample time, so the overview overlay (below)
// only ever annotates a frame with defects that are actually valid at
// that render time — never a stale bbox from an earlier/later sample.
@@ -471,23 +471,29 @@ async function collectGridSamples(
await driver.collectMotionFrame(time, motion.selectors, motion.livenessScopes),
);
}
- if (contrastSet.has(time)) {
- const contrastStart = Date.now();
- // Annotation is a --snapshots-only nicety — skip building it (and the
- // driver's extra overlay screenshot) when nothing will use it; the call
- // shape without --snapshots stays exactly what it was before this existed.
- const capture = options.snapshots
- ? await driver.collectContrast(time, annotationBoxesFrom(issuesAtTime))
- : await driver.collectContrast(time);
- collected.contrastMs += Date.now() - contrastStart;
- collected.contrastEntries.push(...capture.entries);
- collected.screenshots.push({ time, pngBase64: capture.pngBase64 });
- }
+ if (contrastSet.has(time))
+ await collectContrastAt(driver, options, time, issuesAtTime, collected);
}
await collectMotionOverlapSamples(driver, grid, collected, options.range);
return collected;
}
+async function collectContrastAt(
+ driver: CheckAuditDriver,
+ options: CheckOptions,
+ time: number,
+ issuesAtTime: AnchoredLayoutIssue[],
+ collected: GridSamples,
+): Promise<void> {
+ const contrastStart = Date.now();
+ const capture = options.snapshots
+ ? await driver.collectContrast(time, annotationBoxesFrom(issuesAtTime))
+ : await driver.collectContrast(time);
+ collected.contrastMs += Date.now() - contrastStart;
+ collected.contrastEntries.push(...capture.entries);
+ if (capture.pngBase64) collected.screenshots.push({ time, pngBase64: capture.pngBase64 });
+}
+
// Dense grid catches mid-motion text collisions the sparse layout grid seeks past; text-only overlap collection is cheap enough to afford it.
const OVERLAP_SAMPLE_FPS = 8;
// A fixed ceiling, so check's work does not grow with film length: half uniform, half in clip windows it misses.
@@ -560,6 +566,17 @@ function holdProber(driver: CheckAuditDriver, collected: GridSamples, end: numbe
};
}
+async function denseOverlapTimes(
+ driver: CheckAuditDriver,
+ grid: SampleGrid,
+ range?: [number, number],
+): Promise<number[]> {
+ const offset = range?.[0] ?? 0;
+ const edges = (await driver.getClipEdges()).map((time) => time - offset);
+ const overlapTimes = buildOverlapSampleTimes(range ? range[1] - range[0] : grid.duration, edges);
+ return range ? mergeSampleTimes(overlapTimes.map((time) => time + range[0])) : overlapTimes;
+}
+
/** Reruns content_overlap on a fine grid, unconditionally rather than gated on fingerprint change, since an animation aliased to the sparse grid collides between identical-fingerprint samples. */
async function collectMotionOverlapSamples(
driver: CheckAuditDriver,
@@ -568,18 +585,13 @@ async function collectMotionOverlapSamples(
range?: [number, number],
): Promise<void> {
const baseTimes = new Set(grid.layoutSamples);
- const offset = range?.[0] ?? 0;
- const edges = (await driver.getClipEdges()).map((time) => time - offset);
- const overlapTimes = buildOverlapSampleTimes(range ? range[1] - range[0] : grid.duration, edges);
- const shiftedTimes = range
- ? mergeSampleTimes(overlapTimes.map((time) => time + range[0]))
- : overlapTimes;
+ const shiftedTimes = await denseOverlapTimes(driver, grid, range);
const probeHold = holdProber(driver, collected, range ? range[1] : grid.duration);
+ const gridIssues = await driver.collectOverlapGrid(shiftedTimes.filter((t) => !baseTimes.has(t)));
+ let sample = 0;
for (const [index, time] of shiftedTimes.entries()) {
if (baseTimes.has(time)) continue;
- // Settle-free seek: collectOverlap reads getBoundingClientRect geometry, valid synchronously after setTime, so the dense pass skips the per-seek paint settle.
- await driver.seekGeometry(time);
- const issues = await driver.collectOverlap(time);
+ const issues = gridIssues[sample++] ?? [];
collected.layoutIssues.push(...issues);
await probeHold(time, shiftedTimes[index + 1] ?? Number.POSITIVE_INFINITY, issues);
}
diff --git a/packages/cli/src/utils/checkTypes.ts b/packages/cli/src/utils/checkTypes.ts
index 73d38b9b2e..99c1bbc6b9 100644
--- a/packages/cli/src/utils/checkTypes.ts
+++ b/packages/cli/src/utils/checkTypes.ts
@@ -77,6 +77,7 @@ export interface CheckAnchor {
}
export interface CheckFinding extends CheckAnchor {
+ timeRange?: [number, number];
code: string;
severity: CheckSeverity;
message: string;
@@ -108,7 +109,7 @@ export interface CheckContrastFinding extends CheckFinding {
export interface ContrastCapture {
entries: ContrastAuditEntry[];
- pngBase64: string;
+ pngBase64?: string;
}
export interface GeometryCandidateRequest {
@@ -198,7 +199,7 @@ export interface CheckAuditDriver {
getClipEdges(): Promise<number[]>;
getCanvas(): Promise<Canvas>;
findAmbiguousSelectors(selectors: string[]): Promise<AnchoredLayoutIssue[]>;
- seek(time: number): Promise<void>;
+ seek(time: number, screenshotFollows?: boolean): Promise<void>;
/** Settle-free seek for the geometry-only dense content_overlap pass; only collectOverlap consumes it, and getBoundingClientRect is valid synchronously after setTime. */
seekGeometry(time: number): Promise<void>;
collectLayout(
@@ -208,6 +209,7 @@ export interface CheckAuditDriver {
): Promise<AnchoredLayoutIssue[]>;
/** content_overlap only, for the dense re-sampling grid — catches transient text collisions the sparse grid seeks past. */
collectOverlap(time: number): Promise<AnchoredLayoutIssue[]>;
+ collectOverlapGrid(times: number[]): Promise<AnchoredLayoutIssue[][]>;
/** Frozen-sweep guard (#U10): an opaque fingerprint of the seeked visual state, from
* motion-signature.browser.js (shared with motion-sample's liveness). Legacy name kept. */
collectLayoutGeometry(): Promise<string>;
@@ -229,7 +231,10 @@ export interface CheckAuditDriver {
livenessScopes: string[],
): Promise<MotionFrame>;
anchorMotionIssues(issues: LayoutIssue[]): Promise<AnchoredLayoutIssue[]>;
- collectContrast(time: number, annotations?: CheckAnnotationBox[]): Promise<ContrastCapture>;
+ collectContrast(
+ time: number,
+ snapshotAnnotations?: CheckAnnotationBox[],
+ ): Promise<ContrastCapture>;
}
export interface CheckScreenshot {