Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

test(interaction-harness): a starved busy proof fails by name (#10565)

Christopher Chedeauc096afe1c04a
diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md
index a332748ed4..749bc06052 100644
--- a/internal/interaction-harness/README.md
+++ b/internal/interaction-harness/README.md
@@ -311,7 +311,10 @@ moved and its longest stall is no longer than the control's. Both are read from
the same screencast frames, so a compositor animation misses none. A window in
which the control moved on fewer than four frames fails, since the machine
presented too little to judge; the failure names the cast's frames in the
-window and its longest gap. A run of frames with no new position counts as
+window and its longest gap. When no frame came for longer than
+`FRAME_STARVATION_MS` there too, the machine starved the run, and
+`busyWindowProof` throws `FrameStarvationError` instead of returning a verdict.
+A run of frames with no new position counts as
moved when the change that ends it is a single cast step: the component moves
under one step a frame there, as on an easing tail, which the cast cannot show
on every frame. Where it moved under two steps a frame over the frames just
@@ -357,7 +360,8 @@ belongs to its run. A run whose control moved on fewer frames than
`minRunFrames` (1 by default) is too little to judge: the summary names it,
and the pool leaves it out, so it never counts as a pass. A run whose block
began after its motion had settled is left out the same way: its `runs` entry
-is null, and `late` holds its `BusyWindowTooLateError`. The pool fails with
+is null, and `late` holds its `BusyWindowTooLateError`. The pool reads each
+run without the starvation throw. The pool fails with
fewer than `minJudgedRuns` judged runs (half of all the runs, rounded up). In each judged run,
the component must move on every frame its control moved on, so a freeze fails
its run.
diff --git a/internal/interaction-harness/src/busy.control.test.ts b/internal/interaction-harness/src/busy.control.test.ts
index 1ec96a268a..84f1daef31 100644
--- a/internal/interaction-harness/src/busy.control.test.ts
+++ b/internal/interaction-harness/src/busy.control.test.ts
@@ -18,7 +18,7 @@
import {describe, expect, it} from 'vitest';
import {busyWindowProof, BusyWindowTooLateError, pooledBusyProof} from './busy';
-import type {TrackedFrame} from './frames';
+import {FrameStarvationError, type TrackedFrame} from './frames';
const FRAME = 1000 / 60;
const BLOCK = {from: 1000, to: 1700};
@@ -283,6 +283,34 @@ describe('busyWindowProof against a control', () => {
expect(proof.onScreenFromMs).toBeGreaterThanOrEqual(20);
});
+ it('throws FrameStarvationError for a run the machine starved: its control moved on too few frames, and no frame came for longer than 50 ms', () => {
+ const run = () =>
+ busyWindowProof(starvedCast(), BLOCK, {
+ control: asControl(starvedCast()),
+ });
+ expect(run).toThrow(FrameStarvationError);
+ expect(run).toThrow(
+ /busyWindowProof: the page drew 3 frames while a 700 ms block held the main thread, and the control moved on 3 of them, where the proof needs 4\. No frame came for 2\d\d\.\d ms/,
+ );
+ });
+
+ it('fails a component hidden inside an on-pace block as the component, not as starvation', () => {
+ // Every 60 Hz frame came, and none inside the 90 ms block showed the
+ // component: the cast's own frames show it was not starved.
+ const block = {from: 1000, to: 1090};
+ const hidden = moving().map(f =>
+ f.t > block.from && f.t < block.to
+ ? ({...f, box: null} as TrackedFrame)
+ : f,
+ );
+ const proof = busyWindowProof(hidden, block, {
+ control: asControl(moving()),
+ });
+ expect(proof.verdict.failures.join('\n')).toMatch(
+ /moved on 0 of the 3 frames the control moved/,
+ );
+ });
+
it('fails a window in which the control itself presented too little to judge', () => {
const still = asControl(
moving().map(f => ({...f, box: {...f.box!, y: 0}}) as TrackedFrame),
@@ -295,6 +323,17 @@ describe('busyWindowProof against a control', () => {
});
});
+/**
+ * `moving()` with three frames in `BLOCK`, about 200 ms apart, as a cast the
+ * machine starved delivers.
+ */
+function starvedCast(): TrackedFrame[] {
+ const kept = [at(1150), at(1350), at(1550)];
+ return moving().filter(
+ (f, i) => f.t < BLOCK.from || f.t > BLOCK.to || kept.includes(i),
+ );
+}
+
/** A 90 ms block from `from`: its judged part holds three 60 Hz frames. */
const short = (from: number) => ({from, to: from + 90});
@@ -579,6 +618,21 @@ describe('pooledBusyProof', () => {
});
});
+ it('reads a run the machine starved without throwing, and judges it with the others', () => {
+ const run = (tracked: TrackedFrame[]) => ({
+ tracked,
+ control: asControl(tracked),
+ window: BLOCK,
+ });
+ const pooled = pooledBusyProof([
+ run(moving()),
+ run(moving()),
+ run(starvedCast()),
+ ]);
+ expect(pooled.judged).toEqual([true, true, true]);
+ expect(pooled.verdict.failures, pooled.summary).toEqual([]);
+ });
+
it('throws on no runs', () => {
expect(() => pooledBusyProof([])).toThrow(/no runs/);
});
diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts
index dc272dcfcf..79e8f89612 100644
--- a/internal/interaction-harness/src/busy.ts
+++ b/internal/interaction-harness/src/busy.ts
@@ -28,9 +28,12 @@ import type {Locator, Page} from '@playwright/test';
import {defineAnimatedProperties} from './animated-properties';
import {defineCurveEnd} from './curve-end';
import {
+ FRAME_STARVATION_MS,
+ FrameStarvationError,
frameStats,
trackMarker,
type CastFrame,
+ type FrameShortfall,
type FrameStats,
type TrackedFrame,
} from './frames';
@@ -973,7 +976,25 @@ function motionStepPx(values: readonly number[]): number {
* A main-thread animation still fails: the control moves and it does not.
* `maxMedianFrameMs`, when given, still applies; `minMovedFraction` and the
* absolute stall, frame and motion limits do not.
+ *
+ * With a control, a run the machine starved of frames proves nothing. So it
+ * throws `FrameStarvationError` with its numbers and returns no verdict. A
+ * starved run has two marks. Its control moved on fewer than
+ * `minJudgedFrames` frames, and the cast showed no frame for longer than
+ * `FRAME_STARVATION_MS` in the block. Without a control, a block with no
+ * frame is a frozen animation's, and it fails the verdict. `pooledBusyProof`
+ * reads each run without the throw and judges the runs together.
*/
+export function busyWindowProof(
+ ...args: Parameters<typeof readBusyWindow>
+): BusyProof {
+ const {proof, starvation} = readBusyWindow(...args);
+ if (starvation != null) {
+ throw new FrameStarvationError(starvation);
+ }
+ return proof;
+}
+
/**
* How far around its judged part a busy proof reads frames, ms: the rest
* before it and the frame cadence on both sides. A proof that decodes only
@@ -981,7 +1002,8 @@ function motionStepPx(values: readonly number[]): number {
*/
export const BUSY_PROOF_LOOKAROUND_MS = 250;
-export function busyWindowProof(
+/** The proof `busyWindowProof` returns, and the starvation it throws. */
+function readBusyWindow(
tracked: readonly TrackedFrame[],
window: BusyWindow,
{
@@ -1065,7 +1087,7 @@ export function busyWindowProof(
*/
control?: readonly TrackedFrame[];
} = {},
-): BusyProof {
+): {proof: BusyProof; starvation: FrameShortfall | null} {
let end = window.to - marginMs;
if (settledWithin != null && settledWithinSteps != null) {
throw new Error(
@@ -1565,6 +1587,7 @@ export function busyWindowProof(
};
}
const failures: string[] = [];
+ let starvation: FrameShortfall | null = null;
if (stats.frames === 0 && controlRead == null) {
failures.push('no frame presented in the judged part');
} else if (stats.medianFrameMs > maxMedianFrameMs) {
@@ -1575,14 +1598,26 @@ export function busyWindowProof(
if (controlRead != null) {
if (controlRead.moved < minJudgedFrames) {
// How starved the cast was over the whole block, as the pacing and
- // frame errors say it.
+ // frame errors say it. Every frame the cast delivered counts, with the
+ // component on screen or not.
const blockFrames = tracked
- .filter(f => f.box != null && f.t > window.from && f.t < window.to)
+ .filter(f => f.t > window.from && f.t < window.to)
.map(f => f.t);
const castEdges = [window.from, ...blockFrames, window.to];
const castGapMs = Math.max(
...castEdges.slice(1).map((t, k) => t - castEdges[k]),
);
+ // The control is a compositor animation: when it moved too little and
+ // no frame came for that long, the machine starved the run.
+ if (castGapMs > FRAME_STARVATION_MS) {
+ starvation = {
+ by: 'busyWindowProof',
+ during: `a ${Math.round(window.to - window.from)} ms block held the main thread, and the control moved on ${controlRead.moved} of them`,
+ frames: blockFrames.length,
+ min: minJudgedFrames,
+ longestGapMs: castGapMs,
+ };
+ }
failures.push(
`the control moved on ${controlRead.moved} frames in the judged part, under ${minJudgedFrames}: the compositor presented too little in this window to judge (the cast delivered ${blockFrames.length} frame${blockFrames.length === 1 ? '' : 's'} in the ${Math.round(window.to - window.from)} ms block, none for ${Math.round(castGapMs)} ms)`,
);
@@ -1644,7 +1679,7 @@ export function busyWindowProof(
? ''
: `; against the control: moved on ${controlRead.movedWith} of its ${controlRead.moved} frames${controlRead.belowStep > 0 ? ` (${controlRead.belowStep} where the cast could not show the step)` : ''}, its longest stall ${controlRead.longestStallMs.toFixed(1)} ms`) +
notes.map(note => `; note: ${note}`).join('');
- return {
+ const proof = {
...stats,
window,
judged,
@@ -1666,6 +1701,7 @@ export function busyWindowProof(
verdict: {pass: failures.length === 0, failures},
summary,
};
+ return {proof, starvation};
}
/** One run of a pooled busy proof (see `pooledBusyProof`). */
@@ -1759,7 +1795,7 @@ export function pooledBusyProof(
const late: (BusyWindowTooLateError | null)[] = [];
const proofs = runs.map(run => {
try {
- const proof = busyWindowProof(run.tracked, run.window, {
+ const {proof} = readBusyWindow(run.tracked, run.window, {
...windowOptions,
control: run.control,
});
diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts
index b31bd97332..ff47991cc6 100644
--- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts
+++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts
@@ -1151,6 +1151,8 @@ test.describe('frame evidence', () => {
alone,
tracked,
leftAt,
+ // Every place a frame left the box before the block.
+ leftBefore: written.filter(w => w.at < window.from).map(w => w.y),
};
};
@@ -1169,31 +1171,34 @@ test.describe('frame evidence', () => {
alone: rafAlone,
tracked: rafTracked,
leftAt,
+ leftBefore,
} = await run('raf');
testInfo.annotations.push({
type: 'busy-main-thread rAF control',
description: raf.summary,
});
- // Frozen for the block: the compositor sends no frame while nothing on
- // screen changes, so at most the frame presented as the block starts
- // moved, and the bound's verdict fails it as the control's does.
- expect(raf.changed).toBeLessThanOrEqual(1);
- // Still where the last frame before the block left it, within a cast
- // step. A starved compositor can present that frame late in the block.
- const last = rafTracked
- .filter(
- f => f.box != null && f.t > raf.judged.from && f.t < raf.judged.to,
- )
- .at(-1);
+ // Frozen for the block: every frame inside it shows a place a frame left
+ // the box before the block, within a cast step. A starved compositor can
+ // present the last frames before the block late, inside it.
+ const inBlock = rafTracked.filter(
+ f => f.box != null && f.t > raf.judged.from && f.t < raf.judged.to,
+ );
expect(leftAt, 'a frame moved the box before the block').toBeDefined();
- const shown = (last?.box?.y ?? Number.NaN) - rest.cast;
+ for (const frame of inBlock) {
+ const shown = (frame.box?.y ?? Number.NaN) - rest.cast;
+ expect(
+ leftBefore.some(
+ y => Math.abs(y - rest.page - shown) <= compositor.stepPx,
+ ),
+ `the frame at +${(frame.t - raf.window.from).toFixed(0)} ms shows the box ${shown.toFixed(1)} px down, where no frame left it before the block. ${raf.summary}`,
+ ).toBe(true);
+ }
+ // The block ends where the last frame before it left the box.
+ const shown = (inBlock.at(-1)?.box?.y ?? Number.NaN) - rest.cast;
const left = (leftAt ?? Number.NaN) - rest.page;
expect(Math.abs(shown - left), raf.summary).toBeLessThanOrEqual(
compositor.stepPx,
);
- expect(raf.verdict.failures.join('\n')).toMatch(
- /moved on [01] of the \d+ frames the control moved/,
- );
expect(raf.verdict.failures.join('\n')).toMatch(
/longest stall .* longer than the control's/,
);
@@ -1213,6 +1218,46 @@ test.describe('frame evidence', () => {
await control.remove();
});
+ test('a busy proof whose cast the machine starved fails by name: FrameStarvationError', async ({
+ page,
+ }) => {
+ await busyFixture(page);
+ const box = page.locator('#box');
+ await markElement(box, {mode: 'outline'});
+ const control = await startBusyControl(page);
+ // One presented frame in 40 reaches the proof, as on a starved machine.
+ // The block then holds fewer than four of the control's frames, with gaps
+ // past FRAME_STARVATION_MS between them.
+ const cast = await startScreencast(page, {everyNthFrame: 40});
+ await box.evaluate(el =>
+ el.animate(
+ [{transform: 'translateY(0px)'}, {transform: 'translateY(560px)'}],
+ {duration: 1100, easing: 'linear'},
+ ),
+ );
+ const window = await blockMainThread(page, 700, {during: box});
+ const frames = await cast.stop();
+ const tracked = await trackMarker(page, frames);
+ const controlFrames = await trackControl(page, frames);
+ const outcome = (() => {
+ try {
+ return busyWindowProof(tracked, window, {
+ axis: 'y',
+ control: controlFrames,
+ }).summary;
+ } catch (error) {
+ return error;
+ }
+ })();
+ expect(outcome).toBeInstanceOf(FrameStarvationError);
+ const {numbers} = outcome as FrameStarvationError;
+ expect(numbers.by).toBe('busyWindowProof');
+ expect(numbers.min).toBe(4);
+ expect(numbers.frames).toBeLessThan(4);
+ expect(numbers.longestGapMs).toBeGreaterThan(FRAME_STARVATION_MS);
+ await control.remove();
+ });
+
test('a sub-pixel read follows a compositor slide under a pixel a frame through a blocked main thread; a rAF slide still fails', async ({
page,
browserName,
@@ -1309,8 +1354,10 @@ test.describe('frame evidence', () => {
const {fine: raf} = await run('raf');
expect(raf.verdict.pass, raf.summary).toBe(false);
+ // Frozen for the block, it stalls longer than the control. A starved
+ // compositor can present the last frames before the block late, inside it.
expect(raf.verdict.failures.join('\n')).toMatch(
- /moved on [01] of the \d+ frames the control moved/,
+ /longest stall .* longer than the control's/,
);
// A JPEG frame hides the marker's edges in its chroma blocks.
await expect(
@@ -1986,13 +2033,25 @@ test.describe('frame evidence', () => {
};
const compositor = await runs('compositor');
- const single = busyWindowProof(
- compositor[0].tracked,
- compositor[0].window,
- {axis: 'y', control: compositor[0].control},
- );
- expect(single.verdict.failures.join('\n'), single.summary).toMatch(
- /too little in this window to judge/,
+ // One run is too little to judge: its verdict says so, or, when the
+ // machine also starved its cast, the proof throws by name.
+ const single = (() => {
+ try {
+ const proof = busyWindowProof(
+ compositor[0].tracked,
+ compositor[0].window,
+ {axis: 'y', control: compositor[0].control},
+ );
+ return `${proof.verdict.failures.join('\n')} ${proof.summary}`;
+ } catch (error) {
+ if (!(error instanceof FrameStarvationError)) {
+ throw error;
+ }
+ return String(error);
+ }
+ })();
+ expect(single).toMatch(
+ /too little in this window to judge|FrameStarvationError: busyWindowProof/,
);
const pooled = pooledBusyProof(compositor, {axis: 'y'});
testInfo.annotations.push({
diff --git a/packages/core/src/Lightbox/__tests__/LightboxZoom.interaction.browser.spec.ts b/packages/core/src/Lightbox/__tests__/LightboxZoom.interaction.browser.spec.ts
index 2eec52e056..ec98d69a79 100644
--- a/packages/core/src/Lightbox/__tests__/LightboxZoom.interaction.browser.spec.ts
+++ b/packages/core/src/Lightbox/__tests__/LightboxZoom.interaction.browser.spec.ts
@@ -34,6 +34,7 @@ import {
curveOf,
drag,
flick,
+ FrameStarvationError,
frames,
freezePageOn,
freezePageWhen,
@@ -1242,6 +1243,11 @@ for (const pace of ['main-thread', 'frozen'] as const) {
control,
}).verdict.failures;
} catch (error) {
+ // A run the machine starved proves nothing, so it fails by name and
+ // never counts as this proof's red.
+ if (error instanceof FrameStarvationError) {
+ throw error;
+ }
// A zoom that never moved has settled before the window: a red too.
expect(error).toBeInstanceOf(BusyWindowTooLateError);
failures = [(error as Error).message];