An open source design system that's fully customizable and agent ready
TypeScript
test(harness): a pooled busy proof leaves out a starved run by name (#10318)
Christopher Chedeaud98f4ea5feb6
diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.mdindex 44e789c097..efbc2e580f 100644--- a/internal/interaction-harness/README.md+++ b/internal/interaction-harness/README.md@@ -290,7 +290,12 @@ and judge the runs pooled (`pooledBusyProof(runs)`, each run with its owntracked frames, control and window): the control must move on at least fourframes over all the runs, the component on every one of them, and in each runits longest stall may be no longer than that run's control's, since a stall-belongs to its run.+belongs to its run. A run whose control moved on fewer frames than+`minRunFrames` (1 by default) is too little to judge: the summary names it,+and the pool leaves it out, so it never counts as a pass. The pool fails with+fewer than `minJudgedRuns` judged runs (half, rounded up). In each judged run,+the component must move on every frame its control moved on, so a freeze fails+its run.On a loaded machine Chromium's screencast can skip frames the compositorstill presents. `startCompositorTrace(page)`, started with the screencast anddiff --git a/internal/interaction-harness/src/busy.control.test.ts b/internal/interaction-harness/src/busy.control.test.tsindex dcc283d126..2c6d2db964 100644--- a/internal/interaction-harness/src/busy.control.test.ts+++ b/internal/interaction-harness/src/busy.control.test.ts@@ -413,9 +413,89 @@ describe('pooledBusyProof', () => {window: short(from),})),);- expect(pooled.verdict.failures.join('\n')).toMatch(- /the control moved on 0 frames over 2 runs, under 4/,+ expect(pooled.verdict.failures).toEqual([+ '0 of 2 runs judged, under 1: the compositor presented too little in the others to judge',+ ]);+ });++ /** `frames` held at the position they had at `from`, from `from` to `to`. */+ const held = (frames: TrackedFrame[], from: number, to: number) =>+ frames.map(f =>+ f.t >= from && f.t <= to+ ? ({...f, box: {...f.box!, y: 3 * at(from)}} as TrackedFrame)+ : f,+ );++ it('names a run whose control presented nothing as too little to judge and leaves it out: the other runs decide', () => {+ // The control stands still through run 2's block only, as if the machine+ // starved it there.+ const pooled = pooledBusyProof(+ [1000, 1300, 1600].map(from => ({+ tracked: moving(),+ control: asControl(held(moving(), 1300, 1390)),+ window: short(from),+ })),+ );+ expect(pooled.verdict.failures, pooled.summary).toEqual([]);+ expect(pooled.judged).toEqual([true, false, true]);+ expect(pooled.control).toEqual({moved: 6, movedWith: 6});+ expect(pooled.summary).toMatch(+ /^3 runs pooled, 2 judged: .*; run 2: too little to judge, the control moved on 0 frames, under 1: /,+ );+ });++ it('fails a pool in which every run but one was too little to judge, even when that run holds enough frames', () => {+ const windows = [1000, 1300, 1600].map(from => ({from, to: from + 200}));+ const starved = held(held(moving(), 1300, 1500), 1600, 1800);+ const pooled = pooledBusyProof(+ windows.map(window => ({+ tracked: moving(),+ control: asControl(starved),+ window,+ })),+ );+ expect(pooled.judged).toEqual([true, false, false]);+ expect(pooled.control.moved).toBeGreaterThanOrEqual(4);+ expect(pooled.verdict.failures.join('\n'), pooled.summary).toMatch(+ /1 of 3 runs judged, under 2: the compositor presented too little in the others to judge/,+ );+ });++ it('judges a run in which the component froze while its control moved: it is never left out, and it fails on its own', () => {+ const pooled = pooledBusyProof(+ [1000, 1300, 1600].map(from => ({+ tracked: held(moving(), 1300, 1390),+ control: asControl(moving()),+ window: short(from),+ })),);+ expect(pooled.judged).toEqual([true, true, true]);+ const failures = pooled.verdict.failures.join('\n');+ expect(failures).toMatch(+ /moved on 6 of the 9 frames the control moved over 3 runs/,+ );+ expect(failures).toMatch(+ /run 2: moved on 0 of the 3 frames its control moved/,+ );+ expect(failures).not.toMatch(/run [13]: moved on/);+ expect(failures).toMatch(+ /run 2: longest stall .* longer than the control's/,+ );+ });++ it('throws when it needs more judged runs than it has, or none', () => {+ expect(() =>+ pooledBusyProof(+ [{tracked: moving(), control: asControl(moving()), window: BLOCK}],+ {minJudgedRuns: 2},+ ),+ ).toThrow(/minJudgedRuns 2 is not between 1 and the 1 runs/);+ expect(() =>+ pooledBusyProof(+ [{tracked: moving(), control: asControl(moving()), window: BLOCK}],+ {minJudgedRuns: 0},+ ),+ ).toThrow(/minJudgedRuns 0 is not between 1 and the 1 runs/);});it('judges one long run as busyWindowProof judges it against the control', () => {diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.tsindex 40a25d2a4b..55f3f423a4 100644--- a/internal/interaction-harness/src/busy.ts+++ b/internal/interaction-harness/src/busy.ts@@ -1524,8 +1524,13 @@ export interface PooledBusyProof {*/readonly runs: readonly BusyProof[];/**- * The control's new positions over every run's judged part, and those the- * component moved on too.+ * Whether each run was judged. A run whose control moved on fewer than+ * `minRunFrames` frames is too little to judge, and the pool leaves it out.+ */+ readonly judged: readonly boolean[];+ /**+ * The control's new positions over the judged runs' judged parts, and those+ * the component moved on too.*/readonly control: {readonly moved: number; readonly movedWith: number};/** The judgment over the pool (see `pooledBusyProof`). */@@ -1539,12 +1544,20 @@ export interface PooledBusyProof {* control. An animation that lasts a fraction of a second leaves a block only* a few frames to judge: in WebKit, or on a loaded machine, the control can* present fewer than `minJudgedFrames` in one run, and one run cannot decide.- * Pooled, the runs are judged as one long window is:- * - the control moved on at least `minJudgedFrames` frames over all the runs;- * - the component moved on every one of those frames;- * - in each run, its longest stall is no longer than that run's control's: a- * stall belongs to its run and is never pooled.- * `maxMedianFrameMs`, when given, applies to each run. A main-thread+ * A run whose control moved on fewer than `minRunFrames` frames (1 by+ * default) is too little to judge: the machine starved it. The summary names+ * it, and the pool leaves it out, so it never counts as a pass.+ * The judged runs are judged as one long window is:+ * - at least `minJudgedRuns` runs are judged (by default, half the runs,+ * rounded up);+ * - the control moved on at least `minJudgedFrames` frames over them;+ * - in each judged run, the component moved on every frame its control moved+ * on, so a freeze fails its run and is never pooled. A run of unchanged+ * frames counts as motion only where the cast cannot show the motion (see+ * `busyWindowProof`).+ * - in each judged run, its longest stall is no longer than that run's+ * control's.+ * `maxMedianFrameMs`, when given, applies to every run. A main-thread* animation still fails: in every run the control moves and it does not.*/export function pooledBusyProof(@@ -1552,37 +1565,64 @@ export function pooledBusyProof(options: Omit<NonNullable<Parameters<typeof busyWindowProof>[2]>,'control' | 'minMovedFraction'- > = {},+ > & {+ /** The fewest frames a run's control must move on for the run to count. */+ minRunFrames?: number;+ /** The fewest judged runs the pool needs. */+ minJudgedRuns?: number;+ } = {},): PooledBusyProof {if (runs.length === 0) {throw new Error('pooledBusyProof: no runs to judge');}- const {minJudgedFrames = 4, maxMedianFrameMs = Infinity} = options;+ const {+ minRunFrames = 1,+ minJudgedRuns = Math.ceil(runs.length / 2),+ ...windowOptions+ } = options;+ const {minJudgedFrames = 4, maxMedianFrameMs = Infinity} = windowOptions;+ if (minJudgedRuns < 1 || minJudgedRuns > runs.length) {+ throw new Error(+ `pooledBusyProof: minJudgedRuns ${minJudgedRuns} is not between 1 and the ${runs.length} runs`,+ );+ }const proofs = runs.map(run =>busyWindowProof(run.tracked, run.window, {- ...options,+ ...windowOptions,control: run.control,}),);const reads = proofs.map(proof => proof.control as BusyControlRead);- const moved = reads.reduce((sum, read) => sum + read.moved, 0);- const movedWith = reads.reduce((sum, read) => sum + read.movedWith, 0);+ const judged = reads.map(read => read.moved >= minRunFrames);+ const judgedReads = reads.filter((_, i) => judged[i]);+ const total = (key: 'moved' | 'movedWith' | 'belowStep') =>+ judgedReads.reduce((sum, read) => sum + read[key], 0);+ const moved = total('moved');+ const movedWith = total('movedWith');+ const over =+ judgedReads.length === runs.length+ ? `${runs.length} runs`+ : `the ${judgedReads.length} judged runs of ${runs.length}`;const failures: string[] = [];- if (moved < minJudgedFrames) {+ if (judgedReads.length < minJudgedRuns) {failures.push(- `the control moved on ${moved} frames over ${runs.length} runs, under ${minJudgedFrames}: the compositor presented too little in these windows to judge`,+ `${judgedReads.length} of ${runs.length} runs judged, under ${minJudgedRuns}: the compositor presented too little in the others to judge`,);}- const visible =- movedWith - reads.reduce((sum, read) => sum + read.belowStep, 0);- if (visible < minJudgedFrames) {+ if (judgedReads.length > 0 && moved < minJudgedFrames) {failures.push(- `the component showed a new position on ${visible} of the control's frames over ${runs.length} runs, under ${minJudgedFrames}: too little motion in these windows to judge`,+ `the control moved on ${moved} frames over ${over}, under ${minJudgedFrames}: the compositor presented too little in these windows to judge`,+ );+ }+ const visible = movedWith - total('belowStep');+ if (judgedReads.length > 0 && visible < minJudgedFrames) {+ failures.push(+ `the component showed a new position on ${visible} of the control's frames over ${over}, under ${minJudgedFrames}: too little motion in these windows to judge`,);}if (movedWith < moved) {failures.push(- `moved on ${movedWith} of the ${moved} frames the control moved over ${runs.length} runs (${((movedWith / Math.max(1, moved)) * 100).toFixed(0)}%)`,+ `moved on ${movedWith} of the ${moved} frames the control moved over ${over} (${((movedWith / Math.max(1, moved)) * 100).toFixed(0)}%)`,);}proofs.forEach((proof, i) => {@@ -1591,7 +1631,15 @@ export function pooledBusyProof(`run ${i + 1}: median frame interval ${proof.medianFrameMs.toFixed(1)} ms > ${maxMedianFrameMs} ms`,);}+ if (!judged[i]) {+ return;+ }const read = reads[i];+ if (read.movedWith < read.moved) {+ failures.push(+ `run ${i + 1}: moved on ${read.movedWith} of the ${read.moved} frames its control moved`,+ );+ }if (read.componentStallMs > read.longestStallMs) {failures.push(`run ${i + 1}: longest stall ${read.componentStallMs.toFixed(1)} ms, longer than the control's ${read.longestStallMs.toFixed(1)} ms`,@@ -1599,10 +1647,21 @@ export function pooledBusyProof(}});const summary =- `${runs.length} runs pooled: moved on ${movedWith} of the control's ${moved} frames` +- proofs.map((proof, i) => `; run ${i + 1}: ${proof.summary}`).join('');+ `${runs.length} runs pooled` ++ (judgedReads.length === runs.length+ ? ''+ : `, ${judgedReads.length} judged`) ++ `: moved on ${movedWith} of the control's ${moved} frames` ++ proofs+ .map((proof, i) =>+ judged[i]+ ? `; run ${i + 1}: ${proof.summary}`+ : `; run ${i + 1}: too little to judge, the control moved on ${reads[i].moved} frames, under ${minRunFrames}: ${proof.summary}`,+ )+ .join('');return {runs: proofs,+ judged,control: {moved, movedWith},verdict: {pass: failures.length === 0, failures},summary,diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.tsindex 113fd13647..ba634c6e02 100644--- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts+++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts@@ -1255,6 +1255,10 @@ test.describe('frame evidence', () => {const box = page.locator('#box');await markElement(box, {mode: 'outline'});const control = await startBusyControl(page);+ const rest = await box.boundingBox();+ if (rest == null) {+ throw new Error('no box');+ }// Six 70 ms blocks: each judged part (20 ms margins, 30 ms) holds at most// three frames even at WebKit's 14 ms cast, under the four one window@@ -1307,8 +1311,18 @@ test.describe('frame evidence', () => {await page.waitForTimeout(150);}const frames = await cast.stop();- const tracked = await trackMarker(page, frames);- const controlFrames = await trackMarker(page, frames, {rgb: CONTROL_RGB});+ // Each search reads only the strip its box crosses. On a loaded machine+ // a search of every pixel, four drivers by two tracks, held most of the+ // test's time. The box travels 560 px down from where it rests.+ const tracked = await trackMarker(page, frames, {+ region: {+ x: rest.x - 20,+ y: rest.y - 16,+ width: rest.width + 40,+ height: rest.height + 560 + 32,+ },+ });+ const controlFrames = await trackControl(page, frames);return windows.map(window => ({tracked, control: controlFrames, window}));};@@ -1328,6 +1342,34 @@ test.describe('frame evidence', () => {});expect(pooled.verdict.failures, pooled.summary).toEqual([]);+ // The same runs with the control taken out of the last blocks. A starved+ // run is named and left out. With too few judged runs, the pool fails.+ const starve = (count: number) =>+ pooledBusyProof(+ compositor.map((run, i) =>+ i < compositor.length - count+ ? run+ : {+ ...run,+ control: run.control.map(frame =>+ frame.t >= run.window.from && frame.t <= run.window.to+ ? {...frame, box: null}+ : frame,+ ),+ },+ ),+ {axis: 'y'},+ );+ const lastStarved = starve(1);+ expect(lastStarved.judged[5], lastStarved.summary).toBe(false);+ expect(lastStarved.summary).toMatch(+ /; run 6: too little to judge, the control moved on 0 frames/,+ );+ const oneLeft = starve(5);+ expect(oneLeft.verdict.failures.join('\n'), oneLeft.summary).toMatch(+ /[01] of 6 runs judged, under 3/,+ );+// The mutant: a main-thread animation freezes in every block.const raf = pooledBusyProof(await runs('raf'), {axis: 'y'});testInfo.annotations.push({@@ -1336,7 +1378,7 @@ test.describe('frame evidence', () => {});expect(raf.verdict.pass, raf.summary).toBe(false);expect(raf.verdict.failures.join('\n')).toMatch(- /moved on \d+ of the \d+ frames the control moved over 6 runs/,+ /moved on \d+ of the \d+ frames the control moved over (6 runs|the \d judged runs of 6)/,);// A frame-counted loop stalls in every block too and then steps on, one