An open source design system that's fully customizable and agent ready
TypeScript
test(interaction-harness): a proof starved of frames fails by name (#10456)
Christopher Chedeau6706e0177339
diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.mdindex a14e0e7f89..c80c36aa92 100644--- a/internal/interaction-harness/README.md+++ b/internal/interaction-harness/README.md@@ -62,7 +62,7 @@ committed: a pull request links or attaches it.| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- || `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact final-move speed, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. || `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. |-| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `trackMarkers` (several colors from one decode of each frame), `sampleRects` (per-rAF rects), `frameStats`. |+| `frames.ts` | `startScreencast` (timestamped composited frames), `markElement` (a marker chip or outline), `trackMarker` (marker box per frame), `trackMarkers` (several colors from one decode of each frame), `sampleRects` (per-rAF rects), `frameStats`, `requireFrames`. || `busy.ts` | `blockMainThread(page, ms)`, which returns the window it held and the span its loop measured, and `busyWindowProof(tracked, window)`: frames presented, frames that moved, longest stall, a verdict; `presented` adds the compositor's own frames. || `brightness.ts` | `trackBrightness(page, frames, {x, y})`: a colour channel's mean in a square per frame, as a scalar on `box.x`, for a busy-window proof of a fade or a dim. || `webInput.ts` | `webInputDriver(page)`: `wheel({x, y, dx, dy, ctrl})` and `pen(points)` as real engine input, for proofs across pointers (`spec:AST-100` FR4); `wheel` resolves once the page has the step; read back through the input log. |@@ -350,6 +350,16 @@ compositor's frames from the first to the last moved frame; the gaps before andafter are the stall check's. WebKit has no such trace:`stop()` resolves to `[]` and the screencast is judged alone.+**Too few frames.** A proof that samples the page every frame (`sampleRects`,+or a rAF trace of its own) takes its frames through `requireFrames(samples,+{min, during, where})`. It returns the samples that `where` selects when there+are at least `min` of them. A loaded machine, or a long task on the page, can+starve the page of frames. When no frame came for longer than+`FRAME_STARVATION_MS` (50 ms) around the selected samples, the proof fails+with `FrameStarvationError`, which names the frames drawn and the longest gap.+So a run that cannot prove anything still fails, and says why. With too few frames on pace, the motion did not last that+long, and the error says that instead.+**No Layout / Paint.** `traceRendering` records a Chromium trace around theinteraction; `countBetween` counts the steady middle of an animation (itsfirst and last frames may commit). A transform/opacity animation does nodiff --git a/internal/interaction-harness/src/frames.requireFrames.test.ts b/internal/interaction-harness/src/frames.requireFrames.test.tsnew file mode 100644index 0000000000..792304123e--- /dev/null+++ b/internal/interaction-harness/src/frames.requireFrames.test.ts@@ -0,0 +1,89 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file frames.requireFrames.test.ts+ * @input frames.ts's frame-count precondition for proofs+ * @output Unit cases for `requireFrames`. Enough selected frames come back.+ * Too few, where no frame came for longer than FRAME_STARVATION_MS, fail+ * with FrameStarvationError and its numbers. Too few on pace fail with a+ * plain Error. With none selected, every gap counts.+ * @position Node unit tests beside the harness's browser self-tests.+ *+ * SYNC: When modified, update frames.ts (requireFrames).+ */++import {describe, expect, it} from 'vitest';+import {+ FRAME_STARVATION_MS,+ FrameStarvationError,+ requireFrames,+} from './frames';++/** Frames at these page times, ms, each with the value it showed. */+const trace = (times: readonly number[]) =>+ times.map((t, index) => ({t, value: index}));++describe('requireFrames', () => {+ it('gives back the selected frames when there are enough', () => {+ const frames = trace([0, 16.7, 33.3, 50, 66.7, 83.3, 100]);+ const selected = requireFrames(frames, {+ min: 3,+ during: 'a slide',+ where: frame => frame.t >= 50,+ });+ expect(selected.map(frame => frame.t)).toEqual([50, 66.7, 83.3, 100]);+ });++ it('names a starved page, with the longest gap around the selected frames', () => {+ // Two frames inside the motion, 180 ms after the frame before them.+ const frames = trace([0, 20, 200, 220, 400]);+ let error: unknown;+ try {+ requireFrames(frames, {+ min: 6,+ during: 'the screen slides in',+ where: frame => frame.t > 100 && frame.t < 300,+ });+ } catch (thrown) {+ error = thrown;+ }+ expect(error).toBeInstanceOf(FrameStarvationError);+ expect((error as FrameStarvationError).numbers).toEqual({+ during: 'the screen slides in',+ frames: 2,+ min: 6,+ longestGapMs: 180,+ });+ expect(String(error)).toMatch(+ /the page drew 2 frames while the screen slides in, where the proof needs 6\. No frame came for 180\.0 ms: the page was starved of frames/,+ );+ });++ it('fails a motion too short for the proof with a plain error, not as starvation', () => {+ const frames = trace([0, 16.7, 33.3, 50, 66.7]);+ let error: unknown;+ try {+ requireFrames(frames, {+ min: 6,+ during: 'a fade',+ where: frame => frame.t >= 16.7 && frame.t <= 33.3,+ });+ } catch (thrown) {+ error = thrown;+ }+ expect(error).toBeInstanceOf(Error);+ expect(error).not.toBeInstanceOf(FrameStarvationError);+ expect(String(error)).toMatch(/the motion did not last that long/);+ });++ it('counts every gap when no frame is selected', () => {+ const starved = trace([0, 16.7, 16.7 + FRAME_STARVATION_MS + 1]);+ expect(() =>+ requireFrames(starved, {min: 1, during: 'a push', where: () => false}),+ ).toThrow(FrameStarvationError);+ const onPace = trace([0, 16.7, 33.3]);+ expect(() =>+ requireFrames(onPace, {min: 1, during: 'a push', where: () => false}),+ ).toThrow(/the motion did not last that long/);+ });+});diff --git a/internal/interaction-harness/src/frames.ts b/internal/interaction-harness/src/frames.tsindex f8b8b4518e..89fc41475b 100644--- a/internal/interaction-harness/src/frames.ts+++ b/internal/interaction-harness/src/frames.ts@@ -10,6 +10,8 @@* marker box and the cast's position step). `trackMarkers` tracks several* colors from one decode of each frame. Also `colorBlockPx` (a frame's color* resolution), `sampleRects` (per-rAF rect sampling) and `frameStats`.+ * `requireFrames` gives a proof its frames, or fails it by name when the+ * page drew too few.* @position Frame evidence for motion proofs. The screencast is captured off* the page's main thread, from what the compositor presents, so it keeps* delivering frames while a script blocks the main thread: that is what makes@@ -792,3 +794,86 @@ export function frameStats(last: inside[inside.length - 1]?.value ?? null,};}++/**+ * A gap between two frames the page drew, ms, past which the page counts as+ * starved: about three frames at 60 Hz.+ */+export const FRAME_STARVATION_MS = 50;++const framesDrawn = (count: number) =>+ `${count} ${count === 1 ? 'frame' : 'frames'}`;++/** A proof's frames when the page drew too few of them. */+export interface FrameShortfall {+ /** What the frames had to show. */+ readonly during: string;+ /** The frames the page drew during it. */+ readonly frames: number;+ /** The frames the proof needs. */+ readonly min: number;+ /** The longest gap between frames around them, ms. */+ readonly longestGapMs: number;+}++/**+ * The page drew too few frames for a proof, because it was starved of+ * frames: no frame came for longer than `FRAME_STARVATION_MS`. A loaded+ * machine does that, and so does a long task on the page. Such a run proves+ * nothing about the motion, so it fails, and the error says why.+ */+export class FrameStarvationError extends Error {+ readonly numbers: FrameShortfall;+ constructor(numbers: FrameShortfall) {+ super(+ `requireFrames: the page drew ${framesDrawn(numbers.frames)} while ${numbers.during}, where the proof needs ${numbers.min}. No frame came for ${numbers.longestGapMs.toFixed(1)} ms: the page was starved of frames (a loaded machine, or a long task on the page), so the run proves nothing about the motion`,+ );+ this.name = 'FrameStarvationError';+ this.numbers = numbers;+ }+}++/**+ * The samples that `where` selects from the page's own per-frame `samples`+ * (an rAF trace), when there are at least `min` of them. Else the proof+ * fails. The error is `FrameStarvationError` when no frame came for longer+ * than `FRAME_STARVATION_MS` around the selected samples. Otherwise the+ * motion did not last that long, and the error says so. `during` names what+ * the frames had to show.+ */+export function requireFrames<T extends {readonly t: number}>(+ samples: readonly T[],+ {+ min,+ during,+ where = () => true,+ }: {+ readonly min: number;+ readonly during: string;+ readonly where?: (sample: T) => boolean;+ },+): T[] {+ const selected = samples.filter(where);+ if (selected.length >= min) {+ return selected;+ }+ // The gaps from the frame before the first selected sample to the frame+ // after the last one. With none selected, every gap counts.+ const first = selected.length > 0 ? samples.indexOf(selected[0]) : 0;+ const last =+ selected.length > 0+ ? samples.indexOf(selected[selected.length - 1])+ : samples.length - 1;+ const around = samples.slice(Math.max(0, first - 1), last + 2);+ const longestGapMs = Math.max(+ 0,+ ...around.slice(1).map((sample, i) => sample.t - around[i].t),+ );+ const numbers = {during, frames: selected.length, min, longestGapMs};+ if (longestGapMs > FRAME_STARVATION_MS) {+ throw new FrameStarvationError(numbers);+ }+ throw new Error(+ `requireFrames: the page drew ${framesDrawn(selected.length)} while ${during}, where the proof needs ${min}. No frame came more than ${FRAME_STARVATION_MS} ms after the one before: the motion did not last that long`,+ );+}diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.tsindex ebc1de162f..bebd426d15 100644--- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts+++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts@@ -75,6 +75,8 @@ import {drag,flick,frames,+ FRAME_STARVATION_MS,+ FrameStarvationError,freezePage,freezePageOn,freezePageWhen,@@ -97,6 +99,7 @@ import {readInputLog,readLongTasks,releaseAnimations,+ requireFrames,retakeOnWheelPacing,sampleCurve,sampleRects,@@ -1888,6 +1891,40 @@ test.describe('frame evidence', () => {);});+ test('a proof whose page the machine starves of frames fails by name: FrameStarvationError', async ({+ page,+ }) => {+ await page.setContent(+ '<!doctype html><div id="box" style="width:40px;height:40px;background:#000"></div>',+ );+ const box = page.locator('#box');+ // On pace, a 300 ms sample holds enough frames.+ const onPace = await sampleRects(box, 300);+ expect(requireFrames(onPace, {min: 6, during: 'a 300 ms sample'})).toEqual(+ onPace,+ );+ // Each frame now comes 100 ms after the one before, as on a starved+ // machine.+ await page.evaluate(() => {+ const frame = window.requestAnimationFrame.bind(window);+ window.requestAnimationFrame = callback => {+ setTimeout(() => frame(callback), 100);+ return 0;+ };+ });+ const starved = await sampleRects(box, 300);+ let outcome: unknown = 'resolved';+ try {+ requireFrames(starved, {min: 6, during: 'a 300 ms sample'});+ } catch (error) {+ outcome = error;+ }+ expect(outcome).toBeInstanceOf(FrameStarvationError);+ const {numbers} = outcome as FrameStarvationError;+ expect(numbers.frames).toBeLessThan(6);+ expect(numbers.longestGapMs).toBeGreaterThan(FRAME_STARVATION_MS);+ });+test('the trace counts no Layout for a transform animation and one per frame for `top`', async ({page,browserName,diff --git a/packages/lab/src/NavigationStack/__tests__/NavigationStack.interaction.browser.spec.ts b/packages/lab/src/NavigationStack/__tests__/NavigationStack.interaction.browser.spec.tsindex 35b600dc25..9be3c75d8c 100644--- a/packages/lab/src/NavigationStack/__tests__/NavigationStack.interaction.browser.spec.ts+++ b/packages/lab/src/NavigationStack/__tests__/NavigationStack.interaction.browser.spec.ts@@ -39,6 +39,7 @@ import {markElement,openStory,releaseAnimations,+ requireFrames,sampleCurve,sampleRects,startBusyControl,@@ -534,11 +535,13 @@ test('a push’s bar: the new bar slides in with its screen, the covered one ridawait page.waitForTimeout(900);const frames = await stop();const start = firstMove(frames, 'screen-General');- expect(start).toBeDefined();- const during = frames.filter(- f => f.t >= (start?.t ?? 0) && f.cards['screen-General'] > 0.5,- );- expect(during.length).toBeGreaterThan(5);+ expect(start, 'the push started').toBeDefined();+ const during = requireFrames(frames, {+ min: 6,+ during: 'the pushed screen slides in',+ where: f =>+ start != null && f.t >= start.t && f.cards['screen-General'] > 0.5,+ });for (const f of during) {// UIKit's bar copy is part of its screen: it moves with the card (one// clock and spring). It stays within 1 % of the width, as engines sample@@ -1352,13 +1355,14 @@ test.describe('the bars ride with their screens, as UIKit’s', () => {await page.locator('[data-navigation-back]').click();await page.waitForTimeout(900);const frames = await stop();- const moving = frames.filter(- f =>+ const moving = requireFrames(frames, {+ min: 6,+ during: 'the popped screen and the revealed one move',+ where: f =>f.cards['screen-General'] != null &&f.cards['screen-General'] > 0.5 &&f.cards.root < -0.5,- );- expect(moving.length).toBeGreaterThan(5);+ });for (const f of moving) {expect(Math.abs(f.bars['screen-General'] - f.cards['screen-General']),