Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

test(interaction-harness): a paced touch play keeps its spacing after a stalled dispatch and reads the release the page read (#9725)

Christopher Chedeaub35fe838021d
diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md
index 0f9e0381a9..3a85354968 100644
--- a/internal/interaction-harness/README.md
+++ b/internal/interaction-harness/README.md
@@ -378,22 +378,22 @@ time: it fast-forwards whenever the page is idle.
## Engine limits (measured with Playwright 1.61)
-| | Chromium 149 | WebKit 26.5 (Linux build) |
-| ------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Trusted touch + pointer events, multi-touch | yes | yes (a `touchstart` lists only the fingers that landed in it; pointer events exact) |
-| Page sees the scripted timestamps | yes, exactly | no: real dispatch time; pacing keeps velocities within ~1 ms on an idle runner, but a CPU-starved one sends some steps late, then back to back |
-| Sub-pixel touch positions | yes | no: integer CSS px |
-| Native scroll, fling, pinch-zoom from injected touches | yes | no |
-| Lifting one finger while others stay down | yes | pointer events right, but that `touchend` lists no remaining `touches`: skip with `PARTIAL_LIFT_SKIP` |
-| Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence |
-| Curve sampling, storyboards, input log | yes | yes |
-| A CSS animation shorter than a gap between two frames | yes: a pending animation starts at the next frame, and runs then | no: it can start and finish inside the gap, and its start event, sent at the next frame, finds nothing to record; `waitForAnimation` names it |
-| Screencast frame evidence (busy main thread) | yes | yes (WebKit's own screencast; frames keep coming while the main thread is blocked) |
-| Rendering trace, long tasks | yes | no (Playwright exposes neither) |
-| Compositor presentation record | yes | no: `stop()` resolves to `[]`; the busy verdict judges the screencast alone |
-| Wheel steps, exact CSS px deltas (`webInput.ts`) | yes | desktop context only (`isMobile: false`): skip with `WHEEL_SKIP`; the step also moves the mouse there (a `pointermove`) and holds Control as a key |
-| Pen strokes, pressure, scripted timestamps | yes | no: skip with `PEN_SKIP` |
-| Pen over a native scroller | no pan: the pen takes the mouse pipeline, so a stroke scrolls nothing, raises no `pointercancel` and ends in a `click` | no: skip with `PEN_SKIP` |
+| | Chromium 149 | WebKit 26.5 (Linux build) |
+| ------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Trusted touch + pointer events, multi-touch | yes | yes (a `touchstart` lists only the fingers that landed in it; pointer events exact) |
+| Page sees the scripted timestamps | yes, exactly | no: real dispatch time; pacing keeps velocities within ~1 ms on an idle runner, but a CPU-starved one sends some steps late, then back to back; `pacing: 'spacing'` keeps the script's spacing after a stall instead, and fails a play whose page read a resting finger where the script moves, a throw where the script rests, or fewer than two moves, or, given `releaseTolerance`, a release that strays further than that from the script's (each read as Core's velocity tracker reads it): `TouchPageStallError` when the page's own touch or pointer listeners held an event back, `TouchPacingError` otherwise; the lift's `release` names what the page read |
+| Sub-pixel touch positions | yes | no: integer CSS px |
+| Native scroll, fling, pinch-zoom from injected touches | yes | no |
+| Lifting one finger while others stay down | yes | pointer events right, but that `touchend` lists no remaining `touches`: skip with `PARTIAL_LIFT_SKIP` |
+| Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence |
+| Curve sampling, storyboards, input log | yes | yes |
+| A CSS animation shorter than a gap between two frames | yes: a pending animation starts at the next frame, and runs then | no: it can start and finish inside the gap, and its start event, sent at the next frame, finds nothing to record; `waitForAnimation` names it |
+| Screencast frame evidence (busy main thread) | yes | yes (WebKit's own screencast; frames keep coming while the main thread is blocked) |
+| Rendering trace, long tasks | yes | no (Playwright exposes neither) |
+| Compositor presentation record | yes | no: `stop()` resolves to `[]`; the busy verdict judges the screencast alone |
+| Wheel steps, exact CSS px deltas (`webInput.ts`) | yes | desktop context only (`isMobile: false`): skip with `WHEEL_SKIP`; the step also moves the mouse there (a `pointermove`) and holds Control as a key |
+| Pen strokes, pressure, scripted timestamps | yes | no: skip with `PEN_SKIP` |
+| Pen over a native scroller | no pan: the pen takes the mouse pipeline, so a stroke scrolls nothing, raises no `pointercancel` and ends in a `click` | no: skip with `PEN_SKIP` |
What WebKit cannot show here — native scroll hand-off, flings, a finger
lifting while another stays down, finger tracking under a busy main thread, and
@@ -424,9 +424,11 @@ Otherwise the dispatch stalled before the page (a loaded runner) and it throws
`WheelPacingError` with the gaps. Wrap such a
proof in `retakeOnWheelPacing(testInfo, attempt)`: one re-take, only on
`WheelPacingError`, listed as a `wheel-retake` annotation. The burst times the
-page's handling with a listener on the window that runs after the page's own,
-so a listener that stops the step's propagation, or other main-thread work
-between steps (timers, frames), still counts as the dispatch's stall. Chromium
+page's handling with a listener on the window that runs after the page's own.
+A step whose propagation the page stops is marked by a task queued when it
+arrived, and never later than the next step's arrival. So its handling, with
+any work queued before that task, counts as the page's. Other main-thread work
+between steps (timers, frames) still counts as the dispatch's stall. Chromium
stamps a step with its dispatch time, so a page that holds its main thread does
not widen the gaps there, and every gap past `maxGapMs` is a `WheelPacingError`.
diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts
index 0257f0a019..512b799d59 100644
--- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts
+++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts
@@ -99,6 +99,7 @@ import {
thawTimers,
timeline,
touchDriver,
+ TouchPageStallError,
touchStoryboard,
traceRendering,
trackMarker,
@@ -363,6 +364,75 @@ test.describe('touch input', () => {
}
});
+ test("a page whose pointer listener holds its main thread changes a paced flick's release itself: TouchPageStallError, never the runner's", async ({
+ page,
+ browserName,
+ }) => {
+ test.skip(
+ browserName !== 'webkit',
+ 'only WebKit stamps a touch when it arrives; Chromium carries the scripted times',
+ );
+ // Each pointer move holds the main thread for 60 ms, as a component's
+ // pan handler can. A gesture host samples a finger from pointermove.
+ // WebKit hands the page the next touch only once it is free.
+ await page.route('http://pointer-stalling-page.test/', route =>
+ route.fulfill({
+ contentType: 'text/html',
+ body: '<!doctype html><body style="margin:0;height:3000px;touch-action:none"><script>addEventListener("pointermove",()=>{const end=performance.now()+60;while(performance.now()<end){}});</script>',
+ }),
+ );
+ await page.goto('http://pointer-stalling-page.test/');
+ const driver = await touchDriver(page);
+ const outcome = await driver
+ .play(
+ drag({x: 200, y: 400}, {x: 200, y: 300}, {durationMs: 100, hz: 60}),
+ {pacing: 'spacing'},
+ )
+ .then(
+ () => 'resolved',
+ (error: unknown) => error,
+ );
+ expect(outcome).toBeInstanceOf(TouchPageStallError);
+ expect(String(outcome)).toMatch(
+ /the page's own touch and pointer listeners took \d+ ms/,
+ );
+ });
+
+ test("a page whose touch listener holds its main thread changes a paced flick's release itself: TouchPageStallError, never the runner's", async ({
+ page,
+ browserName,
+ }) => {
+ test.skip(
+ browserName !== 'webkit',
+ 'only WebKit stamps a touch when it arrives; Chromium carries the scripted times',
+ );
+ // Each move holds the main thread for 40 ms, past the script's 17 ms
+ // spacing, and WebKit hands the page the next touch only once it is free.
+ await page.route('http://touch-stalling-page.test/', route =>
+ route.fulfill({
+ contentType: 'text/html',
+ body: '<!doctype html><body style="margin:0;height:3000px;touch-action:none"><script>addEventListener("touchmove",e=>{e.preventDefault();const end=performance.now()+40;while(performance.now()<end){}},{passive:false});</script>',
+ }),
+ );
+ await page.goto('http://touch-stalling-page.test/');
+ const driver = await touchDriver(page);
+ const outcome = await driver
+ .play(
+ drag({x: 200, y: 400}, {x: 200, y: 300}, {durationMs: 100, hz: 60}),
+ {
+ pacing: 'spacing',
+ },
+ )
+ .then(
+ () => 'resolved',
+ (error: unknown) => error,
+ );
+ expect(outcome).toBeInstanceOf(TouchPageStallError);
+ expect(String(outcome)).toMatch(
+ /the page's own touch and pointer listeners took \d+ ms/,
+ );
+ });
+
test('a flick flings a native scroller, farther for a faster finger', async ({
page,
browserName,
@@ -1773,6 +1843,56 @@ test.describe('wheel and pen input', () => {
}
});
+ test('a page whose wheel listener stops the step and holds its main thread still splits a burst itself: never re-taken', async ({
+ browser,
+ browserName,
+ }) => {
+ const context = await browser.newContext({
+ viewport: {width: 402, height: 714},
+ isMobile: false,
+ hasTouch: false,
+ });
+ try {
+ const page = await context.newPage();
+ // The body's listener stops each step before it reaches the window,
+ // and blocks the main thread for 150 ms, past the 100 ms bound.
+ await page.route('http://wheel-stopping-page.test/', route =>
+ route.fulfill({
+ contentType: 'text/html',
+ body: '<!doctype html><body style="margin:0;height:3000px"><script>document.body.addEventListener("wheel",e=>{e.preventDefault();e.stopPropagation();const end=performance.now()+150;while(performance.now()<end){}},{passive:false});</script>',
+ }),
+ );
+ await page.goto('http://wheel-stopping-page.test/');
+ const driver = await webInputDriver(page);
+ test.skip(!driver.wheels, WHEEL_SKIP);
+ const testInfo = {
+ annotations: [] as {type: string; description?: string}[],
+ };
+ const attempts: number[] = [];
+ const outcome = await retakeOnWheelPacing(testInfo, async n => {
+ attempts.push(n);
+ return driver.wheelBurst({
+ x: 200,
+ y: 300,
+ steps: Array.from({length: 4}, () => ({dx: 10})),
+ maxGapMs: 100,
+ });
+ }).then(
+ () => 'resolved',
+ (error: unknown) => error,
+ );
+ expect(outcome).not.toBeInstanceOf(WheelPacingError);
+ expect(attempts).toEqual([1]);
+ if (browserName === 'webkit') {
+ // The queued task marks the stopped step's handling, so the stall is
+ // the page's.
+ expect(outcome).toBeInstanceOf(WheelPageStallError);
+ }
+ } finally {
+ await context.close();
+ }
+ });
+
/** A non-passive wheel listener: Chromium then waits for the page itself. */
const listenNonPassively = (page: Page) =>
page.evaluate(() =>
diff --git a/internal/interaction-harness/src/touch.pacing.test.ts b/internal/interaction-harness/src/touch.pacing.test.ts
new file mode 100644
index 0000000000..f55afdab30
--- /dev/null
+++ b/internal/interaction-harness/src/touch.pacing.test.ts
@@ -0,0 +1,374 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file touch.pacing.test.ts
+ * @input touch.ts's pacing rule, its reading of a paced play's release and
+ * its pacing errors
+ * @output Unit cases: under `'schedule'` an event leaves at its scripted time
+ * from the start; under `'spacing'` its scripted interval after the event
+ * before it left, or after that dispatch returned when it outlasted the
+ * interval. The page's release (the last two move intervals, blended as
+ * core blends them) decides a paced play: an early stall leaves it
+ * intact; a short stall or a bunched pair at the end changes it, a fault
+ * only under a tolerance. A lift more than the stillness interval after
+ * the last move reads as rest, a fault unless the script rests too. A
+ * throw where the script rests is a fault, and so are fewer than two
+ * moves. A gap the page's own listeners held is the page's; a late send is
+ * the runner's.
+ * @position Node unit tests beside the harness's browser self-tests.
+ *
+ * SYNC: When modified, update touch.ts (pacedDeadline, touchPacingFault,
+ * TouchPacingError, TouchPageStallError, TOUCH_STILLNESS_MS,
+ * TOUCH_RELEASE_NEWEST_WEIGHT).
+ */
+
+import {describe, expect, it} from 'vitest';
+import type {TouchEventStep} from './scripts';
+import {
+ pacedDeadline,
+ readRelease,
+ touchArrivalsFrom,
+ TOUCH_STILLNESS_MS,
+ touchPacingFault,
+ TouchPacingError,
+ TouchPageStallError,
+ type TouchArrival,
+ type TouchPacing,
+} from './touch';
+
+/**
+ * A flick: the touch-down at 100 px, five moves of 16 px 16 ms apart, then
+ * the lift 16 ms later. Its release is 1,000 px/s.
+ */
+const FLICK: TouchEventStep[] = [0, 16, 32, 48, 64, 80, 96].map((t, i) => {
+ const finger = {id: 0, x: 100 + 16 * Math.min(i, 5), y: 50};
+ return i === 0
+ ? {t, type: 'touchStart', fingers: [finger], changed: [finger]}
+ : i === 6
+ ? {t, type: 'touchEnd', fingers: [], changed: [finger]}
+ : {t, type: 'touchMove', fingers: [finger], changed: [finger]};
+});
+
+/**
+ * The harness's loop on a simulated clock: each event leaves once its
+ * deadline has come (or at once when it is late), and its dispatch returns
+ * `latencyMs[i]` later, when the page has it and has handled it.
+ */
+function simulate(
+ pacing: TouchPacing,
+ latencyMs: readonly number[],
+): TouchArrival[] {
+ const origin = 1000;
+ let now = origin;
+ let previous: {sentAt: number; returnedAt: number} | null = null;
+ return FLICK.map((_, index) => {
+ now = Math.max(now, pacedDeadline(pacing, origin, FLICK, index, previous));
+ const sentAt = now;
+ now += latencyMs[index];
+ previous = {sentAt, returnedAt: now};
+ return {stamp: now, handledAt: now};
+ });
+}
+
+/** The release share a proof that depends on the exact release might allow. */
+const TOLERANCE = 0.2;
+
+/** Arrivals at the given stamps, each handled at once. */
+const at = (stamps: readonly number[]): TouchArrival[] =>
+ stamps.map(stamp => ({stamp, handledAt: stamp}));
+
+describe('pacedDeadline', () => {
+ it("sends each event at its scripted time from the start under 'schedule'", () => {
+ expect(pacedDeadline('schedule', 1000, FLICK, 0, null)).toBe(1000);
+ // The event before it, however late it was answered, does not move it.
+ expect(
+ pacedDeadline('schedule', 1000, FLICK, 3, {
+ sentAt: 1190,
+ returnedAt: 1300,
+ }),
+ ).toBe(1048);
+ });
+
+ it("sends the first event at the start and the rest after the one before, under 'spacing'", () => {
+ expect(pacedDeadline('spacing', 1000, FLICK, 0, null)).toBe(1000);
+ // Answered within its 16 ms interval: 16 ms after it left.
+ expect(
+ pacedDeadline('spacing', 1000, FLICK, 3, {
+ sentAt: 1200,
+ returnedAt: 1205,
+ }),
+ ).toBe(1216);
+ // Outlasted it: 16 ms after its answer.
+ expect(
+ pacedDeadline('spacing', 1000, FLICK, 3, {
+ sentAt: 1200,
+ returnedAt: 1300,
+ }),
+ ).toBe(1316);
+ });
+});
+
+describe("touchPacingFault on a paced play's arrivals", () => {
+ it("reads the script's release when every dispatch takes 5 ms", () => {
+ expect(
+ touchPacingFault(FLICK, simulate('spacing', [5, 5, 5, 5, 5, 5, 5])),
+ ).toBeNull();
+ });
+
+ it("reads a far faster release when 'schedule' bursts the events behind a stall", () => {
+ // The third dispatch stalls 100 ms; the four events behind it were due
+ // during the stall and reach the page 1 ms apart.
+ const fault = touchPacingFault(
+ FLICK,
+ simulate('schedule', [1, 1, 100, 1, 1, 1, 1]),
+ TOLERANCE,
+ );
+ expect(fault?.kind).toBe('release');
+ expect(fault?.pageReleasePxPerS).toBeGreaterThan(10_000);
+ expect(fault?.scriptReleasePxPerS).toBeCloseTo(1000, 6);
+ });
+
+ it("keeps the release intact behind an early stall under 'spacing'", () => {
+ // The stall stretches the second move's gap; the last two move intervals
+ // keep their 16 ms.
+ expect(
+ touchPacingFault(
+ FLICK,
+ simulate('spacing', [1, 1, 100, 1, 1, 1, 1]),
+ TOLERANCE,
+ ),
+ ).toBeNull();
+ });
+
+ it('reads 554 px/s for a 20 ms stall on the move before the last', () => {
+ // Its gap stretches to 35 ms and the next leaves 16 ms after its answer:
+ // 0.2 × 16 px/17 ms + 0.8 × 16 px/35 ms.
+ const fault = touchPacingFault(
+ FLICK,
+ simulate('spacing', [1, 1, 1, 1, 20, 1, 1]),
+ TOLERANCE,
+ );
+ expect(fault?.kind).toBe('release');
+ expect(fault?.pageReleasePxPerS).toBeCloseTo(554, 0);
+ expect(fault?.gapsMs).toEqual([35, 17, 16]);
+ expect(fault?.scriptedGapsMs).toEqual([16, 16, 16]);
+ expect(fault?.heldByPage).toBeNull();
+ });
+
+ it('reads a bunched pair of last moves as a faster release', () => {
+ // The fifth event arrives 28 ms late and the last move 2 ms after it:
+ // 0.2 × 16 px/2 ms + 0.8 × 16 px/44 ms.
+ const fault = touchPacingFault(
+ FLICK,
+ at([0, 16, 32, 48, 92, 94, 110]),
+ TOLERANCE,
+ );
+ expect(fault?.kind).toBe('release');
+ expect(fault?.pageReleasePxPerS).toBeCloseTo(1891, 0);
+ });
+
+ it('leaves a stray release to a proof that gives no tolerance, and still fails a rest', () => {
+ expect(
+ touchPacingFault(FLICK, simulate('spacing', [1, 1, 1, 1, 20, 1, 1])),
+ ).toBeNull();
+ expect(
+ touchPacingFault(
+ FLICK,
+ at([0, 16, 32, 48, 64, 80, 80 + TOUCH_STILLNESS_MS + 1]),
+ )?.kind,
+ ).toBe('rest');
+ });
+
+ it('reads a lift more than the stillness interval after the last move as rest, and one exactly that late as moving', () => {
+ const late = touchPacingFault(
+ FLICK,
+ at([0, 16, 32, 48, 64, 80, 80 + TOUCH_STILLNESS_MS + 1]),
+ );
+ expect(late?.kind).toBe('rest');
+ expect(late?.pageReleasePxPerS).toBe(0);
+ expect(
+ touchPacingFault(
+ FLICK,
+ at([0, 16, 32, 48, 64, 80, 80 + TOUCH_STILLNESS_MS]),
+ ),
+ ).toBeNull();
+ });
+
+ it('leaves a rest the script makes to the script', () => {
+ const resting = FLICK.map((step, i) =>
+ i === 6 ? {...step, t: 80 + TOUCH_STILLNESS_MS + 20} : step,
+ );
+ expect(
+ touchPacingFault(resting, at([0, 16, 32, 48, 64, 80, 400])),
+ ).toBeNull();
+ });
+
+ it("calls a gap the page's when its listeners spent longer than the interval on the event before, and that handling accounts for the gap", () => {
+ // The fourth event's listeners take 30 ms, so its successor reaches the
+ // page 46 ms after it, in the interval the release weighs most.
+ const arrivals = at([0, 16, 32, 48, 94, 110, 126]);
+ const held = arrivals.map((a, i) =>
+ i === 3 ? {...a, handledAt: a.stamp + 30} : a,
+ );
+ expect(touchPacingFault(FLICK, held, TOLERANCE)?.heldByPage).toEqual({
+ gapMs: 46,
+ handlingMs: 30,
+ });
+ // Handled within the interval, the same gap is the runner's.
+ const quick = arrivals.map((a, i) =>
+ i === 3 ? {...a, handledAt: a.stamp + 10} : a,
+ );
+ expect(touchPacingFault(FLICK, quick, TOLERANCE)?.heldByPage).toBeNull();
+ // A gap the handling does not account for is the runner's too.
+ const stalled = at([0, 16, 32, 48, 140, 156, 172]).map((a, i) =>
+ i === 3 ? {...a, handledAt: a.stamp + 30} : a,
+ );
+ expect(touchPacingFault(FLICK, stalled, TOLERANCE)?.heldByPage).toBeNull();
+ });
+});
+
+describe("the release a paced play's page reads, as Core's velocity tracker reads it", () => {
+ it('reads two moves that share a stamp as one, the later replacing the earlier, never Infinity', () => {
+ // The fourth move arrives 16 ms late, at the fifth's stamp. Core keeps
+ // the later sample: 16 px in 16 ms, then 32 px in 32 ms, 1,000 px/s.
+ const arrivals = at([0, 16, 32, 48, 80, 80, 96]);
+ expect(readRelease(FLICK, arrivals)?.pagePxPerS).toBeCloseTo(1000, 6);
+ expect(touchPacingFault(FLICK, arrivals, TOLERANCE)).toBeNull();
+ });
+
+ it('reads 0 after a single move, for the page and the script alike', () => {
+ const steps = [FLICK[0], FLICK[1], {...FLICK[6], t: 32}];
+ const reading = readRelease(steps, at([0, 16, 32]));
+ expect(reading?.pagePxPerS).toBe(0);
+ expect(reading?.scriptPxPerS).toBe(0);
+ });
+
+ it('reads no fault for a single move, however late the lift: one move releases at 0 either way', () => {
+ const steps = [FLICK[0], FLICK[1], {...FLICK[6], t: 32}];
+ expect(touchPacingFault(steps, at([0, 16, 80]))).toBeNull();
+ });
+
+ it('fails a page that read a throw where the script rests before the lift', () => {
+ // The flick lifts 50 ms after its last move, a rest. That move reaches
+ // the page 15 ms late, 35 ms before the lift. So the page reads a throw:
+ // 0.2 × 16 px in 31 ms + 0.8 × 16 px in 16 ms.
+ const steps = [...FLICK.slice(0, 6), {...FLICK[6], t: 130}];
+ const arrivals = at([0, 16, 32, 48, 64, 95, 130]);
+ expect(readRelease(steps, arrivals)?.pagePxPerS).toBeCloseTo(903.2, 1);
+ const fault = touchPacingFault(steps, arrivals);
+ expect(fault?.kind).toBe('throw');
+ expect(fault?.scriptReleasePxPerS).toBe(0);
+ });
+
+ it('fails a page that read every move at one stamp: with fewer than two moves it reads 0', () => {
+ const fault = touchPacingFault(FLICK, at([0, 80, 80, 80, 80, 80, 96]));
+ expect(fault?.kind).toBe('moves');
+ expect(fault?.pageReleasePxPerS).toBe(0);
+ expect(fault?.scriptReleasePxPerS).toBeCloseTo(1000, 6);
+ });
+});
+
+describe('touchArrivalsFrom', () => {
+ it('gives each step one arrival: the earliest stamp of its pointer and touch events, and the latest handling', () => {
+ expect(
+ touchArrivalsFrom({
+ eventTypes: ['pointerdown', 'touchstart', 'pointermove', 'touchmove'],
+ pointerTypes: ['touch', '', 'touch', ''],
+ stamps: [23, 24, 39, 100],
+ handledAt: [23, 24, 100, 100],
+ }),
+ ).toEqual([
+ {stamp: 23, handledAt: 24},
+ {stamp: 39, handledAt: 100},
+ ]);
+ });
+
+ it("leaves a mouse's pointer event out of a touch step", () => {
+ expect(
+ touchArrivalsFrom({
+ eventTypes: ['pointermove', 'pointermove', 'touchmove'],
+ pointerTypes: ['mouse', 'touch', ''],
+ stamps: [5, 39, 39],
+ handledAt: [90, 40, 40],
+ }),
+ ).toEqual([{stamp: 39, handledAt: 40}]);
+ });
+
+ it("reads a page whose pointer listener held each move as the page's rest, which the touch events alone hide", () => {
+ // This is how WebKit recorded a page whose pointermove listener spins
+ // 60 ms. The touch event is stamped once that listener is done. The next
+ // step arrives 16 ms after it.
+ const moves = [16, 92, 168, 244, 320];
+ const record = {
+ eventTypes: [
+ 'pointerdown',
+ 'touchstart',
+ ...moves.flatMap(() => ['pointermove', 'touchmove']),
+ 'pointerup',
+ 'touchend',
+ ],
+ pointerTypes: [
+ 'touch',
+ '',
+ ...moves.flatMap(() => ['touch', '']),
+ 'touch',
+ '',
+ ],
+ stamps: [0, 0, ...moves.flatMap(t => [t, t + 60]), 384, 384],
+ handledAt: [0, 0, ...moves.flatMap(t => [t + 60, t + 60]), 384, 384],
+ };
+ const fault = touchPacingFault(FLICK, touchArrivalsFrom(record));
+ expect(fault?.kind).toBe('rest');
+ // Each move's 60 ms of handling held the 76 ms gap after it.
+ expect(fault?.heldByPage).toEqual({gapMs: 76, handlingMs: 60});
+ // The touch events alone read the lift 4 ms after the last move.
+ const touchOnly = record.eventTypes
+ .map((type, i) => ({type, i}))
+ .filter(({type}) => type.startsWith('touch'))
+ .map(({i}) => ({
+ stamp: record.stamps[i],
+ handledAt: record.handledAt[i],
+ }));
+ expect(touchPacingFault(FLICK, touchOnly)).toBeNull();
+ });
+});
+
+describe('the pacing errors', () => {
+ it("name the release the page read, the script's and the gaps", () => {
+ const fault = touchPacingFault(
+ FLICK,
+ simulate('spacing', [1, 1, 1, 1, 20, 1, 1]),
+ TOLERANCE,
+ );
+ if (fault == null) {
+ throw new Error('expected a fault');
+ }
+ const error = new TouchPacingError(fault);
+ expect(error.name).toBe('TouchPacingError');
+ expect(error.message).toMatch(
+ /the page read a release of 554 px\/s where the script throws 1000 px\/s \(gaps 35, 17, 16 ms where the script spaces 16, 16, 16 ms\): a dispatch stalled before the page/,
+ );
+ expect(error.fault).toBe(fault);
+ });
+
+ it("name the page's handling when its listeners held the gap", () => {
+ const arrivals = at([0, 16, 32, 48, 94, 110, 126]).map((a, i) =>
+ i === 3 ? {...a, handledAt: a.stamp + 30} : a,
+ );
+ const fault = touchPacingFault(FLICK, arrivals, TOLERANCE);
+ if (fault?.heldByPage == null) {
+ throw new Error('expected a fault the page held');
+ }
+ const error = new TouchPageStallError({
+ ...fault,
+ heldByPage: fault.heldByPage,
+ });
+ expect(error.name).toBe('TouchPageStallError');
+ expect(error.message).toMatch(
+ /the page's own touch and pointer listeners took 30 ms to handle an event, so the next reached it 46 ms later/,
+ );
+ expect(error.message).toMatch(
+ /the page changed the release, not the runner/,
+ );
+ });
+});
diff --git a/internal/interaction-harness/src/touch.ts b/internal/interaction-harness/src/touch.ts
index 318d4731dc..8f40b90de0 100644
--- a/internal/interaction-harness/src/touch.ts
+++ b/internal/interaction-harness/src/touch.ts
@@ -25,7 +25,8 @@
* Limits, measured with WebKit 26.5: no timestamp parameter (the page sees
* the real dispatch time, so velocity comes from real-time pacing: within
* ~1 ms on an idle runner, but a runner whose CPU quota stalls the process
- * sends some steps late, and the ones behind them back to back);
+ * sends some steps late, and the ones behind them back to back, unless
+ * the play keeps the script's spacing: `pacing: 'spacing'`);
* integer coordinates only; no native scrolling, fling or pinch-zoom from
* these touches; a `touchstart` lists only the fingers that landed in it
* (pointer events are exact); no click synthesised from a raw touch sequence (use
@@ -35,10 +36,13 @@
* browser (the default), not a remote `connect()`.
*
* SYNC: Keep the engine limits above equal to the README's table and to the
- * harness self-test (../src/harness.interaction.browser.spec.ts).
+ * harness self-test (../src/harness.interaction.browser.spec.ts). Keep
+ * `coreRelease` equal to Core's filter
+ * (/packages/core/src/gestures/VelocityTracker.ts); Core's
+ * VelocityTracker.harness.test.ts compares the two.
*/
-import type {CDPSession, Page} from '@playwright/test';
+import type {CDPSession, JSHandle, Page} from '@playwright/test';
import {
toTouchEvents,
type Finger,
@@ -46,9 +50,38 @@ import {
type TouchEventStep,
type TouchScript,
} from './scripts';
+import {armInputRun, HELD_BACK_MS, type InputRun} from './webInput';
export type Engine = 'chromium' | 'webkit';
+/**
+ * How real-time pacing catches up after a slow dispatch (`PlayOptions.pacing`):
+ * `'schedule'` sends each event at its scripted time from the start, so the
+ * events behind a stalled command go out back to back; `'spacing'` sends each
+ * event its scripted interval after the one before it left, or, when that
+ * dispatch outlasted the interval (a stall), its interval after that dispatch
+ * returned, so a stall shifts the rest of the gesture instead of bunching it.
+ */
+export type TouchPacing = 'schedule' | 'spacing';
+
+/**
+ * The release filter's stillness interval, 40 ms: a contact that has not
+ * moved for longer than this before it lifts releases at 0.
+ *
+ * SYNC: Keep equal to `UIKIT_VELOCITY.stillness` in
+ * packages/core/src/motion/uikit/gestures.ts.
+ */
+export const TOUCH_STILLNESS_MS = 40;
+
+/**
+ * The newest move's share of the release velocity; the move before it
+ * carries the rest.
+ *
+ * SYNC: Keep equal to `UIKIT_VELOCITY.currentWeight` in
+ * packages/core/src/motion/uikit/gestures.ts.
+ */
+export const TOUCH_RELEASE_NEWEST_WEIGHT = 0.2;
+
export interface PlayOptions {
/**
* Pace dispatch on the wall clock so each event leaves at its scripted time
@@ -57,6 +90,29 @@ export interface PlayOptions {
* behaviors that run on the compositor clock (a fling) want real pacing.
*/
readonly realtime?: boolean;
+ /**
+ * How real-time pacing catches up after a slow dispatch (default
+ * `'schedule'`; see `TouchPacing`). A proof whose result depends on a
+ * release velocity (a flick, a coast) passes `'spacing'`: on a driver
+ * without scripted timestamps the page reads the velocity from when events
+ * arrive, and a burst after a stalled command reads as a much faster
+ * finger. There, a paced play reads the release as the page did once its
+ * last event is out (the lift's `PlayedEvent.release`). It throws when the
+ * page read a resting finger where the script moves, a throw where the
+ * script rests, or fewer than two moves. Given `releaseTolerance`, it also
+ * throws when the page's release strays further than that share. The error
+ * is `TouchPageStallError` when the page's own touch and pointer listeners
+ * held an event back, else `TouchPacingError` (see `touchPacingFault`).
+ */
+ readonly pacing?: TouchPacing;
+ /**
+ * How far the page's release may stray from the script's, as a share of
+ * it, before a paced play fails. WebKit delivers touches at its own
+ * cadence, so the release a page reads strays widely from a fast script's:
+ * a proof that depends on the exact release gives one; a proof that only
+ * needs a throw reads the page's release from the lift.
+ */
+ readonly releaseTolerance?: number;
/** Called after each event is dispatched, before the next one (storyboards). */
readonly onEvent?: (
step: TouchEventStep,
@@ -67,8 +123,21 @@ export interface PlayOptions {
export interface PlayedEvent extends TouchEventStep {
/** Wall-clock milliseconds (epoch) at which the event was dispatched. */
readonly dispatchedAt: number;
+ /**
+ * Wall-clock milliseconds (epoch) at which the dispatch command returned:
+ * the page has the event by then.
+ */
+ readonly returnedAt: number;
/** The timestamp handed to the engine, epoch ms (Chromium only). */
readonly timestamp: number | null;
+ /**
+ * On a paced play's lift, where the page reads arrival times: the release
+ * the page read and the script's, px/s.
+ */
+ readonly release?: {
+ readonly pagePxPerS: number;
+ readonly scriptPxPerS: number;
+ };
}
export interface TouchDriver {
@@ -111,26 +180,407 @@ async function until(deadline: number): Promise<void> {
}
}
+/**
+ * When event `index` of `steps` goes out (performance.now() ms): at its
+ * scripted time from `origin`; or, under `'spacing'`, its scripted interval
+ * after the event before it left, so the page reads the script's pace, unless
+ * that dispatch outlasted the interval. WebKit answers a dispatch only once
+ * the page has the event, so after such a stall the page may have received it
+ * as late as the answer, and the next event leaves its interval after that:
+ * never back to back.
+ */
+export function pacedDeadline(
+ pacing: TouchPacing,
+ origin: number,
+ steps: readonly {readonly t: number}[],
+ index: number,
+ previous: {readonly sentAt: number; readonly returnedAt: number} | null,
+): number {
+ if (pacing === 'spacing' && index > 0 && previous != null) {
+ const interval = steps[index].t - steps[index - 1].t;
+ return previous.returnedAt - previous.sentAt <= interval
+ ? previous.sentAt + interval
+ : previous.returnedAt + interval;
+ }
+ return origin + steps[index].t;
+}
+
+/** What the page saw of one touch event of a paced play. */
+export interface TouchArrival {
+ /** The event's own `timeStamp`: when it reached the page, in WebKit. */
+ readonly stamp: number;
+ /**
+ * When the page's own touch and pointer listeners had finished with it;
+ * NaN unseen.
+ */
+ readonly handledAt: number;
+}
+
+/** Where the release a paced play's page read strays from the script's. */
+export interface TouchPacingFault {
+ /**
+ * `'rest'`: the page read a resting finger; `'throw'`: a throw where the
+ * script rests; `'moves'`: fewer than two moves; `'release'`: another
+ * speed.
+ */
+ readonly kind: 'rest' | 'throw' | 'moves' | 'release';
+ /** The release the page read, px/s (0 for a rest or too few moves). */
+ readonly pageReleasePxPerS: number;
+ /** The script's release, px/s. */
+ readonly scriptReleasePxPerS: number;
+ /** The gaps the release reads, as the page received them, ms. */
+ readonly gapsMs: readonly number[];
+ /** The same gaps as the script spaces them, ms. */
+ readonly scriptedGapsMs: readonly number[];
+ /**
+ * The widest of those gaps when the page's own touch and pointer listeners held its
+ * event back (they were still handling the event before), or null.
+ */
+ readonly heldByPage: {
+ readonly gapMs: number;
+ readonly handlingMs: number;
+ } | null;
+}
+
+/** The lifting finger's release, read from the page's stamps and the script's. */
+export interface ReleaseReading {
+ readonly pagePxPerS: number;
+ readonly scriptPxPerS: number;
+ /** The page read a resting finger. */
+ readonly rests: boolean;
+ /**
+ * The script rests before the lift, but the page read a throw: its last
+ * move reached the page within the stillness interval of the lift.
+ */
+ readonly throws: boolean;
+ /**
+ * The page read fewer than two moves where the script makes more: its
+ * moves arrived at one time, so it reads no release.
+ */
+ readonly fewMoves: boolean;
+ /** The page's release strays from the script's by this share of it. */
+ readonly strayShare: number;
+ /** The events whose gaps the release reads (each gap ends at one). */
+ readonly span: readonly number[];
+}
+
+/** One sample of the lifting finger: its position and when it was read. */
+interface FingerSample {
+ readonly index: number;
+ readonly x: number;
+ readonly y: number;
+ readonly time: number;
+}
+
+/** What Core's velocity tracker reads from a finger's samples at its lift. */
+interface CoreReading {
+ readonly x: number;
+ readonly y: number;
+ /** Moved samples after the touch-down. */
+ readonly moves: number;
+ /** The finger was still for longer than the stillness interval. */
+ readonly rests: boolean;
+ /** The moved samples the release reads, oldest first (up to three). */
+ readonly moved: readonly FingerSample[];
+}
+
+/**
+ * The release Core's velocity tracker reads, px/s. It copies that filter:
+ * Core's `VelocityTracker` is internal to the core package, so the harness
+ * cannot call it. Keep the two equal (`add`, `releases`, `panVelocity`):
+ * Core's VelocityTracker.harness.test.ts reads the same samples with both.
+ * Only a sample that moves
+ * counts, and a sample at the newest moved sample's time replaces it. The
+ * newest interval weighs 0.2 and the one before it 0.8, and an interval
+ * across no time reads 0. The release is 0 with fewer than two moves after
+ * the touch-down, or after a rest longer than the stillness interval.
+ */
+function coreRelease(
+ samples: readonly FingerSample[],
+ liftTime: number,
+): CoreReading {
+ const moved: FingerSample[] = [];
+ let moves = 0;
+ let newest = Number.NEGATIVE_INFINITY;
+ for (const sample of samples) {
+ if (sample.time < newest) {
+ continue;
+ }
+ newest = sample.time;
+ const last = moved[moved.length - 1];
+ if (last != null && sample.x === last.x && sample.y === last.y) {
+ continue;
+ }
+ if (last != null && sample.time === last.time) {
+ moved[moved.length - 1] = sample;
+ } else {
+ moved.push(sample);
+ if (moved.length > 3) {
+ moved.shift();
+ }
+ if (last != null) {
+ moves += 1;
+ }
+ }
+ }
+ const last = moved[moved.length - 1];
+ const rests = last != null && liftTime - last.time > TOUCH_STILLNESS_MS;
+ if (moves < 2 || last == null || rests) {
+ return {x: 0, y: 0, moves, rests, moved};
+ }
+ const pair = (a: FingerSample, b: FingerSample) => {
+ const seconds = (b.time - a.time) / 1000;
+ return seconds > 0
+ ? {x: (b.x - a.x) / seconds, y: (b.y - a.y) / seconds}
+ : {x: 0, y: 0};
+ };
+ const n = moved.length;
+ const current = pair(moved[n - 2], moved[n - 1]);
+ const before = n === 3 ? pair(moved[0], moved[1]) : current;
+ const w = TOUCH_RELEASE_NEWEST_WEIGHT;
+ return {
+ x: w * current.x + (1 - w) * before.x,
+ y: w * current.y + (1 - w) * before.y,
+ moves,
+ rests,
+ moved,
+ };
+}
+
+/**
+ * Reads a paced play's release as Core's velocity tracker reads it, from the
+ * page's stamps and from the script's times. Both read the same positions,
+ * rounded once: WebKit delivers whole pixels. Each reading has its own rest,
+ * so a page that does not rest where the script does fails the play.
+ */
+export function readRelease(
+ steps: readonly TouchEventStep[],
+ arrivals: readonly TouchArrival[],
+): ReleaseReading | null {
+ const lift = steps.length - 1;
+ const id =
+ steps[lift]?.type === 'touchEnd' ? steps[lift].changed[0]?.id : undefined;
+ if (id == null || arrivals.length !== steps.length) {
+ return null;
+ }
+ const positions: {index: number; x: number; y: number}[] = [];
+ steps.forEach((step, index) => {
+ const finger =
+ step.type === 'touchStart' || step.type === 'touchMove'
+ ? step.changed.find(f => f.id === id)
+ : undefined;
+ if (finger != null) {
+ positions.push({index, x: Math.round(finger.x), y: Math.round(finger.y)});
+ }
+ });
+ if (positions.length === 0) {
+ return null;
+ }
+ const script = coreRelease(
+ positions.map(p => ({...p, time: steps[p.index].t})),
+ steps[lift].t,
+ );
+ const page = coreRelease(
+ positions.map(p => ({...p, time: arrivals[p.index].stamp})),
+ arrivals[lift].stamp,
+ );
+ const scriptPxPerS = Math.hypot(script.x, script.y);
+ const first = page.moved[0]?.index ?? positions[0].index;
+ return {
+ pagePxPerS: Math.hypot(page.x, page.y),
+ scriptPxPerS,
+ // One move releases at 0 for the script and the page alike, so a late
+ // lift after it changes nothing.
+ rests: script.moves >= 2 && !script.rests && page.rests,
+ throws: script.rests && !page.rests && page.moves >= 2,
+ fewMoves:
+ !script.rests && !page.rests && page.moves < 2 && script.moves >= 2,
+ strayShare:
+ scriptPxPerS > 0
+ ? Math.hypot(page.x - script.x, page.y - script.y) / scriptPxPerS
+ : 0,
+ span: Array.from({length: lift - first}, (_, k) => first + 1 + k),
+ };
+}
+
+/**
+ * Reads a paced play's release as the page did and returns its fault, or
+ * null. The kinds: `rest`, a resting finger where the script moves; `throw`,
+ * a throw where the script rests; `moves`, fewer than two moves. Given
+ * `tolerance`, `release` is a release that strays further than that share.
+ * The release blends the lifting finger's last two move intervals (its
+ * touch-down counts as the first event). It is 0 once the finger was still
+ * past the stillness interval before it lifts. So only those gaps decide it,
+ * and an earlier stall leaves it intact. Each reading keeps its own rest.
+ * WebKit dispatches a touch only once the page's main thread is free, so a
+ * gap is the page's when the page's listeners spent longer on the event
+ * before it than the script's interval to the next, and that handling
+ * accounts for the gap's excess over the script within a frame. Under
+ * `'spacing'` the harness itself waits out a late answer, so its send
+ * spacing cannot tell a page stall from a runner stall, as the wheel path's
+ * does (`wheelGapCauses`); the page's handling time can. The fault is the
+ * page's when any gap the release reads is.
+ */
+export function touchPacingFault(
+ steps: readonly TouchEventStep[],
+ arrivals: readonly TouchArrival[],
+ tolerance?: number,
+): TouchPacingFault | null {
+ const reading = readRelease(steps, arrivals);
+ if (
+ reading == null ||
+ !(
+ reading.rests ||
+ reading.throws ||
+ reading.fewMoves ||
+ (tolerance != null && reading.strayShare > tolerance)
+ )
+ ) {
+ return null;
+ }
+ const {span} = reading;
+ const gapsMs = span.map(i => arrivals[i].stamp - arrivals[i - 1].stamp);
+ const scriptedGapsMs = span.map(i => steps[i].t - steps[i - 1].t);
+ let heldByPage: TouchPacingFault['heldByPage'] = null;
+ span.forEach((i, k) => {
+ const handlingMs = arrivals[i - 1].handledAt - arrivals[i - 1].stamp;
+ const held =
+ Number.isFinite(handlingMs) &&
+ handlingMs > scriptedGapsMs[k] &&
+ handlingMs + HELD_BACK_MS >= gapsMs[k] - scriptedGapsMs[k];
+ if (held && (heldByPage == null || handlingMs > heldByPage.handlingMs)) {
+ heldByPage = {gapMs: gapsMs[k], handlingMs};
+ }
+ });
+ return {
+ kind: reading.rests
+ ? 'rest'
+ : reading.throws
+ ? 'throw'
+ : reading.fewMoves
+ ? 'moves'
+ : 'release',
+ pageReleasePxPerS: reading.pagePxPerS,
+ scriptReleasePxPerS: reading.scriptPxPerS,
+ gapsMs,
+ scriptedGapsMs,
+ heldByPage,
+ };
+}
+
+function describeFault(fault: TouchPacingFault): string {
+ const gaps = `gaps ${fault.gapsMs.map(g => g.toFixed(0)).join(', ')} ms where the script spaces ${fault.scriptedGapsMs.map(g => g.toFixed(0)).join(', ')} ms`;
+ if (fault.kind === 'rest') {
+ return `the page read a resting finger: the lift reached it more than ${TOUCH_STILLNESS_MS} ms after the last move (${gaps})`;
+ }
+ if (fault.kind === 'throw') {
+ return `the page read a throw of ${fault.pageReleasePxPerS.toFixed(0)} px/s where the script rests more than ${TOUCH_STILLNESS_MS} ms before the lift (${gaps})`;
+ }
+ if (fault.kind === 'moves') {
+ return `the page read fewer than two moves, so no release, where the script throws ${fault.scriptReleasePxPerS.toFixed(0)} px/s: its moves reached it at one time (${gaps})`;
+ }
+ return `the page read a release of ${fault.pageReleasePxPerS.toFixed(0)} px/s where the script throws ${fault.scriptReleasePxPerS.toFixed(0)} px/s (${gaps})`;
+}
+
+/**
+ * A paced play's page read a release that is not the script's, and its own
+ * touch and pointer listeners did not hold the events back: a dispatch stalled before
+ * the page. The runner failed, not the component under test.
+ */
+export class TouchPacingError extends Error {
+ constructor(readonly fault: TouchPacingFault) {
+ super(
+ `touchDriver: ${describeFault(fault)}: a dispatch stalled before the page`,
+ );
+ this.name = 'TouchPacingError';
+ }
+}
+
+/**
+ * A paced play's page read a release that is not the script's because its
+ * own touch and pointer listeners held an event back: they were still handling the
+ * event before, and WebKit dispatches the next touch only once the page is
+ * free. The page changed the release, so this is the proof's own failure.
+ */
+export class TouchPageStallError extends Error {
+ constructor(
+ readonly fault: TouchPacingFault & {
+ readonly heldByPage: NonNullable<TouchPacingFault['heldByPage']>;
+ },
+ ) {
+ super(
+ `touchDriver: the page's own touch and pointer listeners took ${fault.heldByPage.handlingMs.toFixed(0)} ms to handle an event, so the next reached it ${fault.heldByPage.gapMs.toFixed(0)} ms later, and ${describeFault(fault)}: the page changed the release, not the runner`,
+ );
+ this.name = 'TouchPageStallError';
+ }
+}
+
+/** Records what the page saw of a paced play's touches (WebKit). */
+interface TouchArrivalProbe {
+ arm(): Promise<void>;
+ read(): Promise<TouchArrival[]>;
+}
+
async function playSteps(
script: TouchScript,
options: PlayOptions,
dispatch: (step: TouchEventStep, timestamp: number) => Promise<void>,
stampFor: (t: number) => number,
+ probe: TouchArrivalProbe | null,
): Promise<PlayedEvent[]> {
- const {realtime = true, onEvent} = options;
+ const {
+ realtime = true,
+ onEvent,
+ pacing = 'schedule',
+ releaseTolerance,
+ } = options;
const steps = toTouchEvents(script);
+ // Only where the page reads a velocity from arrival times: scripted
+ // timestamps carry it.
+ const judges = probe != null && realtime && pacing === 'spacing';
+ if (judges) {
+ await probe.arm();
+ }
const played: PlayedEvent[] = [];
const origin = performance.now();
+ let previous: {sentAt: number; returnedAt: number} | null = null;
for (const [index, step] of steps.entries()) {
if (realtime) {
- await until(origin + step.t);
+ await until(pacedDeadline(pacing, origin, steps, index, previous));
}
const timestamp = stampFor(step.t);
const dispatchedAt = Date.now();
+ const sentAt = performance.now();
await dispatch(step, timestamp);
- played.push({...step, dispatchedAt, timestamp});
+ previous = {sentAt, returnedAt: performance.now()};
+ played.push({...step, dispatchedAt, returnedAt: Date.now(), timestamp});
await onEvent?.(step, index);
}
+ // Judged once the lift is out, so no finger is left down.
+ if (judges) {
+ const arrivals = await probe.read();
+ if (arrivals.length !== steps.length) {
+ throw new Error(
+ `touchDriver: the page received ${arrivals.length} of a paced play's ${steps.length} touch events`,
+ );
+ }
+ const fault = touchPacingFault(steps, arrivals, releaseTolerance);
+ if (fault != null) {
+ throw fault.heldByPage == null
+ ? new TouchPacingError(fault)
+ : new TouchPageStallError({...fault, heldByPage: fault.heldByPage});
+ }
+ const reading = readRelease(steps, arrivals);
+ if (reading != null) {
+ played[played.length - 1] = {
+ ...played[played.length - 1],
+ release: {
+ pagePxPerS: reading.pagePxPerS,
+ scriptPxPerS: reading.scriptPxPerS,
+ },
+ };
+ }
+ }
return played;
}
@@ -174,6 +624,7 @@ async function chromiumDriver(page: Page): Promise<TouchDriver> {
});
},
t => base + t,
+ null,
);
},
async tap(at) {
@@ -207,6 +658,89 @@ export function webkitPageSession(page: Page): WebKitPageProxySession {
return session;
}
+/**
+ * The events a paced play's probe records: a gesture host samples a finger
+ * from its pointer events, and a page can handle either kind.
+ */
+const TOUCH_TYPES = [
+ 'pointerdown',
+ 'pointermove',
+ 'pointerup',
+ 'pointercancel',
+ 'touchstart',
+ 'touchmove',
+ 'touchend',
+ 'touchcancel',
+];
+
+/**
+ * One arrival for each touch event a probe recorded, in dispatch order. WebKit
+ * dispatches each step's pointer events just before its touch event, and
+ * stamps the touch event only once the page's pointer listeners are done.
+ * So an arrival takes the earliest stamp of its step's events, when a
+ * gesture host samples the finger, and the latest handling mark of them.
+ */
+export function touchArrivalsFrom(record: {
+ readonly eventTypes: readonly string[];
+ readonly pointerTypes: readonly string[];
+ readonly stamps: readonly number[];
+ readonly handledAt: readonly number[];
+}): TouchArrival[] {
+ const arrivals: TouchArrival[] = [];
+ let stamp = Number.POSITIVE_INFINITY;
+ let handledAt = Number.NaN;
+ record.eventTypes.forEach((type, i) => {
+ // A mouse's or a pen's pointer event is not this play's.
+ if (!type.startsWith('touch') && record.pointerTypes[i] !== 'touch') {
+ return;
+ }
+ stamp = Math.min(stamp, record.stamps[i]);
+ const mark = record.handledAt[i];
+ if (Number.isFinite(mark)) {
+ handledAt = Number.isNaN(handledAt) ? mark : Math.max(handledAt, mark);
+ }
+ if (type.startsWith('touch')) {
+ arrivals.push({stamp, handledAt});
+ stamp = Number.POSITIVE_INFINITY;
+ handledAt = Number.NaN;
+ }
+ });
+ return arrivals;
+}
+
+/**
+ * Records, for each touch event of a paced play, when it reached the page
+ * and when the page's own listeners had finished with it (`armInputRun`).
+ */
+function touchArrivalProbe(page: Page): TouchArrivalProbe {
+ let armed: JSHandle<InputRun> | null = null;
+ return {
+ async arm() {
+ await armed?.evaluate(run => run.stop());
+ await armed?.dispose();
+ armed = await armInputRun(page, TOUCH_TYPES);
+ },
+ async read() {
+ if (armed == null) {
+ return [];
+ }
+ const run = armed;
+ armed = null;
+ const record = await run.evaluate(recorded => {
+ recorded.stop();
+ return {
+ eventTypes: recorded.eventTypes.slice(),
+ pointerTypes: recorded.pointerTypes.slice(),
+ stamps: recorded.stamps.slice(),
+ handledAt: recorded.handledAt.slice(),
+ };
+ });
+ await run.dispose();
+ return touchArrivalsFrom(record);
+ },
+ };
+}
+
async function webkitDriver(page: Page): Promise<TouchDriver> {
const session = webkitPageSession(page);
// WebKit's command takes integer CSS px and applies the event's phase to
@@ -233,6 +767,7 @@ async function webkitDriver(page: Page): Promise<TouchDriver> {
});
},
() => Date.now(),
+ touchArrivalProbe(page),
).then(played => played.map(p => ({...p, timestamp: null})));
},
async tap(at) {
diff --git a/internal/interaction-harness/src/webInput.ts b/internal/interaction-harness/src/webInput.ts
index 369070c9e5..9f32f478de 100644
--- a/internal/interaction-harness/src/webInput.ts
+++ b/internal/interaction-harness/src/webInput.ts
@@ -126,11 +126,11 @@ export interface WheelPageStall {
}
/**
- * How soon after the page's wheel listeners finish one step the next must
+ * How soon after the page's input listeners finish one event the next must
* reach the page for the page to have held it back: one frame, with a few ms
- * for the dispatch itself.
+ * for the dispatch itself. The wheel and the touch paths both read it.
*/
-const HELD_BACK_MS = 20;
+export const HELD_BACK_MS = 20;
/**
* When each received step of a burst reached the page, ms on the page clock:
@@ -198,7 +198,8 @@ export function wheelSendGaps(
/**
* Sorts a burst's gaps past `maxGapMs` by cause. `stamps` are when the
* received steps reached the page (`wheelArrivals`); `handledAt` is when the
- * page's wheel listeners had finished with each (NaN where that was not seen);
+ * page's wheel listeners had finished with each (for a step whose propagation
+ * the page stopped, when a task queued at its arrival ran; NaN unseen);
* `sendGapsMs` is how far apart the harness sent the steps behind each gap
* (`wheelSendGaps`; `null` where unknown). In WebKit, which dispatches a step
* only once the page's main thread is free, a gap is the page's when the step
@@ -210,7 +211,8 @@ export function wheelSendGaps(
* page right after its listeners). Every other gap is the dispatch's, and in
* Chromium, which stamps a step with its dispatch time, so the page cannot
* move its stamps, every gap is. Other work on the page's main thread between
- * steps (timers, frame callbacks) counts as the dispatch's.
+ * steps (timers, frame callbacks) counts as the dispatch's, unless it ran
+ * before a stopped step's marking task.
*/
export function wheelGapCauses(
engine: Engine,
@@ -371,58 +373,115 @@ async function untilReceived(
}
}
-interface WheelRun {
+/** What the page saw of each input event of one run (`armInputRun`). */
+export interface InputRun {
+ /** The event types the run records. */
+ types: string[];
+ /** Each event's own `timeStamp`. */
stamps: number[];
- /** When the page's first listener received each step. */
+ /** When the page's first listener received each event. */
receivedAt: number[];
- /** When the page's wheel listeners had finished with each step; NaN until seen. */
+ /**
+ * When the page's own listeners had finished with each event, NaN until
+ * seen. An event whose propagation the page stops is marked by a task
+ * queued when it arrived, and never later than the next event's arrival.
+ * So page work queued before that task, such as a timer, counts as its
+ * handling.
+ */
handledAt: number[];
- /** Each received step's deltas, CSS px: a coalesced step carries the sum. */
+ /** Each event's type. */
+ eventTypes: string[];
+ /** Each event's pointer type, or '' for an event that has none. */
+ pointerTypes: string[];
+ /** Each wheel step's deltas, CSS px: a coalesced step carries the sum. */
deltas: [number, number][];
dx: number;
dy: number;
+ /** Removes the listeners and settles every handling mark. */
stop: () => void;
}
/**
- * Arms a passive capture listener that stamps every wheel step the page
- * receives and sums its deltas, before a burst is dispatched, and a passive
- * bubble listener on the window, after the page's own, that notes when the
- * page has finished with each step (a step whose propagation the page stops
- * is never noted, and counts as handled at once).
+ * Arms two passive listeners on the window before a run is dispatched. A
+ * capture listener stamps each trusted event of `types` before the page's
+ * own listeners. A bubble listener, after them, notes when the page has
+ * finished with the event. The wheel and touch drivers share it.
*/
-function armWheelRun(page: Page): Promise<JSHandle<WheelRun>> {
- return page.evaluateHandle(() => {
- const first = {capture: true, passive: true};
- const last = {capture: false, passive: true};
- const run = {
- stamps: [] as number[],
- receivedAt: [] as number[],
- handledAt: [] as number[],
- deltas: [] as [number, number][],
- dx: 0,
- dy: 0,
- stop: () => {
- window.removeEventListener('wheel', received, first);
- window.removeEventListener('wheel', handled, last);
- },
- };
- const received = (event: Event) => {
- const wheel = event as WheelEvent;
- run.stamps.push(wheel.timeStamp);
- run.receivedAt.push(performance.now());
- run.handledAt.push(Number.NaN);
- run.deltas.push([wheel.deltaX, wheel.deltaY]);
- run.dx += wheel.deltaX;
- run.dy += wheel.deltaY;
- };
- const handled = () => {
- run.handledAt[run.handledAt.length - 1] = performance.now();
- };
- window.addEventListener('wheel', received, first);
- window.addEventListener('wheel', handled, last);
- return run;
- });
+export function armInputRun(
+ page: Page,
+ types: readonly string[],
+): Promise<JSHandle<InputRun>> {
+ return page.evaluateHandle(
+ eventTypes => {
+ const first = {capture: true, passive: true};
+ const last = {capture: false, passive: true};
+ const index = new WeakMap<Event, number>();
+ // A message runs as a task of its own, after the dispatch that queued
+ // it: it marks an event whose bubble listener never ran.
+ const channel = new MessageChannel();
+ const run = {
+ types: [...eventTypes],
+ stamps: [] as number[],
+ receivedAt: [] as number[],
+ handledAt: [] as number[],
+ eventTypes: [] as string[],
+ pointerTypes: [] as string[],
+ deltas: [] as [number, number][],
+ dx: 0,
+ dy: 0,
+ stop: () => {
+ for (const type of eventTypes) {
+ window.removeEventListener(type, received, first);
+ window.removeEventListener(type, handled, last);
+ }
+ channel.port1.onmessage = null;
+ channel.port1.close();
+ for (let i = 0; i + 1 < run.handledAt.length; i++) {
+ run.handledAt[i] = Math.min(
+ run.handledAt[i],
+ run.receivedAt[i + 1],
+ );
+ }
+ },
+ };
+ channel.port1.onmessage = (message: MessageEvent<number>) => {
+ if (Number.isNaN(run.handledAt[message.data])) {
+ run.handledAt[message.data] = performance.now();
+ }
+ };
+ const received = (event: Event) => {
+ if (!event.isTrusted) {
+ return;
+ }
+ const i = run.stamps.length;
+ index.set(event, i);
+ run.stamps.push(event.timeStamp);
+ run.receivedAt.push(performance.now());
+ run.handledAt.push(Number.NaN);
+ run.eventTypes.push(event.type);
+ run.pointerTypes.push(
+ 'pointerType' in event ? String(event.pointerType) : '',
+ );
+ const wheel = event.type === 'wheel' ? (event as WheelEvent) : null;
+ run.deltas.push([wheel?.deltaX ?? 0, wheel?.deltaY ?? 0]);
+ run.dx += wheel?.deltaX ?? 0;
+ run.dy += wheel?.deltaY ?? 0;
+ channel.port2.postMessage(i);
+ };
+ const handled = (event: Event) => {
+ const i = index.get(event);
+ if (i != null) {
+ run.handledAt[i] = performance.now();
+ }
+ };
+ for (const type of eventTypes) {
+ window.addEventListener(type, received, first);
+ window.addEventListener(type, handled, last);
+ }
+ return run;
+ },
+ [...types],
+ );
}
/**
@@ -432,7 +491,7 @@ function armWheelRun(page: Page): Promise<JSHandle<WheelRun>> {
async function untilRunReceived(
page: Page,
engine: Engine,
- armed: JSHandle<WheelRun>,
+ armed: JSHandle<InputRun>,
{steps, maxGapMs}: Pick<WheelBurst, 'steps' | 'maxGapMs'>,
sentAt: readonly number[],
): Promise<{receivedSteps: number; receivedGapsMs: number[]}> {
@@ -565,7 +624,7 @@ async function chromiumWebInput(page: Page): Promise<WebInputDriver> {
const scale = await page.evaluate(
() => window.devicePixelRatio * (window.visualViewport?.scale ?? 1),
);
- const armed = await armWheelRun(page);
+ const armed = await armInputRun(page, ['wheel']);
const sentAt = await dispatchOnSchedule(
steps,
everyMs,
@@ -675,7 +734,7 @@ async function webkitWebInput(page: Page): Promise<WebInputDriver> {
await page.keyboard.down('Control');
}
try {
- const armed = await armWheelRun(page);
+ const armed = await armInputRun(page, ['wheel']);
const sentAt = await dispatchOnSchedule(
steps,
everyMs,
diff --git a/packages/core/src/gestures/__tests__/VelocityTracker.harness.test.ts b/packages/core/src/gestures/__tests__/VelocityTracker.harness.test.ts
new file mode 100644
index 0000000000..fb959cb6f7
--- /dev/null
+++ b/packages/core/src/gestures/__tests__/VelocityTracker.harness.test.ts
@@ -0,0 +1,134 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file VelocityTracker.harness.test.ts
+ * @input VelocityTracker and the interaction harness's `readRelease`
+ * @output Checks that the harness reads a paced play's release as this
+ * tracker does, on the same samples
+ * @position Testing. The harness copies this tracker's filter because it
+ * cannot import Core's gestures. This test keeps the two equal.
+ *
+ * SYNC: When modified, update
+ * /internal/interaction-harness/src/touch.ts (`coreRelease`).
+ */
+
+import {
+ readRelease,
+ type TouchEventStep,
+} from '@astryxdesign/interaction-harness';
+import {describe, expect, it} from 'vitest';
+import {VelocityTracker} from '../VelocityTracker';
+
+/** A finger's samples as [x, y, ms], in order, and its lift time. */
+interface Gesture {
+ readonly samples: ReadonlyArray<readonly [number, number, number]>;
+ readonly lift: number;
+}
+
+const GESTURES: Record<string, Gesture> = {
+ 'a steady flick': {
+ samples: [
+ [100, 50, 0],
+ [116, 50, 16],
+ [132, 50, 32],
+ [148, 50, 48],
+ [164, 50, 64],
+ ],
+ lift: 80,
+ },
+ 'an accelerating throw': {
+ samples: [
+ [100, 50, 0],
+ [104, 50, 16],
+ [112, 50, 32],
+ [128, 50, 48],
+ [160, 50, 64],
+ ],
+ lift: 72,
+ },
+ 'two moves at one time': {
+ samples: [
+ [100, 50, 0],
+ [116, 50, 16],
+ [132, 50, 32],
+ [140, 50, 32],
+ [156, 50, 48],
+ ],
+ lift: 56,
+ },
+ 'a sample that does not move': {
+ samples: [
+ [100, 50, 0],
+ [116, 50, 16],
+ [116, 50, 32],
+ [132, 50, 48],
+ ],
+ lift: 56,
+ },
+ 'a diagonal drag': {
+ samples: [
+ [100, 300, 0],
+ [110, 290, 17],
+ [125, 270, 33],
+ [140, 255, 50],
+ ],
+ lift: 66,
+ },
+ 'a single move': {
+ samples: [
+ [100, 50, 0],
+ [116, 50, 16],
+ ],
+ lift: 32,
+ },
+ 'a rest before the lift': {
+ samples: [
+ [100, 50, 0],
+ [116, 50, 16],
+ [132, 50, 32],
+ ],
+ lift: 120,
+ },
+};
+
+/** The gesture as a paced play's touch events, one finger. */
+function stepsOf({samples, lift}: Gesture): TouchEventStep[] {
+ const steps: TouchEventStep[] = samples.map(([x, y, t], i) => {
+ const finger = {id: 0, x, y};
+ return {
+ t,
+ type: i === 0 ? 'touchStart' : 'touchMove',
+ fingers: [finger],
+ changed: [finger],
+ };
+ });
+ const [x, y] = samples[samples.length - 1];
+ steps.push({
+ t: lift,
+ type: 'touchEnd',
+ fingers: [],
+ changed: [{id: 0, x, y}],
+ });
+ return steps;
+}
+
+describe("the interaction harness's copy of the release filter", () => {
+ for (const [name, gesture] of Object.entries(GESTURES)) {
+ it(`reads ${name} as VelocityTracker does`, () => {
+ const tracker = new VelocityTracker();
+ for (const [x, y, t] of gesture.samples) {
+ tracker.add(x, y, t);
+ }
+ const velocity = tracker.velocity(gesture.lift);
+ const core = Math.hypot(velocity.x, velocity.y);
+ // The page received each event at its scripted time.
+ const steps = stepsOf(gesture);
+ const reading = readRelease(
+ steps,
+ steps.map(step => ({stamp: step.t, handledAt: step.t})),
+ );
+ expect(reading?.scriptPxPerS).toBeCloseTo(core, 6);
+ expect(reading?.pagePxPerS).toBeCloseTo(core, 6);
+ });
+ }
+});