Skip to content

Write HTML. Render video. Built for agents.

fix(studio): the timeline viewport gate judges p95 over every measured scroll step (#4847)

Miguel Ángelc20934a7205b
diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts b/packages/studio/src/player/lib/timelineViewportBudgets.test.ts
index 5689b5609a..a75d630ebe 100644
--- a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts
+++ b/packages/studio/src/player/lib/timelineViewportBudgets.test.ts
@@ -23,7 +23,7 @@ describe("timeline viewport budgets", () => {
constrainedLongTaskLimitMs: 300,
posterCoverageRatio: 0.9,
supportedFixtureFallbackRatio: 0.02,
- scrollSamplesPerRun: 21,
+ scrollSamplesPerRun: 63,
warmupRuns: 3,
measuredRuns: 5,
requiredPassingRuns: 4,
diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.ts b/packages/studio/src/player/lib/timelineViewportBudgets.ts
index b57bf325ec..771a37cd32 100644
--- a/packages/studio/src/player/lib/timelineViewportBudgets.ts
+++ b/packages/studio/src/player/lib/timelineViewportBudgets.ts
@@ -99,7 +99,7 @@ export const TIMELINE_VIEWPORT_BUDGETS: Readonly<TimelineViewportBudgets> = Obje
richPreviewP95Ms: 750,
constrainedRichPreviewP95Ms: 1_200,
supportedFixtureFallbackRatio: 0.02,
- scrollSamplesPerRun: 21,
+ scrollSamplesPerRun: 63,
warmupRuns: 3,
measuredRuns: 5,
requiredPassingRuns: 4,
diff --git a/packages/studio/tests/e2e/chrome-executable.mjs b/packages/studio/tests/e2e/chrome-executable.mjs
index b1a868916e..8fc89f490e 100644
--- a/packages/studio/tests/e2e/chrome-executable.mjs
+++ b/packages/studio/tests/e2e/chrome-executable.mjs
@@ -1,6 +1,7 @@
import { existsSync, readdirSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
+import puppeteer from "puppeteer-core";
/** Resolve the same Chrome binary for every local Studio browser acceptance test. */
export function resolveChromeExecutable() {
@@ -22,3 +23,18 @@ export function resolveChromeExecutable() {
...installed,
].find((candidate) => candidate && existsSync(candidate));
}
+
+/** Headless Chrome for a Studio browser gate; exits 2 when no Chrome is installed. */
+export async function launchStudioChrome() {
+ const executablePath = resolveChromeExecutable();
+ if (!executablePath) {
+ console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH");
+ process.exit(2);
+ }
+ const browser = await puppeteer.launch({
+ executablePath,
+ headless: true,
+ args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"],
+ });
+ return { browser, executablePath };
+}
diff --git a/packages/studio/tests/e2e/fade-handles.mjs b/packages/studio/tests/e2e/fade-handles.mjs
index 18f5cb3c16..a4e4a9a5c2 100644
--- a/packages/studio/tests/e2e/fade-handles.mjs
+++ b/packages/studio/tests/e2e/fade-handles.mjs
@@ -1,8 +1,7 @@
#!/usr/bin/env node
// On a clip narrower than two fade hit boxes, each dot must still lay out at 10 x 10.
// Flex shrinking only happens in real layout, so this measures it in Chrome.
-import puppeteer from "puppeteer-core";
-import { resolveChromeExecutable } from "./chrome-executable.mjs";
+import { launchStudioChrome } from "./chrome-executable.mjs";
const STUDIO_URL = process.env.STUDIO_URL;
const DOT_PX = 10;
@@ -12,17 +11,7 @@ if (!STUDIO_URL) {
console.error("STUDIO_URL is required and must point at the fade-handles fixture");
process.exit(2);
}
-const executablePath = resolveChromeExecutable();
-if (!executablePath) {
- console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH");
- process.exit(2);
-}
-
-const browser = await puppeteer.launch({
- executablePath,
- headless: true,
- args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"],
-});
+const { browser } = await launchStudioChrome();
const failures = [];
let evidence = {};
try {
diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs
new file mode 100644
index 0000000000..d9f0f3f639
--- /dev/null
+++ b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs
@@ -0,0 +1,54 @@
+/**
+ * The timeline viewport gate's verdict, kept apart from the script that drives Chrome so it can be tested.
+ */
+
+/** Nearest-rank percentile: `ratio` 0.95 of 315 values is the 16th-worst. */
+export function percentile(values, ratio) {
+ if (values.length === 0) return 0;
+ const sorted = [...values].sort((a, b) => a - b);
+ return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)];
+}
+
+function assertSampleCount(expected, interactions, frameIntervals) {
+ if (expected > 0 && interactions.length === expected && frameIntervals.length === expected)
+ return;
+ throw new Error(
+ `Expected ${expected} scroll samples, measured ${interactions.length} interactions ` +
+ `and ${frameIntervals.length} frame intervals`,
+ );
+}
+
+/**
+ * p95 over every measured step pooled: one run's p95 is only its 4th-worst step, so a brief runner stall failed it.
+ * Throws when any run is short of samples, so a missing measurement cannot read as a fast one.
+ */
+export function judgeResponsiveness(
+ runs,
+ { samplesPerRun, interactionLimitMs, frameIntervalLimitMs },
+) {
+ const interactions = runs.flatMap((run) => run.interactions);
+ const frameIntervals = runs.flatMap((run) => run.frameIntervals);
+ assertSampleCount(runs.length * samplesPerRun, interactions, frameIntervals);
+ const interactionP95Ms = percentile(interactions, 0.95);
+ const frameIntervalP95Ms = percentile(frameIntervals, 0.95);
+ return {
+ interactionP95Ms,
+ frameIntervalP95Ms,
+ passed: interactionP95Ms <= interactionLimitMs && frameIntervalP95Ms <= frameIntervalLimitMs,
+ };
+}
+
+export function gatePassed({
+ directScrollApproved,
+ responsivenessPassed,
+ passingRuns,
+ requiredPassingRuns,
+ memoryReturned,
+}) {
+ return (
+ directScrollApproved &&
+ responsivenessPassed &&
+ passingRuns >= requiredPassingRuns &&
+ memoryReturned
+ );
+}
diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs
new file mode 100644
index 0000000000..c5a54af4b9
--- /dev/null
+++ b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs
@@ -0,0 +1,88 @@
+import { describe, expect, it } from "vitest";
+import { gatePassed, judgeResponsiveness, percentile } from "./timeline-viewport-verdict.mjs";
+
+const LIMITS = { samplesPerRun: 63, interactionLimitMs: 75, frameIntervalLimitMs: 75 };
+const FAST = 49;
+const SLOW = 83;
+
+/** Five runs of 63 steps; `slowAt(run, step)` marks the steps that take five frames. */
+function runs(slowAt) {
+ return Array.from({ length: 5 }, (_, run) => {
+ const interactions = Array.from({ length: 63 }, (_, step) => (slowAt(run, step) ? SLOW : FAST));
+ return { interactions, frameIntervals: interactions.map(() => 33.3) };
+ });
+}
+
+const slowSteps = (count) => (run, step) => run * 63 + step < count;
+
+describe("percentile", () => {
+ it("takes the nearest rank, so the p95 of 315 steps is the 16th-worst", () => {
+ const values = Array.from({ length: 315 }, (_, index) => index);
+ expect(percentile(values, 0.95)).toBe(299);
+ expect(percentile([3, 1, 2], 0.95)).toBe(3);
+ });
+});
+
+describe("judgeResponsiveness", () => {
+ it("fails 16 slow steps of 315 and passes 15", () => {
+ expect(judgeResponsiveness(runs(slowSteps(16)), LIMITS)).toMatchObject({
+ interactionP95Ms: SLOW,
+ passed: false,
+ });
+ expect(judgeResponsiveness(runs(slowSteps(15)), LIMITS)).toMatchObject({
+ interactionP95Ms: FAST,
+ passed: true,
+ });
+ });
+
+ it("passes two slow steps in every run, 10 of 315", () => {
+ expect(
+ judgeResponsiveness(
+ runs((_, step) => step < 2),
+ LIMITS,
+ ).passed,
+ ).toBe(true);
+ });
+
+ it("fails one run that is slow throughout", () => {
+ expect(
+ judgeResponsiveness(
+ runs((run) => run === 2),
+ LIMITS,
+ ).passed,
+ ).toBe(false);
+ });
+
+ it("fails on the frame interval alone", () => {
+ const measured = runs(() => false).map((run) => ({
+ ...run,
+ frameIntervals: run.frameIntervals.map(() => 83),
+ }));
+ expect(judgeResponsiveness(measured, LIMITS).passed).toBe(false);
+ });
+
+ it("throws on a run short of samples instead of reading it as fast", () => {
+ const measured = runs(() => false);
+ measured[1] = { interactions: [], frameIntervals: [] };
+ expect(() => judgeResponsiveness(measured, LIMITS)).toThrow("Expected 315 scroll samples");
+ expect(() => judgeResponsiveness([], LIMITS)).toThrow("Expected 0 scroll samples");
+ });
+});
+
+describe("gatePassed", () => {
+ const passing = {
+ directScrollApproved: true,
+ responsivenessPassed: true,
+ passingRuns: 5,
+ requiredPassingRuns: 4,
+ memoryReturned: true,
+ };
+
+ it("passes only when every check holds", () => {
+ expect(gatePassed(passing)).toBe(true);
+ expect(gatePassed({ ...passing, responsivenessPassed: false })).toBe(false);
+ expect(gatePassed({ ...passing, directScrollApproved: false })).toBe(false);
+ expect(gatePassed({ ...passing, passingRuns: 3 })).toBe(false);
+ expect(gatePassed({ ...passing, memoryReturned: false })).toBe(false);
+ });
+});
diff --git a/packages/studio/tests/e2e/timeline-virtualization.mjs b/packages/studio/tests/e2e/timeline-virtualization.mjs
index 521da8ffee..ff6435755f 100644
--- a/packages/studio/tests/e2e/timeline-virtualization.mjs
+++ b/packages/studio/tests/e2e/timeline-virtualization.mjs
@@ -22,8 +22,8 @@
* it, and a mismatch would otherwise pass silently against the wrong build.
*/
import { platform, arch } from "node:os";
-import puppeteer from "puppeteer-core";
-import { resolveChromeExecutable } from "./chrome-executable.mjs";
+import { launchStudioChrome } from "./chrome-executable.mjs";
+import { gatePassed, judgeResponsiveness } from "./timeline-viewport-verdict.mjs";
const STUDIO_URL = process.env.STUDIO_URL;
const PROFILE = process.env.TIMELINE_PROFILE || "dense-short";
@@ -61,12 +61,6 @@ if (ROW_VIRTUALIZATION === "off" && ELEMENT_COUNT === 50_000) {
process.exit(2);
}
-function percentile(values, ratio) {
- if (values.length === 0) return 0;
- const sorted = [...values].sort((a, b) => a - b);
- return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)];
-}
-
async function collectHeapBytes(client) {
const usage = await client.send("Runtime.getHeapUsage");
return usage.usedSize;
@@ -90,6 +84,8 @@ async function collectRun(page, injectedLongTaskMs = 0) {
interactionP95Ms: percentileInPage(interactions, 0.95),
frameIntervalP95Ms: percentileInPage(frameIntervals, 0.95),
scrollSampleCount: interactions.length,
+ interactions,
+ frameIntervals,
longestTaskMs: Math.max(0, ...longTasks),
scrollWidth: scroller.scrollWidth,
scrollHeight: scroller.scrollHeight,
@@ -227,17 +223,7 @@ async function measureMaximumReliableScrollWidth(page) {
});
}
-const executablePath = resolveChromeExecutable();
-if (!executablePath) {
- console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH");
- process.exit(2);
-}
-
-const browser = await puppeteer.launch({
- executablePath,
- headless: true,
- args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"],
-});
+const { browser, executablePath } = await launchStudioChrome();
let exitCode = 1;
try {
const version = await browser.version();
@@ -319,10 +305,7 @@ try {
// so a skipped budget never reads as a passed one.
const domBudgetsApply = ROW_VIRTUALIZATION === "on";
for (const run of runs) {
- run.responsivenessPassed =
- run.interactionP95Ms <= interactionLimitMs &&
- run.frameIntervalP95Ms <= frameIntervalLimitMs &&
- run.longestTaskMs <= longTaskLimitMs;
+ run.longTaskPassed = run.longestTaskMs <= longTaskLimitMs;
run.timelineMounted = run.diagnostics.timelineRoots === 1;
run.domSizePassed = domBudgetsApply
? run.diagnostics.mountedRows <= budgets.maxMountedRows &&
@@ -330,8 +313,13 @@ try {
run.diagnostics.maxMountedClipRootsInOneRow <= budgets.maxMountedClipRootsPerRow &&
run.diagnostics.mountedTimelineDescendants <= budgets.maxMountedTimelineDescendants
: null;
- run.passed = run.responsivenessPassed && run.timelineMounted && run.domSizePassed !== false;
+ run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false;
}
+ const responsiveness = judgeResponsiveness(runs, {
+ samplesPerRun: budgets.scrollSamplesPerRun,
+ interactionLimitMs,
+ frameIntervalLimitMs,
+ });
await page.evaluate(() => window.__studioTest.resetTimelinePerformanceFixture());
await page.waitForFunction(
@@ -386,14 +374,9 @@ try {
directScrollGate,
runs,
aggregate: {
- interactionP95Ms: percentile(
- runs.map((run) => run.interactionP95Ms),
- 0.95,
- ),
- frameIntervalP95Ms: percentile(
- runs.map((run) => run.frameIntervalP95Ms),
- 0.95,
- ),
+ interactionP95Ms: responsiveness.interactionP95Ms,
+ frameIntervalP95Ms: responsiveness.frameIntervalP95Ms,
+ responsivenessPassed: responsiveness.passed,
passingRuns,
baselineHeapBytes,
returnedHeapBytes,
@@ -401,12 +384,15 @@ try {
},
};
console.log(JSON.stringify(evidence, null, 2));
- exitCode =
- directScrollGate.decision === "approved" &&
- passingRuns >= budgets.requiredPassingRuns &&
- memoryReturned
- ? 0
- : 1;
+ exitCode = gatePassed({
+ directScrollApproved: directScrollGate.decision === "approved",
+ responsivenessPassed: responsiveness.passed,
+ passingRuns,
+ requiredPassingRuns: budgets.requiredPassingRuns,
+ memoryReturned,
+ })
+ ? 0
+ : 1;
} finally {
await browser.close();
}