Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(studio): the timeline viewport gate judges p95 over every measured scroll step (#4847)
Miguel Ángelc20934a7205b
diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts b/packages/studio/src/player/lib/timelineViewportBudgets.test.tsindex 5689b5609a..a75d630ebe 100644--- a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts+++ b/packages/studio/src/player/lib/timelineViewportBudgets.test.ts@@ -23,7 +23,7 @@ describe("timeline viewport budgets", () => {constrainedLongTaskLimitMs: 300,posterCoverageRatio: 0.9,supportedFixtureFallbackRatio: 0.02,- scrollSamplesPerRun: 21,+ scrollSamplesPerRun: 63,warmupRuns: 3,measuredRuns: 5,requiredPassingRuns: 4,diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.ts b/packages/studio/src/player/lib/timelineViewportBudgets.tsindex b57bf325ec..771a37cd32 100644--- a/packages/studio/src/player/lib/timelineViewportBudgets.ts+++ b/packages/studio/src/player/lib/timelineViewportBudgets.ts@@ -99,7 +99,7 @@ export const TIMELINE_VIEWPORT_BUDGETS: Readonly<TimelineViewportBudgets> = ObjerichPreviewP95Ms: 750,constrainedRichPreviewP95Ms: 1_200,supportedFixtureFallbackRatio: 0.02,- scrollSamplesPerRun: 21,+ scrollSamplesPerRun: 63,warmupRuns: 3,measuredRuns: 5,requiredPassingRuns: 4,diff --git a/packages/studio/tests/e2e/chrome-executable.mjs b/packages/studio/tests/e2e/chrome-executable.mjsindex b1a868916e..8fc89f490e 100644--- a/packages/studio/tests/e2e/chrome-executable.mjs+++ b/packages/studio/tests/e2e/chrome-executable.mjs@@ -1,6 +1,7 @@import { existsSync, readdirSync } from "node:fs";import { homedir } from "node:os";import { join } from "node:path";+import puppeteer from "puppeteer-core";/** Resolve the same Chrome binary for every local Studio browser acceptance test. */export function resolveChromeExecutable() {@@ -22,3 +23,18 @@ export function resolveChromeExecutable() {...installed,].find((candidate) => candidate && existsSync(candidate));}++/** Headless Chrome for a Studio browser gate; exits 2 when no Chrome is installed. */+export async function launchStudioChrome() {+ const executablePath = resolveChromeExecutable();+ if (!executablePath) {+ console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH");+ process.exit(2);+ }+ const browser = await puppeteer.launch({+ executablePath,+ headless: true,+ args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"],+ });+ return { browser, executablePath };+}diff --git a/packages/studio/tests/e2e/fade-handles.mjs b/packages/studio/tests/e2e/fade-handles.mjsindex 18f5cb3c16..a4e4a9a5c2 100644--- a/packages/studio/tests/e2e/fade-handles.mjs+++ b/packages/studio/tests/e2e/fade-handles.mjs@@ -1,8 +1,7 @@#!/usr/bin/env node// On a clip narrower than two fade hit boxes, each dot must still lay out at 10 x 10.// Flex shrinking only happens in real layout, so this measures it in Chrome.-import puppeteer from "puppeteer-core";-import { resolveChromeExecutable } from "./chrome-executable.mjs";+import { launchStudioChrome } from "./chrome-executable.mjs";const STUDIO_URL = process.env.STUDIO_URL;const DOT_PX = 10;@@ -12,17 +11,7 @@ if (!STUDIO_URL) {console.error("STUDIO_URL is required and must point at the fade-handles fixture");process.exit(2);}-const executablePath = resolveChromeExecutable();-if (!executablePath) {- console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH");- process.exit(2);-}--const browser = await puppeteer.launch({- executablePath,- headless: true,- args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"],-});+const { browser } = await launchStudioChrome();const failures = [];let evidence = {};try {diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.mjsnew file mode 100644index 0000000000..d9f0f3f639--- /dev/null+++ b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs@@ -0,0 +1,54 @@+/**+ * The timeline viewport gate's verdict, kept apart from the script that drives Chrome so it can be tested.+ */++/** Nearest-rank percentile: `ratio` 0.95 of 315 values is the 16th-worst. */+export function percentile(values, ratio) {+ if (values.length === 0) return 0;+ const sorted = [...values].sort((a, b) => a - b);+ return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)];+}++function assertSampleCount(expected, interactions, frameIntervals) {+ if (expected > 0 && interactions.length === expected && frameIntervals.length === expected)+ return;+ throw new Error(+ `Expected ${expected} scroll samples, measured ${interactions.length} interactions ` ++ `and ${frameIntervals.length} frame intervals`,+ );+}++/**+ * p95 over every measured step pooled: one run's p95 is only its 4th-worst step, so a brief runner stall failed it.+ * Throws when any run is short of samples, so a missing measurement cannot read as a fast one.+ */+export function judgeResponsiveness(+ runs,+ { samplesPerRun, interactionLimitMs, frameIntervalLimitMs },+) {+ const interactions = runs.flatMap((run) => run.interactions);+ const frameIntervals = runs.flatMap((run) => run.frameIntervals);+ assertSampleCount(runs.length * samplesPerRun, interactions, frameIntervals);+ const interactionP95Ms = percentile(interactions, 0.95);+ const frameIntervalP95Ms = percentile(frameIntervals, 0.95);+ return {+ interactionP95Ms,+ frameIntervalP95Ms,+ passed: interactionP95Ms <= interactionLimitMs && frameIntervalP95Ms <= frameIntervalLimitMs,+ };+}++export function gatePassed({+ directScrollApproved,+ responsivenessPassed,+ passingRuns,+ requiredPassingRuns,+ memoryReturned,+}) {+ return (+ directScrollApproved &&+ responsivenessPassed &&+ passingRuns >= requiredPassingRuns &&+ memoryReturned+ );+}diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjsnew file mode 100644index 0000000000..c5a54af4b9--- /dev/null+++ b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs@@ -0,0 +1,88 @@+import { describe, expect, it } from "vitest";+import { gatePassed, judgeResponsiveness, percentile } from "./timeline-viewport-verdict.mjs";++const LIMITS = { samplesPerRun: 63, interactionLimitMs: 75, frameIntervalLimitMs: 75 };+const FAST = 49;+const SLOW = 83;++/** Five runs of 63 steps; `slowAt(run, step)` marks the steps that take five frames. */+function runs(slowAt) {+ return Array.from({ length: 5 }, (_, run) => {+ const interactions = Array.from({ length: 63 }, (_, step) => (slowAt(run, step) ? SLOW : FAST));+ return { interactions, frameIntervals: interactions.map(() => 33.3) };+ });+}++const slowSteps = (count) => (run, step) => run * 63 + step < count;++describe("percentile", () => {+ it("takes the nearest rank, so the p95 of 315 steps is the 16th-worst", () => {+ const values = Array.from({ length: 315 }, (_, index) => index);+ expect(percentile(values, 0.95)).toBe(299);+ expect(percentile([3, 1, 2], 0.95)).toBe(3);+ });+});++describe("judgeResponsiveness", () => {+ it("fails 16 slow steps of 315 and passes 15", () => {+ expect(judgeResponsiveness(runs(slowSteps(16)), LIMITS)).toMatchObject({+ interactionP95Ms: SLOW,+ passed: false,+ });+ expect(judgeResponsiveness(runs(slowSteps(15)), LIMITS)).toMatchObject({+ interactionP95Ms: FAST,+ passed: true,+ });+ });++ it("passes two slow steps in every run, 10 of 315", () => {+ expect(+ judgeResponsiveness(+ runs((_, step) => step < 2),+ LIMITS,+ ).passed,+ ).toBe(true);+ });++ it("fails one run that is slow throughout", () => {+ expect(+ judgeResponsiveness(+ runs((run) => run === 2),+ LIMITS,+ ).passed,+ ).toBe(false);+ });++ it("fails on the frame interval alone", () => {+ const measured = runs(() => false).map((run) => ({+ ...run,+ frameIntervals: run.frameIntervals.map(() => 83),+ }));+ expect(judgeResponsiveness(measured, LIMITS).passed).toBe(false);+ });++ it("throws on a run short of samples instead of reading it as fast", () => {+ const measured = runs(() => false);+ measured[1] = { interactions: [], frameIntervals: [] };+ expect(() => judgeResponsiveness(measured, LIMITS)).toThrow("Expected 315 scroll samples");+ expect(() => judgeResponsiveness([], LIMITS)).toThrow("Expected 0 scroll samples");+ });+});++describe("gatePassed", () => {+ const passing = {+ directScrollApproved: true,+ responsivenessPassed: true,+ passingRuns: 5,+ requiredPassingRuns: 4,+ memoryReturned: true,+ };++ it("passes only when every check holds", () => {+ expect(gatePassed(passing)).toBe(true);+ expect(gatePassed({ ...passing, responsivenessPassed: false })).toBe(false);+ expect(gatePassed({ ...passing, directScrollApproved: false })).toBe(false);+ expect(gatePassed({ ...passing, passingRuns: 3 })).toBe(false);+ expect(gatePassed({ ...passing, memoryReturned: false })).toBe(false);+ });+});diff --git a/packages/studio/tests/e2e/timeline-virtualization.mjs b/packages/studio/tests/e2e/timeline-virtualization.mjsindex 521da8ffee..ff6435755f 100644--- a/packages/studio/tests/e2e/timeline-virtualization.mjs+++ b/packages/studio/tests/e2e/timeline-virtualization.mjs@@ -22,8 +22,8 @@* it, and a mismatch would otherwise pass silently against the wrong build.*/import { platform, arch } from "node:os";-import puppeteer from "puppeteer-core";-import { resolveChromeExecutable } from "./chrome-executable.mjs";+import { launchStudioChrome } from "./chrome-executable.mjs";+import { gatePassed, judgeResponsiveness } from "./timeline-viewport-verdict.mjs";const STUDIO_URL = process.env.STUDIO_URL;const PROFILE = process.env.TIMELINE_PROFILE || "dense-short";@@ -61,12 +61,6 @@ if (ROW_VIRTUALIZATION === "off" && ELEMENT_COUNT === 50_000) {process.exit(2);}-function percentile(values, ratio) {- if (values.length === 0) return 0;- const sorted = [...values].sort((a, b) => a - b);- return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)];-}-async function collectHeapBytes(client) {const usage = await client.send("Runtime.getHeapUsage");return usage.usedSize;@@ -90,6 +84,8 @@ async function collectRun(page, injectedLongTaskMs = 0) {interactionP95Ms: percentileInPage(interactions, 0.95),frameIntervalP95Ms: percentileInPage(frameIntervals, 0.95),scrollSampleCount: interactions.length,+ interactions,+ frameIntervals,longestTaskMs: Math.max(0, ...longTasks),scrollWidth: scroller.scrollWidth,scrollHeight: scroller.scrollHeight,@@ -227,17 +223,7 @@ async function measureMaximumReliableScrollWidth(page) {});}-const executablePath = resolveChromeExecutable();-if (!executablePath) {- console.error("No Chrome executable found; set PUPPETEER_EXECUTABLE_PATH");- process.exit(2);-}--const browser = await puppeteer.launch({- executablePath,- headless: true,- args: ["--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu"],-});+const { browser, executablePath } = await launchStudioChrome();let exitCode = 1;try {const version = await browser.version();@@ -319,10 +305,7 @@ try {// so a skipped budget never reads as a passed one.const domBudgetsApply = ROW_VIRTUALIZATION === "on";for (const run of runs) {- run.responsivenessPassed =- run.interactionP95Ms <= interactionLimitMs &&- run.frameIntervalP95Ms <= frameIntervalLimitMs &&- run.longestTaskMs <= longTaskLimitMs;+ run.longTaskPassed = run.longestTaskMs <= longTaskLimitMs;run.timelineMounted = run.diagnostics.timelineRoots === 1;run.domSizePassed = domBudgetsApply? run.diagnostics.mountedRows <= budgets.maxMountedRows &&@@ -330,8 +313,13 @@ try {run.diagnostics.maxMountedClipRootsInOneRow <= budgets.maxMountedClipRootsPerRow &&run.diagnostics.mountedTimelineDescendants <= budgets.maxMountedTimelineDescendants: null;- run.passed = run.responsivenessPassed && run.timelineMounted && run.domSizePassed !== false;+ run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false;}+ const responsiveness = judgeResponsiveness(runs, {+ samplesPerRun: budgets.scrollSamplesPerRun,+ interactionLimitMs,+ frameIntervalLimitMs,+ });await page.evaluate(() => window.__studioTest.resetTimelinePerformanceFixture());await page.waitForFunction(@@ -386,14 +374,9 @@ try {directScrollGate,runs,aggregate: {- interactionP95Ms: percentile(- runs.map((run) => run.interactionP95Ms),- 0.95,- ),- frameIntervalP95Ms: percentile(- runs.map((run) => run.frameIntervalP95Ms),- 0.95,- ),+ interactionP95Ms: responsiveness.interactionP95Ms,+ frameIntervalP95Ms: responsiveness.frameIntervalP95Ms,+ responsivenessPassed: responsiveness.passed,passingRuns,baselineHeapBytes,returnedHeapBytes,@@ -401,12 +384,15 @@ try {},};console.log(JSON.stringify(evidence, null, 2));- exitCode =- directScrollGate.decision === "approved" &&- passingRuns >= budgets.requiredPassingRuns &&- memoryReturned- ? 0- : 1;+ exitCode = gatePassed({+ directScrollApproved: directScrollGate.decision === "approved",+ responsivenessPassed: responsiveness.passed,+ passingRuns,+ requiredPassingRuns: budgets.requiredPassingRuns,+ memoryReturned,+ })+ ? 0+ : 1;} finally {await browser.close();}