Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(studio): the timeline scroll gate catches dropped frames in the virtualized view (#4855)
Miguel Ángel21a29c2ac747
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 1491d220a8..a72ef42c90 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -925,6 +925,14 @@ jobs:echo "### Timeline viewport gate"echo "- Default arm exit: ${DEFAULT_STATUS}"echo "- Explicitly disabled arm exit: ${DISABLED_STATUS}"+ for arm in default disabled; do+ node -e '+ const e = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));+ const b = e.environment.appliedBudgets;+ for (const a of e.attempts)+ console.log(`- ${process.argv[2]} attempt ${a.attempt}: interaction p95 ${a.interactionP95Ms.toFixed(1)}/${b.interactionP95Ms} ms, frame p95 ${a.frameIntervalP95Ms.toFixed(1)}/${b.frameIntervalP95Ms} ms, ${a.passingRuns}/${a.runs.length} runs, ${a.passed ? "pass" : "fail"}`);+ ' "/tmp/timeline-gate-${arm}.json" "$arm" || echo "- ${arm}: no evidence"+ done} >> "$GITHUB_STEP_SUMMARY"if (( DEFAULT_STATUS != 0 || DISABLED_STATUS != 0 )); thendiff --git a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts b/packages/studio/src/player/lib/timelineViewportBudgets.test.tsindex a75d630ebe..3c8a56496c 100644--- a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts+++ b/packages/studio/src/player/lib/timelineViewportBudgets.test.ts@@ -19,6 +19,8 @@ describe("timeline viewport budgets", () => {interactionP95Ms: 50,constrainedInteractionP95Ms: 75,constrainedFrameIntervalP95Ms: 75,+ ciVirtualizedInteractionP95Ms: 58.3,+ ciVirtualizedFrameIntervalP95Ms: 25,longTaskLimitMs: 50,constrainedLongTaskLimitMs: 300,posterCoverageRatio: 0.9,diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.ts b/packages/studio/src/player/lib/timelineViewportBudgets.tsindex 771a37cd32..4e4c2e3e1c 100644--- a/packages/studio/src/player/lib/timelineViewportBudgets.ts+++ b/packages/studio/src/player/lib/timelineViewportBudgets.ts@@ -28,6 +28,8 @@ export interface TimelineViewportBudgets {frameIntervalP95Ms: number;constrainedInteractionP95Ms: number;constrainedFrameIntervalP95Ms: number;+ ciVirtualizedInteractionP95Ms: number;+ ciVirtualizedFrameIntervalP95Ms: number;longTaskLimitMs: number;constrainedLongTaskLimitMs: number;memoryReturnToleranceRatio: number;@@ -86,6 +88,8 @@ export const TIMELINE_VIEWPORT_BUDGETS: Readonly<TimelineViewportBudgets> = ObjeframeIntervalP95Ms: 33.3,constrainedInteractionP95Ms: 75,constrainedFrameIntervalP95Ms: 75,+ ciVirtualizedInteractionP95Ms: 58.3,+ ciVirtualizedFrameIntervalP95Ms: 25,longTaskLimitMs: 50,constrainedLongTaskLimitMs: 300,memoryReturnToleranceRatio: 0.15,diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.mjsindex d9f0f3f639..1e7bcfdf78 100644--- a/packages/studio/tests/e2e/timeline-viewport-verdict.mjs+++ b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs@@ -9,6 +9,24 @@ export function percentile(values, ratio) {return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)];}+/** The p95 pair a tier is held to; the CI virtualized arm has its own, tighter one. */+export function responsivenessLimits(budgets, tier, rowVirtualization) {+ if (tier === "primary")+ return {+ interactionLimitMs: budgets.interactionP95Ms,+ frameIntervalLimitMs: budgets.frameIntervalP95Ms,+ };+ if (tier === "ci" && rowVirtualization === "on")+ return {+ interactionLimitMs: budgets.ciVirtualizedInteractionP95Ms,+ frameIntervalLimitMs: budgets.ciVirtualizedFrameIntervalP95Ms,+ };+ return {+ interactionLimitMs: budgets.constrainedInteractionP95Ms,+ frameIntervalLimitMs: budgets.constrainedFrameIntervalP95Ms,+ };+}+function assertSampleCount(expected, interactions, frameIntervals) {if (expected > 0 && interactions.length === expected && frameIntervals.length === expected)return;@@ -38,17 +56,14 @@ export function judgeResponsiveness(};}-export function gatePassed({- directScrollApproved,- responsivenessPassed,- passingRuns,- requiredPassingRuns,- memoryReturned,-}) {- return (- directScrollApproved &&- responsivenessPassed &&- passingRuns >= requiredPassingRuns &&- memoryReturned- );+/** A failed timing attempt is measured once more, so one bad stretch of a shared runner cannot fail the gate alone. */+export const TIMING_ATTEMPTS = 2;++export function attemptPassed({ responsivenessPassed, passingRuns, requiredPassingRuns }) {+ return responsivenessPassed && passingRuns >= requiredPassingRuns;+}++export function gatePassed({ directScrollApproved, attempts, memoryReturned }) {+ const timingPassed = attempts.slice(0, TIMING_ATTEMPTS).some((attempt) => attempt.passed);+ return directScrollApproved && timingPassed && memoryReturned;}diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjsindex c5a54af4b9..91c52153ef 100644--- a/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs+++ b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs@@ -1,5 +1,12 @@import { describe, expect, it } from "vitest";-import { gatePassed, judgeResponsiveness, percentile } from "./timeline-viewport-verdict.mjs";+import { TIMELINE_VIEWPORT_BUDGETS } from "../../src/player/lib/timelineViewportBudgets";+import {+ attemptPassed,+ gatePassed,+ judgeResponsiveness,+ percentile,+ responsivenessLimits,+} from "./timeline-viewport-verdict.mjs";const LIMITS = { samplesPerRun: 63, interactionLimitMs: 75, frameIntervalLimitMs: 75 };const FAST = 49;@@ -69,20 +76,73 @@ describe("judgeResponsiveness", () => {});});-describe("gatePassed", () => {- const passing = {- directScrollApproved: true,- responsivenessPassed: true,- passingRuns: 5,- requiredPassingRuns: 4,- memoryReturned: true,+describe("the CI virtualized arm's limits", () => {+ const limits = {+ samplesPerRun: 63,+ ...responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "ci", "on"),};+ // Two frames is a normal step, four a step two frames late; one frame is a normal interval, two a dropped one.+ const steps = (count, normal, slow) =>+ Array.from({ length: 5 }, (_, run) =>+ Array.from({ length: 63 }, (_, step) => (run * 63 + step < count ? slow : normal)),+ );+ const interactionRuns = (count) =>+ steps(count, 33.3, 66.7).map((interactions) => ({+ interactions,+ frameIntervals: interactions.map(() => 16.7),+ }));+ const frameRuns = (count) =>+ steps(count, 16.7, 33.3).map((frameIntervals) => ({+ interactions: frameIntervals.map(() => 33.3),+ frameIntervals,+ }));++ it("fails 16 of 315 steps two frames late and passes 15", () => {+ expect(judgeResponsiveness(interactionRuns(16), limits).passed).toBe(false);+ expect(judgeResponsiveness(interactionRuns(15), limits).passed).toBe(true);+ });++ it("fails 16 of 315 frame intervals that drop a frame and passes 15", () => {+ expect(judgeResponsiveness(frameRuns(16), limits).passed).toBe(false);+ expect(judgeResponsiveness(frameRuns(15), limits).passed).toBe(true);+ });++ it("leaves the unvirtualized arm and the other constrained tiers at 75 ms", () => {+ const loose = { interactionLimitMs: 75, frameIntervalLimitMs: 75 };+ expect(responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "ci", "off")).toEqual(loose);+ expect(responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "low-resource", "on")).toEqual(loose);+ expect(responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "primary", "on")).toEqual({+ interactionLimitMs: 50,+ frameIntervalLimitMs: 33.3,+ });+ });+});++describe("attemptPassed", () => {+ const passing = { responsivenessPassed: true, passingRuns: 5, requiredPassingRuns: 4 };++ it("needs pooled responsiveness and enough passing runs", () => {+ expect(attemptPassed(passing)).toBe(true);+ expect(attemptPassed({ ...passing, responsivenessPassed: false })).toBe(false);+ expect(attemptPassed({ ...passing, passingRuns: 3 })).toBe(false);+ });+});++describe("gatePassed", () => {+ const pass = { passed: true };+ const fail = { passed: false };+ const passing = { directScrollApproved: true, attempts: [pass], memoryReturned: true };it("passes only when every check holds", () => {expect(gatePassed(passing)).toBe(true);- expect(gatePassed({ ...passing, responsivenessPassed: false })).toBe(false);expect(gatePassed({ ...passing, directScrollApproved: false })).toBe(false);- expect(gatePassed({ ...passing, passingRuns: 3 })).toBe(false);expect(gatePassed({ ...passing, memoryReturned: false })).toBe(false);});++ it("fails timing only when the attempt and its one rerun both fail", () => {+ expect(gatePassed({ ...passing, attempts: [fail, pass] })).toBe(true);+ expect(gatePassed({ ...passing, attempts: [fail, fail] })).toBe(false);+ expect(gatePassed({ ...passing, attempts: [fail, fail, pass] })).toBe(false);+ expect(gatePassed({ ...passing, attempts: [] })).toBe(false);+ });});diff --git a/packages/studio/tests/e2e/timeline-virtualization.mjs b/packages/studio/tests/e2e/timeline-virtualization.mjsindex ff6435755f..b53c07065a 100644--- a/packages/studio/tests/e2e/timeline-virtualization.mjs+++ b/packages/studio/tests/e2e/timeline-virtualization.mjs@@ -10,9 +10,9 @@* TIMELINE_TIER selects the budget set and the emulation applied. "primary" is* a developer machine and holds the strict budgets. "low-resource" and* "high-dpr" add CPU throttling and a 2x scale factor respectively. "ci" is a- * shared runner: no emulation, but the constrained budgets, because a hosted- * runner is already slower and noisier than the machine the strict numbers were- * recorded on. Throttling it further would measure the throttle, not the build.+ * shared runner with no emulation: its virtualized arm holds its own limits+ * (responsivenessLimits), its unvirtualized arm the constrained ones. Throttling+ * it further would measure the throttle, not the build.* CI also requires production React and reports the observed runtime so Vite's* development-only checks can never contaminate the shipped-code measurement.*@@ -23,7 +23,13 @@*/import { platform, arch } from "node:os";import { launchStudioChrome } from "./chrome-executable.mjs";-import { gatePassed, judgeResponsiveness } from "./timeline-viewport-verdict.mjs";+import {+ attemptPassed,+ gatePassed,+ judgeResponsiveness,+ responsivenessLimits,+ TIMING_ATTEMPTS,+} from "./timeline-viewport-verdict.mjs";const STUDIO_URL = process.env.STUDIO_URL;const PROFILE = process.env.TIMELINE_PROFILE || "dense-short";@@ -290,36 +296,61 @@ try {);}- const runs = [];- const interactionLimitMs =- TIER === "primary" ? budgets.interactionP95Ms : budgets.constrainedInteractionP95Ms;- const frameIntervalLimitMs =- TIER === "primary" ? budgets.frameIntervalP95Ms : budgets.constrainedFrameIntervalP95Ms;- for (let index = 0; index < budgets.warmupRuns + budgets.measuredRuns; index += 1) {- const run = await collectRun(page);- if (index >= budgets.warmupRuns) runs.push(run);- }+ const { interactionLimitMs, frameIntervalLimitMs } = responsivenessLimits(+ budgets,+ TIER,+ ROW_VIRTUALIZATION,+ );// Latency, long tasks and memory are product promises and hold for both// builds. The DOM-size budgets describe what windowing achieves, so they only// apply when windowing is on. They are skipped explicitly rather than relaxed,// so a skipped budget never reads as a passed one.const domBudgetsApply = ROW_VIRTUALIZATION === "on";- for (const run of runs) {- run.longTaskPassed = run.longestTaskMs <= longTaskLimitMs;- run.timelineMounted = run.diagnostics.timelineRoots === 1;- run.domSizePassed = domBudgetsApply- ? run.diagnostics.mountedRows <= budgets.maxMountedRows &&- run.diagnostics.mountedClipRoots <= budgets.maxMountedClipRoots &&- run.diagnostics.maxMountedClipRootsInOneRow <= budgets.maxMountedClipRootsPerRow &&- run.diagnostics.mountedTimelineDescendants <= budgets.maxMountedTimelineDescendants- : null;- run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false;+ const attempts = [];+ while (attempts.length < TIMING_ATTEMPTS && !attempts.some((attempt) => attempt.passed)) {+ const runs = [];+ for (let index = 0; index < budgets.warmupRuns + budgets.measuredRuns; index += 1) {+ const run = await collectRun(page);+ if (index >= budgets.warmupRuns) runs.push(run);+ }+ for (const run of runs) {+ run.longTaskPassed = run.longestTaskMs <= longTaskLimitMs;+ run.timelineMounted = run.diagnostics.timelineRoots === 1;+ run.domSizePassed = domBudgetsApply+ ? run.diagnostics.mountedRows <= budgets.maxMountedRows &&+ run.diagnostics.mountedClipRoots <= budgets.maxMountedClipRoots &&+ run.diagnostics.maxMountedClipRootsInOneRow <= budgets.maxMountedClipRootsPerRow &&+ run.diagnostics.mountedTimelineDescendants <= budgets.maxMountedTimelineDescendants+ : null;+ run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false;+ }+ const responsiveness = judgeResponsiveness(runs, {+ samplesPerRun: budgets.scrollSamplesPerRun,+ interactionLimitMs,+ frameIntervalLimitMs,+ });+ const passingRuns = runs.filter((run) => run.passed).length;+ const attempt = {+ attempt: attempts.length + 1,+ interactionP95Ms: responsiveness.interactionP95Ms,+ frameIntervalP95Ms: responsiveness.frameIntervalP95Ms,+ responsivenessPassed: responsiveness.passed,+ passingRuns,+ passed: attemptPassed({+ responsivenessPassed: responsiveness.passed,+ passingRuns,+ requiredPassingRuns: budgets.requiredPassingRuns,+ }),+ runs,+ };+ attempts.push(attempt);+ console.error(+ `timeline gate ${ROW_VIRTUALIZATION} attempt ${attempt.attempt}: ` ++ `interaction p95 ${attempt.interactionP95Ms.toFixed(1)}/${interactionLimitMs} ms, ` ++ `frame p95 ${attempt.frameIntervalP95Ms.toFixed(1)}/${frameIntervalLimitMs} ms, ` ++ `${passingRuns}/${runs.length} runs passed, ${attempt.passed ? "PASS" : "FAIL"}`,+ );}- const responsiveness = judgeResponsiveness(runs, {- samplesPerRun: budgets.scrollSamplesPerRun,- interactionLimitMs,- frameIntervalLimitMs,- });await page.evaluate(() => window.__studioTest.resetTimelinePerformanceFixture());await page.waitForFunction(@@ -331,8 +362,10 @@ try {const returnedHeapBytes = await collectHeapBytes(client);const memoryReturned =returnedHeapBytes <= baselineHeapBytes * (1 + budgets.memoryReturnToleranceRatio);- const passingRuns = runs.filter((run) => run.passed).length;- const maxTimelineContentWidthPx = Math.max(0, ...runs.map((run) => run.scrollWidth));+ const maxTimelineContentWidthPx = Math.max(+ 0,+ ...attempts.flatMap((attempt) => attempt.runs.map((run) => run.scrollWidth)),+ );const directScrollGate = {safetyEnvelopePx: budgets.directScrollSafetyPx,maxTimelineContentWidthPx,@@ -369,15 +402,13 @@ try {warmups: budgets.warmupRuns,measured: budgets.measuredRuns,requiredPassing: budgets.requiredPassingRuns,+ timingAttempts: TIMING_ATTEMPTS,},},directScrollGate,- runs,+ attempts,aggregate: {- interactionP95Ms: responsiveness.interactionP95Ms,- frameIntervalP95Ms: responsiveness.frameIntervalP95Ms,- responsivenessPassed: responsiveness.passed,- passingRuns,+ timingPassed: attempts.some((attempt) => attempt.passed),baselineHeapBytes,returnedHeapBytes,memoryReturned,@@ -386,9 +417,7 @@ try {console.log(JSON.stringify(evidence, null, 2));exitCode = gatePassed({directScrollApproved: directScrollGate.decision === "approved",- responsivenessPassed: responsiveness.passed,- passingRuns,- requiredPassingRuns: budgets.requiredPassingRuns,+ attempts,memoryReturned,})? 0