Skip to content

Write HTML. Render video. Built for agents.

fix(studio): the timeline scroll gate catches dropped frames in the virtualized view (#4855)

Miguel Ángel21a29c2ac747
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1491d220a8..a72ef42c90 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -925,6 +925,14 @@ jobs:
echo "### Timeline viewport gate"
echo "- Default arm exit: ${DEFAULT_STATUS}"
echo "- Explicitly disabled arm exit: ${DISABLED_STATUS}"
+ for arm in default disabled; do
+ node -e '
+ const e = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
+ const b = e.environment.appliedBudgets;
+ for (const a of e.attempts)
+ console.log(`- ${process.argv[2]} attempt ${a.attempt}: interaction p95 ${a.interactionP95Ms.toFixed(1)}/${b.interactionP95Ms} ms, frame p95 ${a.frameIntervalP95Ms.toFixed(1)}/${b.frameIntervalP95Ms} ms, ${a.passingRuns}/${a.runs.length} runs, ${a.passed ? "pass" : "fail"}`);
+ ' "/tmp/timeline-gate-${arm}.json" "$arm" || echo "- ${arm}: no evidence"
+ done
} >> "$GITHUB_STEP_SUMMARY"
if (( DEFAULT_STATUS != 0 || DISABLED_STATUS != 0 )); then
diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts b/packages/studio/src/player/lib/timelineViewportBudgets.test.ts
index a75d630ebe..3c8a56496c 100644
--- a/packages/studio/src/player/lib/timelineViewportBudgets.test.ts
+++ b/packages/studio/src/player/lib/timelineViewportBudgets.test.ts
@@ -19,6 +19,8 @@ describe("timeline viewport budgets", () => {
interactionP95Ms: 50,
constrainedInteractionP95Ms: 75,
constrainedFrameIntervalP95Ms: 75,
+ ciVirtualizedInteractionP95Ms: 58.3,
+ ciVirtualizedFrameIntervalP95Ms: 25,
longTaskLimitMs: 50,
constrainedLongTaskLimitMs: 300,
posterCoverageRatio: 0.9,
diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.ts b/packages/studio/src/player/lib/timelineViewportBudgets.ts
index 771a37cd32..4e4c2e3e1c 100644
--- a/packages/studio/src/player/lib/timelineViewportBudgets.ts
+++ b/packages/studio/src/player/lib/timelineViewportBudgets.ts
@@ -28,6 +28,8 @@ export interface TimelineViewportBudgets {
frameIntervalP95Ms: number;
constrainedInteractionP95Ms: number;
constrainedFrameIntervalP95Ms: number;
+ ciVirtualizedInteractionP95Ms: number;
+ ciVirtualizedFrameIntervalP95Ms: number;
longTaskLimitMs: number;
constrainedLongTaskLimitMs: number;
memoryReturnToleranceRatio: number;
@@ -86,6 +88,8 @@ export const TIMELINE_VIEWPORT_BUDGETS: Readonly<TimelineViewportBudgets> = Obje
frameIntervalP95Ms: 33.3,
constrainedInteractionP95Ms: 75,
constrainedFrameIntervalP95Ms: 75,
+ ciVirtualizedInteractionP95Ms: 58.3,
+ ciVirtualizedFrameIntervalP95Ms: 25,
longTaskLimitMs: 50,
constrainedLongTaskLimitMs: 300,
memoryReturnToleranceRatio: 0.15,
diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs
index d9f0f3f639..1e7bcfdf78 100644
--- a/packages/studio/tests/e2e/timeline-viewport-verdict.mjs
+++ b/packages/studio/tests/e2e/timeline-viewport-verdict.mjs
@@ -9,6 +9,24 @@ export function percentile(values, ratio) {
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)];
}
+/** The p95 pair a tier is held to; the CI virtualized arm has its own, tighter one. */
+export function responsivenessLimits(budgets, tier, rowVirtualization) {
+ if (tier === "primary")
+ return {
+ interactionLimitMs: budgets.interactionP95Ms,
+ frameIntervalLimitMs: budgets.frameIntervalP95Ms,
+ };
+ if (tier === "ci" && rowVirtualization === "on")
+ return {
+ interactionLimitMs: budgets.ciVirtualizedInteractionP95Ms,
+ frameIntervalLimitMs: budgets.ciVirtualizedFrameIntervalP95Ms,
+ };
+ return {
+ interactionLimitMs: budgets.constrainedInteractionP95Ms,
+ frameIntervalLimitMs: budgets.constrainedFrameIntervalP95Ms,
+ };
+}
+
function assertSampleCount(expected, interactions, frameIntervals) {
if (expected > 0 && interactions.length === expected && frameIntervals.length === expected)
return;
@@ -38,17 +56,14 @@ export function judgeResponsiveness(
};
}
-export function gatePassed({
- directScrollApproved,
- responsivenessPassed,
- passingRuns,
- requiredPassingRuns,
- memoryReturned,
-}) {
- return (
- directScrollApproved &&
- responsivenessPassed &&
- passingRuns >= requiredPassingRuns &&
- memoryReturned
- );
+/** A failed timing attempt is measured once more, so one bad stretch of a shared runner cannot fail the gate alone. */
+export const TIMING_ATTEMPTS = 2;
+
+export function attemptPassed({ responsivenessPassed, passingRuns, requiredPassingRuns }) {
+ return responsivenessPassed && passingRuns >= requiredPassingRuns;
+}
+
+export function gatePassed({ directScrollApproved, attempts, memoryReturned }) {
+ const timingPassed = attempts.slice(0, TIMING_ATTEMPTS).some((attempt) => attempt.passed);
+ return directScrollApproved && timingPassed && memoryReturned;
}
diff --git a/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs
index c5a54af4b9..91c52153ef 100644
--- a/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs
+++ b/packages/studio/tests/e2e/timeline-viewport-verdict.test.mjs
@@ -1,5 +1,12 @@
import { describe, expect, it } from "vitest";
-import { gatePassed, judgeResponsiveness, percentile } from "./timeline-viewport-verdict.mjs";
+import { TIMELINE_VIEWPORT_BUDGETS } from "../../src/player/lib/timelineViewportBudgets";
+import {
+ attemptPassed,
+ gatePassed,
+ judgeResponsiveness,
+ percentile,
+ responsivenessLimits,
+} from "./timeline-viewport-verdict.mjs";
const LIMITS = { samplesPerRun: 63, interactionLimitMs: 75, frameIntervalLimitMs: 75 };
const FAST = 49;
@@ -69,20 +76,73 @@ describe("judgeResponsiveness", () => {
});
});
-describe("gatePassed", () => {
- const passing = {
- directScrollApproved: true,
- responsivenessPassed: true,
- passingRuns: 5,
- requiredPassingRuns: 4,
- memoryReturned: true,
+describe("the CI virtualized arm's limits", () => {
+ const limits = {
+ samplesPerRun: 63,
+ ...responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "ci", "on"),
};
+ // Two frames is a normal step, four a step two frames late; one frame is a normal interval, two a dropped one.
+ const steps = (count, normal, slow) =>
+ Array.from({ length: 5 }, (_, run) =>
+ Array.from({ length: 63 }, (_, step) => (run * 63 + step < count ? slow : normal)),
+ );
+ const interactionRuns = (count) =>
+ steps(count, 33.3, 66.7).map((interactions) => ({
+ interactions,
+ frameIntervals: interactions.map(() => 16.7),
+ }));
+ const frameRuns = (count) =>
+ steps(count, 16.7, 33.3).map((frameIntervals) => ({
+ interactions: frameIntervals.map(() => 33.3),
+ frameIntervals,
+ }));
+
+ it("fails 16 of 315 steps two frames late and passes 15", () => {
+ expect(judgeResponsiveness(interactionRuns(16), limits).passed).toBe(false);
+ expect(judgeResponsiveness(interactionRuns(15), limits).passed).toBe(true);
+ });
+
+ it("fails 16 of 315 frame intervals that drop a frame and passes 15", () => {
+ expect(judgeResponsiveness(frameRuns(16), limits).passed).toBe(false);
+ expect(judgeResponsiveness(frameRuns(15), limits).passed).toBe(true);
+ });
+
+ it("leaves the unvirtualized arm and the other constrained tiers at 75 ms", () => {
+ const loose = { interactionLimitMs: 75, frameIntervalLimitMs: 75 };
+ expect(responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "ci", "off")).toEqual(loose);
+ expect(responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "low-resource", "on")).toEqual(loose);
+ expect(responsivenessLimits(TIMELINE_VIEWPORT_BUDGETS, "primary", "on")).toEqual({
+ interactionLimitMs: 50,
+ frameIntervalLimitMs: 33.3,
+ });
+ });
+});
+
+describe("attemptPassed", () => {
+ const passing = { responsivenessPassed: true, passingRuns: 5, requiredPassingRuns: 4 };
+
+ it("needs pooled responsiveness and enough passing runs", () => {
+ expect(attemptPassed(passing)).toBe(true);
+ expect(attemptPassed({ ...passing, responsivenessPassed: false })).toBe(false);
+ expect(attemptPassed({ ...passing, passingRuns: 3 })).toBe(false);
+ });
+});
+
+describe("gatePassed", () => {
+ const pass = { passed: true };
+ const fail = { passed: false };
+ const passing = { directScrollApproved: true, attempts: [pass], memoryReturned: true };
it("passes only when every check holds", () => {
expect(gatePassed(passing)).toBe(true);
- expect(gatePassed({ ...passing, responsivenessPassed: false })).toBe(false);
expect(gatePassed({ ...passing, directScrollApproved: false })).toBe(false);
- expect(gatePassed({ ...passing, passingRuns: 3 })).toBe(false);
expect(gatePassed({ ...passing, memoryReturned: false })).toBe(false);
});
+
+ it("fails timing only when the attempt and its one rerun both fail", () => {
+ expect(gatePassed({ ...passing, attempts: [fail, pass] })).toBe(true);
+ expect(gatePassed({ ...passing, attempts: [fail, fail] })).toBe(false);
+ expect(gatePassed({ ...passing, attempts: [fail, fail, pass] })).toBe(false);
+ expect(gatePassed({ ...passing, attempts: [] })).toBe(false);
+ });
});
diff --git a/packages/studio/tests/e2e/timeline-virtualization.mjs b/packages/studio/tests/e2e/timeline-virtualization.mjs
index ff6435755f..b53c07065a 100644
--- a/packages/studio/tests/e2e/timeline-virtualization.mjs
+++ b/packages/studio/tests/e2e/timeline-virtualization.mjs
@@ -10,9 +10,9 @@
* TIMELINE_TIER selects the budget set and the emulation applied. "primary" is
* a developer machine and holds the strict budgets. "low-resource" and
* "high-dpr" add CPU throttling and a 2x scale factor respectively. "ci" is a
- * shared runner: no emulation, but the constrained budgets, because a hosted
- * runner is already slower and noisier than the machine the strict numbers were
- * recorded on. Throttling it further would measure the throttle, not the build.
+ * shared runner with no emulation: its virtualized arm holds its own limits
+ * (responsivenessLimits), its unvirtualized arm the constrained ones. Throttling
+ * it further would measure the throttle, not the build.
* CI also requires production React and reports the observed runtime so Vite's
* development-only checks can never contaminate the shipped-code measurement.
*
@@ -23,7 +23,13 @@
*/
import { platform, arch } from "node:os";
import { launchStudioChrome } from "./chrome-executable.mjs";
-import { gatePassed, judgeResponsiveness } from "./timeline-viewport-verdict.mjs";
+import {
+ attemptPassed,
+ gatePassed,
+ judgeResponsiveness,
+ responsivenessLimits,
+ TIMING_ATTEMPTS,
+} from "./timeline-viewport-verdict.mjs";
const STUDIO_URL = process.env.STUDIO_URL;
const PROFILE = process.env.TIMELINE_PROFILE || "dense-short";
@@ -290,36 +296,61 @@ try {
);
}
- const runs = [];
- const interactionLimitMs =
- TIER === "primary" ? budgets.interactionP95Ms : budgets.constrainedInteractionP95Ms;
- const frameIntervalLimitMs =
- TIER === "primary" ? budgets.frameIntervalP95Ms : budgets.constrainedFrameIntervalP95Ms;
- for (let index = 0; index < budgets.warmupRuns + budgets.measuredRuns; index += 1) {
- const run = await collectRun(page);
- if (index >= budgets.warmupRuns) runs.push(run);
- }
+ const { interactionLimitMs, frameIntervalLimitMs } = responsivenessLimits(
+ budgets,
+ TIER,
+ ROW_VIRTUALIZATION,
+ );
// Latency, long tasks and memory are product promises and hold for both
// builds. The DOM-size budgets describe what windowing achieves, so they only
// apply when windowing is on. They are skipped explicitly rather than relaxed,
// so a skipped budget never reads as a passed one.
const domBudgetsApply = ROW_VIRTUALIZATION === "on";
- for (const run of runs) {
- run.longTaskPassed = run.longestTaskMs <= longTaskLimitMs;
- run.timelineMounted = run.diagnostics.timelineRoots === 1;
- run.domSizePassed = domBudgetsApply
- ? run.diagnostics.mountedRows <= budgets.maxMountedRows &&
- run.diagnostics.mountedClipRoots <= budgets.maxMountedClipRoots &&
- run.diagnostics.maxMountedClipRootsInOneRow <= budgets.maxMountedClipRootsPerRow &&
- run.diagnostics.mountedTimelineDescendants <= budgets.maxMountedTimelineDescendants
- : null;
- run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false;
+ const attempts = [];
+ while (attempts.length < TIMING_ATTEMPTS && !attempts.some((attempt) => attempt.passed)) {
+ const runs = [];
+ for (let index = 0; index < budgets.warmupRuns + budgets.measuredRuns; index += 1) {
+ const run = await collectRun(page);
+ if (index >= budgets.warmupRuns) runs.push(run);
+ }
+ for (const run of runs) {
+ run.longTaskPassed = run.longestTaskMs <= longTaskLimitMs;
+ run.timelineMounted = run.diagnostics.timelineRoots === 1;
+ run.domSizePassed = domBudgetsApply
+ ? run.diagnostics.mountedRows <= budgets.maxMountedRows &&
+ run.diagnostics.mountedClipRoots <= budgets.maxMountedClipRoots &&
+ run.diagnostics.maxMountedClipRootsInOneRow <= budgets.maxMountedClipRootsPerRow &&
+ run.diagnostics.mountedTimelineDescendants <= budgets.maxMountedTimelineDescendants
+ : null;
+ run.passed = run.longTaskPassed && run.timelineMounted && run.domSizePassed !== false;
+ }
+ const responsiveness = judgeResponsiveness(runs, {
+ samplesPerRun: budgets.scrollSamplesPerRun,
+ interactionLimitMs,
+ frameIntervalLimitMs,
+ });
+ const passingRuns = runs.filter((run) => run.passed).length;
+ const attempt = {
+ attempt: attempts.length + 1,
+ interactionP95Ms: responsiveness.interactionP95Ms,
+ frameIntervalP95Ms: responsiveness.frameIntervalP95Ms,
+ responsivenessPassed: responsiveness.passed,
+ passingRuns,
+ passed: attemptPassed({
+ responsivenessPassed: responsiveness.passed,
+ passingRuns,
+ requiredPassingRuns: budgets.requiredPassingRuns,
+ }),
+ runs,
+ };
+ attempts.push(attempt);
+ console.error(
+ `timeline gate ${ROW_VIRTUALIZATION} attempt ${attempt.attempt}: ` +
+ `interaction p95 ${attempt.interactionP95Ms.toFixed(1)}/${interactionLimitMs} ms, ` +
+ `frame p95 ${attempt.frameIntervalP95Ms.toFixed(1)}/${frameIntervalLimitMs} ms, ` +
+ `${passingRuns}/${runs.length} runs passed, ${attempt.passed ? "PASS" : "FAIL"}`,
+ );
}
- const responsiveness = judgeResponsiveness(runs, {
- samplesPerRun: budgets.scrollSamplesPerRun,
- interactionLimitMs,
- frameIntervalLimitMs,
- });
await page.evaluate(() => window.__studioTest.resetTimelinePerformanceFixture());
await page.waitForFunction(
@@ -331,8 +362,10 @@ try {
const returnedHeapBytes = await collectHeapBytes(client);
const memoryReturned =
returnedHeapBytes <= baselineHeapBytes * (1 + budgets.memoryReturnToleranceRatio);
- const passingRuns = runs.filter((run) => run.passed).length;
- const maxTimelineContentWidthPx = Math.max(0, ...runs.map((run) => run.scrollWidth));
+ const maxTimelineContentWidthPx = Math.max(
+ 0,
+ ...attempts.flatMap((attempt) => attempt.runs.map((run) => run.scrollWidth)),
+ );
const directScrollGate = {
safetyEnvelopePx: budgets.directScrollSafetyPx,
maxTimelineContentWidthPx,
@@ -369,15 +402,13 @@ try {
warmups: budgets.warmupRuns,
measured: budgets.measuredRuns,
requiredPassing: budgets.requiredPassingRuns,
+ timingAttempts: TIMING_ATTEMPTS,
},
},
directScrollGate,
- runs,
+ attempts,
aggregate: {
- interactionP95Ms: responsiveness.interactionP95Ms,
- frameIntervalP95Ms: responsiveness.frameIntervalP95Ms,
- responsivenessPassed: responsiveness.passed,
- passingRuns,
+ timingPassed: attempts.some((attempt) => attempt.passed),
baselineHeapBytes,
returnedHeapBytes,
memoryReturned,
@@ -386,9 +417,7 @@ try {
console.log(JSON.stringify(evidence, null, 2));
exitCode = gatePassed({
directScrollApproved: directScrollGate.decision === "approved",
- responsivenessPassed: responsiveness.passed,
- passingRuns,
- requiredPassingRuns: budgets.requiredPassingRuns,
+ attempts,
memoryReturned,
})
? 0