Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(cli): check skips WebGL draws on its box-only seeks so Three.js scenes finish (#5825)
Miguel Ángele9181d02f5f4
diff --git a/packages/cli/src/capture/captureCompositionFrame.test.ts b/packages/cli/src/capture/captureCompositionFrame.test.tsindex cd9a3f5ea9..7793f356bb 100644--- a/packages/cli/src/capture/captureCompositionFrame.test.ts+++ b/packages/cli/src/capture/captureCompositionFrame.test.ts@@ -107,7 +107,7 @@ describe("seekCompositionTimeline", () => {expect(waitForFunction).not.toHaveBeenCalled();expect(evaluate).toHaveBeenCalledTimes(4);- expect(evaluate).toHaveBeenNthCalledWith(1, expect.any(Function), 1.25, false, false);+ expect(evaluate).toHaveBeenNthCalledWith(1, expect.any(Function), 1.25, false, false, false);expect(evaluate).toHaveBeenNthCalledWith(2, expect.any(Function), { videoFrames: true });expect(evaluate.mock.calls[2]?.[0]).toContain("window.setTimeout(finish, 100)");// Post-seek font settle: a seek can reveal glyphs whose unicode-range@@ -212,7 +212,7 @@ describe("seekCompositionTimeline", () => {expect(waitForFunction).toHaveBeenCalledWith(expect.any(Function), { timeout: 500 });expect(evaluate).toHaveBeenCalledTimes(2);- expect(evaluate).toHaveBeenCalledWith(expect.any(Function), 3, true, false);+ expect(evaluate).toHaveBeenCalledWith(expect.any(Function), 3, true, false, false);});it("supports layout's ordered double-frame, bounded font, and sleep settles", async () => {@@ -229,7 +229,7 @@ describe("seekCompositionTimeline", () => {await pending;expect(evaluate).toHaveBeenCalledTimes(4);- expect(evaluate).toHaveBeenNthCalledWith(1, expect.any(Function), 4, true, false);+ expect(evaluate).toHaveBeenNthCalledWith(1, expect.any(Function), 4, true, false, false);expect(evaluate).toHaveBeenNthCalledWith(2, expect.any(Function), { videoFrames: true });expect(evaluate).toHaveBeenNthCalledWith(3, expect.any(Function));expect(evaluate).toHaveBeenNthCalledWith(4, expect.any(Function), 500);diff --git a/packages/cli/src/capture/captureCompositionFrame.ts b/packages/cli/src/capture/captureCompositionFrame.tsindex 7a1406004b..989fdb222d 100644--- a/packages/cli/src/capture/captureCompositionFrame.ts+++ b/packages/cli/src/capture/captureCompositionFrame.ts@@ -42,6 +42,7 @@ export const DENSE_GEOMETRY_SEEK_OPTIONS = {waitForFontsMs: 0,settleMs: 0,waitForVideoFrames: false,+ skipWebglDraws: true,} as const;export interface SeekCompositionTimelineOptions {@@ -52,6 +53,8 @@ export interface SeekCompositionTimelineOptions {waitForFontsMs?: number;settleMs?: number;waitForVideoFrames?: boolean;+ // Box-only reads: software GL spends about a second per frame on draws nothing here looks at.+ skipWebglDraws?: boolean;}type SeekBarrierOptions = { videoFrames: boolean };@@ -61,7 +64,12 @@ type CompositionPageFunction =| (() => unknown)| ((value: number) => unknown)| ((barrier: SeekBarrierOptions) => unknown)- | ((value: number, fallbackToBridgeAndTimelines: boolean, exactTime: boolean) => unknown);+ | ((+ value: number,+ fallbackToBridgeAndTimelines: boolean,+ exactTime: boolean,+ skipWebglDraws: boolean,+ ) => unknown);export interface CompositionEvaluationPage {evaluate(@@ -69,6 +77,7 @@ export interface CompositionEvaluationPage {value?: number | SeekBarrierOptions,fallbackToBridgeAndTimelines?: boolean,exactTime?: boolean,+ skipWebglDraws?: boolean,): Promise<unknown>;}@@ -297,7 +306,7 @@ export async function seekCompositionTimeline(await page.evaluate(// Serialized into the page; the seek-target cascade must stay one function.// fallow-ignore-next-line complexity- (t: number, fallbackToBridgeAndTimelines: boolean, exactTime: boolean) => {+ (t: number, fallbackToBridgeAndTimelines: boolean, exactTime: boolean, skipDraws: boolean) => {const getProperty = (target: unknown, key: string): unknown => {if ((typeof target !== "object" || target === null) && typeof target !== "function") {return undefined;@@ -318,33 +327,85 @@ export async function seekCompositionTimeline(const playerSeek = getProperty(player, "seek");const hf = Reflect.get(window, "__hf");const bridgeSeek = getProperty(hf, "seek");+ const gsap = Reflect.get(window, "gsap");+ const ticker = getProperty(gsap, "ticker");++ // fallow-ignore-next-line complexity+ const seekTo = (time: number): void => {+ // Prefer renderSeek because it also runs the runtime's data-start/data-duration+ // visibility sync; raw timeline seeks leave off-window clips visible to audits.+ if (call(renderSeek, player, exactTime ? [time, { exact: true }] : [time])) {+ // Preferred runtime target handled the seek.+ } else if (fallbackToBridgeAndTimelines && call(bridgeSeek, hf, [time])) {+ // Producer bridge handled the seek.+ } else if (call(playerSeek, player, [time])) {+ // Legacy player target handled the seek.+ } else if (fallbackToBridgeAndTimelines) {+ const timelines = Reflect.get(window, "__timelines");+ if (typeof timelines === "object" && timelines !== null) {+ for (const key of Object.keys(timelines)) {+ const timeline = Reflect.get(timelines, key);+ call(getProperty(timeline, "pause"), timeline, []);+ call(getProperty(timeline, "seek"), timeline, [time]);+ }+ }+ }+ call(getProperty(ticker, "tick"), ticker, []);+ };- // Prefer renderSeek because it also runs the runtime's data-start/data-duration- // visibility sync; raw timeline seeks leave off-window clips visible to audits.- if (call(renderSeek, player, exactTime ? [safe, { exact: true }] : [safe])) {- // Preferred runtime target handled the seek.- } else if (fallbackToBridgeAndTimelines && call(bridgeSeek, hf, [safe])) {- // Producer bridge handled the seek.- } else if (call(playerSeek, player, [safe])) {- // Legacy player target handled the seek.- } else if (fallbackToBridgeAndTimelines) {- const timelines = Reflect.get(window, "__timelines");- if (typeof timelines === "object" && timelines !== null) {- for (const key of Object.keys(timelines)) {- const timeline = Reflect.get(timelines, key);- call(getProperty(timeline, "pause"), timeline, []);- call(getProperty(timeline, "seek"), timeline, [safe]);+ const gateSlot = Symbol.for("hyperframes.webglDrawGate");+ let gate = Reflect.get(window, gateSlot) as { skip: boolean; skipped: boolean } | undefined;+ if (skipDraws && !gate) {+ const installed = { skip: false, skipped: false };+ for (const name of ["WebGLRenderingContext", "WebGL2RenderingContext"]) {+ const proto = getProperty(Reflect.get(window, name), "prototype");+ if (!proto) continue;+ for (const method of [+ "clear",+ "drawArrays",+ "drawElements",+ "drawArraysInstanced",+ "drawElementsInstanced",+ "drawRangeElements",+ "blitFramebuffer",+ ]) {+ const draw = getProperty(proto, method);+ if (typeof draw !== "function") continue;+ Reflect.set(proto, method, function (this: unknown, ...args: unknown[]) {+ if (!installed.skip) return Reflect.apply(draw, this, args);+ installed.skipped = true;+ return undefined;+ });}}+ Reflect.set(window, gateSlot, installed);+ gate = installed;}+ const seekWithoutDraws = (time: number): void => {+ if (!gate) return seekTo(time);+ gate.skip = true;+ try {+ seekTo(time);+ } finally {+ gate.skip = false;+ }+ };- const gsap = Reflect.get(window, "gsap");- const ticker = getProperty(gsap, "ticker");- call(getProperty(ticker, "tick"), ticker, []);+ if (skipDraws) {+ seekWithoutDraws(safe);+ return;+ }+ // A skipped draw left the canvas stale and a same-time seek redraws nothing: step off first.+ if (gate?.skipped) {+ seekWithoutDraws(safe >= 0.001 ? safe - 0.001 : safe + 0.001);+ gate.skipped = false;+ }+ seekTo(safe);},timeSeconds,options.fallbackToBridgeAndTimelines === true,options.exactTime === true,+ options.skipWebglDraws === true,);await page.evaluate(diff --git a/packages/cli/src/capture/captureCompositionFrame.webglDraws.browser.test.ts b/packages/cli/src/capture/captureCompositionFrame.webglDraws.browser.test.tsnew file mode 100644index 0000000000..dfae17194d--- /dev/null+++ b/packages/cli/src/capture/captureCompositionFrame.webglDraws.browser.test.ts@@ -0,0 +1,75 @@+import puppeteer, { type Browser, type Page } from "puppeteer-core";+import { afterAll, beforeAll, expect, it } from "vitest";+import { ensureBrowser } from "../browser/manager.js";+import { DENSE_GEOMETRY_SEEK_OPTIONS, seekCompositionTimeline } from "./captureCompositionFrame.js";++// Red channel = time * 25. Like GSAP, seeking to the current time renders nothing.+const PAGE = `<canvas id="c" width="4" height="4"></canvas>+<script>+ const gl = document.getElementById("c").getContext("webgl", { preserveDrawingBuffer: true });+ const paint = (t) => {+ gl.clearColor((t * 25) / 255, 0, 0, 1);+ gl.clear(gl.COLOR_BUFFER_BIT);+ };+ let now = 0;+ window.seeks = [];+ window.__player = {+ renderSeek: (t) => {+ window.seeks.push(t);+ if (t === now) return;+ now = t;+ paint(t);+ },+ };+ window.paint = paint;+ window.red = () => {+ const px = new Uint8Array(4);+ gl.readPixels(0, 0, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, px);+ return px[0];+ };+ paint(0);+</script>`;++const PIXEL_SEEK = { animationFrameSettle: "none", waitForFontsMs: 0 } as const;++let browser: Browser;+let page: Page;++beforeAll(async () => {+ const { executablePath } = await ensureBrowser();+ browser = await puppeteer.launch({+ executablePath,+ headless: true,+ args: [+ "--no-sandbox",+ "--use-gl=angle",+ "--use-angle=swiftshader",+ "--enable-unsafe-swiftshader",+ ],+ });+ page = await browser.newPage();+ await page.setContent(PAGE);+}, 60_000);++afterAll(async () => {+ await browser?.close();+});++const red = () => page.evaluate(() => Reflect.apply(Reflect.get(window, "red"), window, []));+const seeks = () => page.evaluate(() => [...Reflect.get(window, "seeks")]);++it("skips WebGL draws on geometry seeks and repaints on the next pixel seek to the same time", async () => {+ await seekCompositionTimeline(page, 4, DENSE_GEOMETRY_SEEK_OPTIONS);+ expect(await red()).toBe(0);++ await seekCompositionTimeline(page, 4, PIXEL_SEEK);+ expect(await red()).toBe(100);+ expect(await seeks()).toEqual([4, 3.999, 4]);++ await seekCompositionTimeline(page, 6, PIXEL_SEEK);+ expect(await red()).toBe(150);+ expect(await seeks()).toEqual([4, 3.999, 4, 6]);++ await page.evaluate(() => Reflect.apply(Reflect.get(window, "paint"), window, [2]));+ expect(await red()).toBe(50);+});