Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): render workers stop loading the videos whose frames come from FFmpeg (#5795)
Miguel Ángel6ed89a1e4c93
diff --git a/packages/engine/src/services/__fixtures__/loopback.ts b/packages/engine/src/services/__fixtures__/loopback.tsnew file mode 100644index 0000000000..9149eff4c8--- /dev/null+++ b/packages/engine/src/services/__fixtures__/loopback.ts@@ -0,0 +1,18 @@+import type { Server } from "node:http";+import puppeteer, { type Browser } from "puppeteer-core";++/** Listens on a free loopback port and returns the server's origin. */+export async function listenOnLoopback(server: Server): Promise<string> {+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));+ const address = server.address();+ if (!address || typeof address === "string") throw new Error("No test server address");+ return `http://127.0.0.1:${address.port}`;+}++export function launchTestChrome(executablePath: string | undefined): Promise<Browser> {+ return puppeteer.launch({+ executablePath,+ headless: true,+ args: ["--no-sandbox", "--disable-dev-shm-usage"],+ });+}diff --git a/packages/engine/src/services/frameCapture-releaseInjectedVideoMedia.browser.test.ts b/packages/engine/src/services/frameCapture-releaseInjectedVideoMedia.browser.test.tsnew file mode 100644index 0000000000..f7659b19c4--- /dev/null+++ b/packages/engine/src/services/frameCapture-releaseInjectedVideoMedia.browser.test.ts@@ -0,0 +1,91 @@+import { spawnSync } from "node:child_process";+import { createServer, type ServerResponse } from "node:http";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import type { Browser } from "puppeteer-core";+import { resolveHeadlessShellPath } from "./browserManager.js";+import {+ applyVideoMetadataHints,+ releasableVideoIds,+ releaseInjectedVideoMedia,+} from "./frameCapture.js";+import { listenOnLoopback, launchTestChrome } from "./__fixtures__/loopback.js";++const chromePath = resolveHeadlessShellPath();+const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;++describe("releasableVideoIds", () => {+ it("keeps the media of a skipped video no poster sizes, such as a native HDR one", () => {+ expect(+ releasableVideoIds({+ skipReadinessVideoIds: ["hdr", "sdr"],+ videoMetadataHints: [{ id: "sdr", width: 640, height: 360 }],+ } as Parameters<typeof releasableVideoIds>[0]),+ ).toEqual(["sdr"]);+ });+});++describe.skipIf(!hasBrowser)("releaseInjectedVideoMedia in Chrome", () => {+ const openMedia = new Set<ServerResponse>();+ let origin = "";+ let browser: Browser;+ const server = createServer((req, res) => {+ if (req.url?.startsWith("/media/")) {+ res.writeHead(200, { "Content-Type": "video/mp4" });+ res.write(Buffer.alloc(1024));+ openMedia.add(res);+ res.on("close", () => openMedia.delete(res));+ return;+ }+ const videos = ["a", "b", "c", "live"].map(+ (id) =>+ `<video id="${id}" preload="auto" src="/media/${id}.mp4" style="width:200px"></video>`,+ );+ res.writeHead(200, { "Content-Type": "text/html" });+ res.end(`<html><body style="margin:0">${videos.join("")}</body></html>`);+ });++ beforeAll(async () => {+ origin = await listenOnLoopback(server);+ browser = await launchTestChrome(chromePath);+ });+ afterAll(async () => {+ for (const res of openMedia) res.destroy();+ await browser?.close();+ await new Promise<void>((resolve) => server.close(() => resolve()));+ });++ it("drops the media of videos whose frames come from FFmpeg and keeps their layout", async () => {+ const page = await browser.newPage();+ await page.goto(`${origin}/`);+ const deadline = Date.now() + 10_000;+ while (openMedia.size < 4 && Date.now() < deadline) {+ await new Promise((resolve) => setTimeout(resolve, 50));+ }+ expect(openMedia.size).toBe(4);+ const injected = ["a", "b", "c"];+ await applyVideoMetadataHints(+ page,+ injected.map((id) => ({ id, width: 640, height: 360 })),+ );++ await releaseInjectedVideoMedia(page, injected);++ const released = await page.evaluate((ids) => {+ return ids.map((id) => {+ const video = document.getElementById(id) as HTMLVideoElement;+ const box = video.getBoundingClientRect();+ return `${id}:${video.networkState}:${box.width}x${box.height}`;+ });+ }, injected);+ expect(released).toEqual(["a:0:200x112.5", "b:0:200x112.5", "c:0:200x112.5"]);+ const releaseDeadline = Date.now() + 10_000;+ while (openMedia.size > 1 && Date.now() < releaseDeadline) {+ await new Promise((resolve) => setTimeout(resolve, 50));+ }+ expect(openMedia.size).toBe(1);+ expect(await page.$eval("#live", (video) => (video as HTMLVideoElement).currentSrc)).toBe(+ `${origin}/media/live.mp4`,+ );+ await page.close();+ }, 30_000);+});diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.tsindex 8c24bfe849..99b0aef478 100644--- a/packages/engine/src/services/frameCapture.ts+++ b/packages/engine/src/services/frameCapture.ts@@ -2217,6 +2217,33 @@ async function decodeAllImages(page: Page): Promise<void> {});}+/** Out-of-band videos a poster now sizes, so dropping their media cannot move layout. */+export function releasableVideoIds(options: CaptureOptions): string[] {+ const hinted = new Set((options.videoMetadataHints ?? []).map((hint) => hint.id));+ return (options.skipReadinessVideoIds ?? []).filter((id) => hinted.has(id));+}++/** FFmpeg supplies these videos' frames, so their own players would only hold decoders and buffers in every worker. */+export async function releaseInjectedVideoMedia(+ page: Page,+ videoIds: readonly string[],+): Promise<void> {+ if (videoIds.length === 0) return;+ await page.evaluate(+ (ids: string[]) => {+ for (const id of ids) {+ const video = window.__hfMediaEl?.(id) ?? document.getElementById(id);+ if (!(video instanceof HTMLVideoElement)) continue;+ video.pause();+ for (const source of video.querySelectorAll("source")) source.remove();+ video.removeAttribute("src");+ video.load();+ }+ },+ [...videoIds],+ );+}+export async function applyVideoMetadataHints(page: Page,hints: readonly CaptureVideoMetadataHint[] | undefined,@@ -2505,6 +2532,7 @@ export async function initializeSession(session: CaptureSession): Promise<void>recordSubTimelineWarning(session, pageReadyTimeout);await applyVideoMetadataHints(page, session.options.videoMetadataHints);+ await releaseInjectedVideoMedia(page, releasableVideoIds(session.options));logInitPhase("applyVideoMetadataHints complete");// Run independent readiness checks in parallel — videos, images, fonts,@@ -2661,6 +2689,7 @@ export async function initializeSession(session: CaptureSession): Promise<void>recordSubTimelineWarning(session, pageReadyTimeout);await applyVideoMetadataHints(page, session.options.videoMetadataHints);+ await releaseInjectedVideoMedia(page, releasableVideoIds(session.options));logInitPhase("applyVideoMetadataHints complete");// Run independent readiness checks in parallel — videos, images, fonts,@@ -3455,6 +3484,7 @@ export async function createStaticVerificationPage(session: CaptureSession): ProknownUnregisteredIds: session.subTimelineWaitMemo.unregisteredIds,});await applyVideoMetadataHints(page, session.options.videoMetadataHints);+ await releaseInjectedVideoMedia(page, releasableVideoIds(session.options));const skipVideoIds = session.options.skipReadinessVideoIds ?? [];await Promise.all([diff --git a/packages/engine/src/services/parallelWorkerBrowsers.browser.test.ts b/packages/engine/src/services/parallelWorkerBrowsers.browser.test.tsindex 9fcc287e76..6d4fe0bd7a 100644--- a/packages/engine/src/services/parallelWorkerBrowsers.browser.test.ts+++ b/packages/engine/src/services/parallelWorkerBrowsers.browser.test.ts@@ -3,6 +3,7 @@ import { createServer, type ServerResponse } from "node:http";import { afterAll, beforeAll, describe, expect, it } from "vitest";import { acquireBrowser, drainBrowserPool, resolveHeadlessShellPath } from "./browserManager.js";import { shouldDisableBrowserPoolForParallelWorker } from "./parallelCoordinator.js";+import { listenOnLoopback } from "./__fixtures__/loopback.js";const chromePath = resolveHeadlessShellPath();const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;@@ -35,10 +36,7 @@ describe.skipIf(!hasBrowser)("parallel worker browsers", () => {});beforeAll(async () => {- await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));- const address = server.address();- if (!address || typeof address === "string") throw new Error("No test server address");- origin = `http://127.0.0.1:${address.port}`;+ origin = await listenOnLoopback(server);});afterAll(async () => {diff --git a/packages/engine/src/services/videoFrameTransport.browser.test.ts b/packages/engine/src/services/videoFrameTransport.browser.test.tsindex 398b8f3587..0eeaa7aa7d 100644--- a/packages/engine/src/services/videoFrameTransport.browser.test.ts+++ b/packages/engine/src/services/videoFrameTransport.browser.test.ts@@ -1,9 +1,10 @@import { spawnSync } from "node:child_process";import { createServer } from "node:http";import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";-import puppeteer, { type Browser, type Page } from "puppeteer-core";+import type { Browser, Page } from "puppeteer-core";import { resolveHeadlessShellPath } from "./browserManager.js";import { injectVideoFramesBatch } from "./screenshotService.js";+import { listenOnLoopback, launchTestChrome } from "./__fixtures__/loopback.js";const chromePath = resolveHeadlessShellPath();const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;@@ -30,15 +31,8 @@ describe.skipIf(!hasBrowser)("video frame transport in Chrome", () => {});beforeAll(async () => {- await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));- const address = server.address();- if (!address || typeof address === "string") throw new Error("No test server address");- origin = `http://127.0.0.1:${address.port}`;- browser = await puppeteer.launch({- executablePath: chromePath,- headless: true,- args: ["--no-sandbox", "--disable-dev-shm-usage"],- });+ origin = await listenOnLoopback(server);+ browser = await launchTestChrome(chromePath);});beforeEach(async () => {if (page) await page.close();