Skip to content

Write HTML. Render video. Built for agents.

fix(engine): render workers stop loading the videos whose frames come from FFmpeg (#5795)

Miguel Ángel6ed89a1e4c93
diff --git a/packages/engine/src/services/__fixtures__/loopback.ts b/packages/engine/src/services/__fixtures__/loopback.ts
new file mode 100644
index 0000000000..9149eff4c8
--- /dev/null
+++ b/packages/engine/src/services/__fixtures__/loopback.ts
@@ -0,0 +1,18 @@
+import type { Server } from "node:http";
+import puppeteer, { type Browser } from "puppeteer-core";
+
+/** Listens on a free loopback port and returns the server's origin. */
+export async function listenOnLoopback(server: Server): Promise<string> {
+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+ const address = server.address();
+ if (!address || typeof address === "string") throw new Error("No test server address");
+ return `http://127.0.0.1:${address.port}`;
+}
+
+export function launchTestChrome(executablePath: string | undefined): Promise<Browser> {
+ return puppeteer.launch({
+ executablePath,
+ headless: true,
+ args: ["--no-sandbox", "--disable-dev-shm-usage"],
+ });
+}
diff --git a/packages/engine/src/services/frameCapture-releaseInjectedVideoMedia.browser.test.ts b/packages/engine/src/services/frameCapture-releaseInjectedVideoMedia.browser.test.ts
new file mode 100644
index 0000000000..f7659b19c4
--- /dev/null
+++ b/packages/engine/src/services/frameCapture-releaseInjectedVideoMedia.browser.test.ts
@@ -0,0 +1,91 @@
+import { spawnSync } from "node:child_process";
+import { createServer, type ServerResponse } from "node:http";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import type { Browser } from "puppeteer-core";
+import { resolveHeadlessShellPath } from "./browserManager.js";
+import {
+ applyVideoMetadataHints,
+ releasableVideoIds,
+ releaseInjectedVideoMedia,
+} from "./frameCapture.js";
+import { listenOnLoopback, launchTestChrome } from "./__fixtures__/loopback.js";
+
+const chromePath = resolveHeadlessShellPath();
+const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
+
+describe("releasableVideoIds", () => {
+ it("keeps the media of a skipped video no poster sizes, such as a native HDR one", () => {
+ expect(
+ releasableVideoIds({
+ skipReadinessVideoIds: ["hdr", "sdr"],
+ videoMetadataHints: [{ id: "sdr", width: 640, height: 360 }],
+ } as Parameters<typeof releasableVideoIds>[0]),
+ ).toEqual(["sdr"]);
+ });
+});
+
+describe.skipIf(!hasBrowser)("releaseInjectedVideoMedia in Chrome", () => {
+ const openMedia = new Set<ServerResponse>();
+ let origin = "";
+ let browser: Browser;
+ const server = createServer((req, res) => {
+ if (req.url?.startsWith("/media/")) {
+ res.writeHead(200, { "Content-Type": "video/mp4" });
+ res.write(Buffer.alloc(1024));
+ openMedia.add(res);
+ res.on("close", () => openMedia.delete(res));
+ return;
+ }
+ const videos = ["a", "b", "c", "live"].map(
+ (id) =>
+ `<video id="${id}" preload="auto" src="/media/${id}.mp4" style="width:200px"></video>`,
+ );
+ res.writeHead(200, { "Content-Type": "text/html" });
+ res.end(`<html><body style="margin:0">${videos.join("")}</body></html>`);
+ });
+
+ beforeAll(async () => {
+ origin = await listenOnLoopback(server);
+ browser = await launchTestChrome(chromePath);
+ });
+ afterAll(async () => {
+ for (const res of openMedia) res.destroy();
+ await browser?.close();
+ await new Promise<void>((resolve) => server.close(() => resolve()));
+ });
+
+ it("drops the media of videos whose frames come from FFmpeg and keeps their layout", async () => {
+ const page = await browser.newPage();
+ await page.goto(`${origin}/`);
+ const deadline = Date.now() + 10_000;
+ while (openMedia.size < 4 && Date.now() < deadline) {
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ expect(openMedia.size).toBe(4);
+ const injected = ["a", "b", "c"];
+ await applyVideoMetadataHints(
+ page,
+ injected.map((id) => ({ id, width: 640, height: 360 })),
+ );
+
+ await releaseInjectedVideoMedia(page, injected);
+
+ const released = await page.evaluate((ids) => {
+ return ids.map((id) => {
+ const video = document.getElementById(id) as HTMLVideoElement;
+ const box = video.getBoundingClientRect();
+ return `${id}:${video.networkState}:${box.width}x${box.height}`;
+ });
+ }, injected);
+ expect(released).toEqual(["a:0:200x112.5", "b:0:200x112.5", "c:0:200x112.5"]);
+ const releaseDeadline = Date.now() + 10_000;
+ while (openMedia.size > 1 && Date.now() < releaseDeadline) {
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ expect(openMedia.size).toBe(1);
+ expect(await page.$eval("#live", (video) => (video as HTMLVideoElement).currentSrc)).toBe(
+ `${origin}/media/live.mp4`,
+ );
+ await page.close();
+ }, 30_000);
+});
diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts
index 8c24bfe849..99b0aef478 100644
--- a/packages/engine/src/services/frameCapture.ts
+++ b/packages/engine/src/services/frameCapture.ts
@@ -2217,6 +2217,33 @@ async function decodeAllImages(page: Page): Promise<void> {
});
}
+/** Out-of-band videos a poster now sizes, so dropping their media cannot move layout. */
+export function releasableVideoIds(options: CaptureOptions): string[] {
+ const hinted = new Set((options.videoMetadataHints ?? []).map((hint) => hint.id));
+ return (options.skipReadinessVideoIds ?? []).filter((id) => hinted.has(id));
+}
+
+/** FFmpeg supplies these videos' frames, so their own players would only hold decoders and buffers in every worker. */
+export async function releaseInjectedVideoMedia(
+ page: Page,
+ videoIds: readonly string[],
+): Promise<void> {
+ if (videoIds.length === 0) return;
+ await page.evaluate(
+ (ids: string[]) => {
+ for (const id of ids) {
+ const video = window.__hfMediaEl?.(id) ?? document.getElementById(id);
+ if (!(video instanceof HTMLVideoElement)) continue;
+ video.pause();
+ for (const source of video.querySelectorAll("source")) source.remove();
+ video.removeAttribute("src");
+ video.load();
+ }
+ },
+ [...videoIds],
+ );
+}
+
export async function applyVideoMetadataHints(
page: Page,
hints: readonly CaptureVideoMetadataHint[] | undefined,
@@ -2505,6 +2532,7 @@ export async function initializeSession(session: CaptureSession): Promise<void>
recordSubTimelineWarning(session, pageReadyTimeout);
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
+ await releaseInjectedVideoMedia(page, releasableVideoIds(session.options));
logInitPhase("applyVideoMetadataHints complete");
// Run independent readiness checks in parallel — videos, images, fonts,
@@ -2661,6 +2689,7 @@ export async function initializeSession(session: CaptureSession): Promise<void>
recordSubTimelineWarning(session, pageReadyTimeout);
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
+ await releaseInjectedVideoMedia(page, releasableVideoIds(session.options));
logInitPhase("applyVideoMetadataHints complete");
// Run independent readiness checks in parallel — videos, images, fonts,
@@ -3455,6 +3484,7 @@ export async function createStaticVerificationPage(session: CaptureSession): Pro
knownUnregisteredIds: session.subTimelineWaitMemo.unregisteredIds,
});
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
+ await releaseInjectedVideoMedia(page, releasableVideoIds(session.options));
const skipVideoIds = session.options.skipReadinessVideoIds ?? [];
await Promise.all([
diff --git a/packages/engine/src/services/parallelWorkerBrowsers.browser.test.ts b/packages/engine/src/services/parallelWorkerBrowsers.browser.test.ts
index 9fcc287e76..6d4fe0bd7a 100644
--- a/packages/engine/src/services/parallelWorkerBrowsers.browser.test.ts
+++ b/packages/engine/src/services/parallelWorkerBrowsers.browser.test.ts
@@ -3,6 +3,7 @@ import { createServer, type ServerResponse } from "node:http";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { acquireBrowser, drainBrowserPool, resolveHeadlessShellPath } from "./browserManager.js";
import { shouldDisableBrowserPoolForParallelWorker } from "./parallelCoordinator.js";
+import { listenOnLoopback } from "./__fixtures__/loopback.js";
const chromePath = resolveHeadlessShellPath();
const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
@@ -35,10 +36,7 @@ describe.skipIf(!hasBrowser)("parallel worker browsers", () => {
});
beforeAll(async () => {
- await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
- const address = server.address();
- if (!address || typeof address === "string") throw new Error("No test server address");
- origin = `http://127.0.0.1:${address.port}`;
+ origin = await listenOnLoopback(server);
});
afterAll(async () => {
diff --git a/packages/engine/src/services/videoFrameTransport.browser.test.ts b/packages/engine/src/services/videoFrameTransport.browser.test.ts
index 398b8f3587..0eeaa7aa7d 100644
--- a/packages/engine/src/services/videoFrameTransport.browser.test.ts
+++ b/packages/engine/src/services/videoFrameTransport.browser.test.ts
@@ -1,9 +1,10 @@
import { spawnSync } from "node:child_process";
import { createServer } from "node:http";
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
-import puppeteer, { type Browser, type Page } from "puppeteer-core";
+import type { Browser, Page } from "puppeteer-core";
import { resolveHeadlessShellPath } from "./browserManager.js";
import { injectVideoFramesBatch } from "./screenshotService.js";
+import { listenOnLoopback, launchTestChrome } from "./__fixtures__/loopback.js";
const chromePath = resolveHeadlessShellPath();
const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
@@ -30,15 +31,8 @@ describe.skipIf(!hasBrowser)("video frame transport in Chrome", () => {
});
beforeAll(async () => {
- await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
- const address = server.address();
- if (!address || typeof address === "string") throw new Error("No test server address");
- origin = `http://127.0.0.1:${address.port}`;
- browser = await puppeteer.launch({
- executablePath: chromePath,
- headless: true,
- args: ["--no-sandbox", "--disable-dev-shm-usage"],
- });
+ origin = await listenOnLoopback(server);
+ browser = await launchTestChrome(chromePath);
});
beforeEach(async () => {
if (page) await page.close();