Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(producer): cloud render chunks start their capture browsers while video frames extract (#6031)
Miguel Ángel0716670faf68
diff --git a/packages/producer/src/services/distributed/chunkFrames.ts b/packages/producer/src/services/distributed/chunkFrames.tsindex 82ef686af1..4a5e5e6b34 100644--- a/packages/producer/src/services/distributed/chunkFrames.ts+++ b/packages/producer/src/services/distributed/chunkFrames.ts@@ -54,9 +54,16 @@ export class RenderChunkValidationError extends Error {/** One injector per browser session: a hook reused on a fresh page can skip its first injection. */export function createChunkVideoFrameInjectorFactory(- frameLookup: FrameLookupTable | null,+ frameLookup: FrameLookupTable | Promise<FrameLookupTable> | null,): () => BeforeCaptureHook | null {- return () => createVideoFrameInjector(frameLookup);+ if (!(frameLookup instanceof Promise)) return () => createVideoFrameInjector(frameLookup);+ return () => {+ let hook: BeforeCaptureHook | null | undefined;+ return async (page, time, heldVideoTime) => {+ hook ??= createVideoFrameInjector(await frameLookup);+ await hook?.(page, time, heldVideoTime);+ };+ };}function frameNumberFromFileName(name: string): number | null {diff --git a/packages/producer/src/services/distributed/renderChunk.test.ts b/packages/producer/src/services/distributed/renderChunk.test.tsindex 0404923c64..1e22cb10f2 100644--- a/packages/producer/src/services/distributed/renderChunk.test.ts+++ b/packages/producer/src/services/distributed/renderChunk.test.ts@@ -38,7 +38,11 @@ import { createServer } from "node:http";import type { AddressInfo } from "node:net";import { tmpdir } from "node:os";import { join } from "node:path";-import { type CaptureOptions, extractVideoFramesRange } from "@hyperframes/engine";+import {+ type CaptureOptions,+ extractVideoFramesRange,+ VideoSourceExtractionError,+} from "@hyperframes/engine";import { HOST_CHROME_FAILURE_PATTERNS } from "./__test_utils__/hostChromeFailures.js";import { defaultLogger } from "../../logger.js";import { RenderQualityError } from "../renderOrchestrator.js";@@ -733,6 +737,61 @@ describe("renderChunk() — a video shown smaller than its source", () => {}, 120_000);});+describe("renderChunk() — a video whose frames cannot be extracted", () => {+ it("fails with the extraction's own error while the capture browsers start", async () => {+ if (!hasChrome) {+ console.warn(+ "[renderChunk.test] skipping extraction-failure test — chrome-headless-shell not available",+ );+ return;+ }+ const projectDir = join(runRoot, "project-extract-fail");+ mkdirSync(projectDir, { recursive: true });+ const synth = spawnSync("ffmpeg", [+ ...["-v", "error", "-y", "-f", "lavfi", "-i", "testsrc2=s=128x72:r=30:d=2"],+ ...["-c:v", "libx264", "-pix_fmt", "yuv420p", join(projectDir, "clip.mp4")],+ ]);+ if (synth.status !== 0) throw new Error(synth.stderr.toString());+ writeFileSync(+ join(projectDir, "index.html"),+ // A clip over a second long, so the planner defers it to the chunks.+ FIXTURE_HTML.replace('data-duration="0.16667"', 'data-duration="1.2"').replace(+ "</div>",+ '<video id="clip" src="clip.mp4" muted data-start="0" data-duration="1.2" style="width:128px;height:72px"></video></div>',+ ),+ "utf-8",+ );+ const planDir = join(runRoot, "plan-extract-fail");+ await planV2(+ projectDir,+ { fps: 30, width: 160, height: 120, format: "png-sequence", chunkSize: 36 },+ planDir,+ );+ const chunkDir = join(runRoot, "chunk0-extract-fail");+ // The clip is read by URL, as cloud chunks read it, from a port nothing listens on.+ materializePlanV2Target(planDir, { role: "chunk", chunkIndex: 0 }, chunkDir, {+ remoteSources: { "compiled/clip.mp4": "http://127.0.0.1:1/clip.mp4" },+ });++ let caught: unknown;+ try {+ await renderChunk(chunkDir, 0, join(runRoot, "chunk0-extract-fail-out"));+ } catch (error) {+ caught = error;+ }+ const message = caught instanceof Error ? caught.message : String(caught);+ if (HOST_CHROME_FAILURE_PATTERNS.test(message)) {+ console.warn(+ "[renderChunk.test] skipping extraction-failure test — host Chrome stack can't render:",+ message.slice(0, 240),+ );+ return;+ }+ // The extraction's own error, not the capture stage that waited on it.+ expect(caught).toBeInstanceOf(VideoSourceExtractionError);+ }, 120_000);+});+describe("renderChunk() — frameRange", () => {// A box sliding across the frame, so every frame differs and a misplaced,// dropped or repeated frame changes the assembled output.diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.tsindex 201ebaffc6..b0c82d2b39 100644--- a/packages/producer/src/services/distributed/renderChunk.ts+++ b/packages/producer/src/services/distributed/renderChunk.ts@@ -723,6 +723,7 @@ export async function renderChunk(const captureAttempts: Parameters<typeof runCaptureStage>[0]["captureAttempts"] = [];let forceScreenshotForChunk = encoder.forceScreenshot;let shownBoxes: Map<string, ShownBox> | undefined;+ let extraction: Promise<unknown> | undefined;try {if (chunkWorkerCount === 1 || !forceScreenshotForChunk) {// Sequential capture reuses this session. Parallel BeginFrame capture@@ -801,7 +802,7 @@ export async function renderChunk(// per video; sharing that state across a fresh retry page can suppress the// first injection and produce a blank/stale frame.const deferredVideoFrames = planVideos- ? await extractDeferredVideoFramesForChunk({+ ? extractDeferredVideoFramesForChunk({planDir,planVideos,frames: chunkFrameNumbers,@@ -811,22 +812,27 @@ export async function renderChunk(remoteSources,shownBoxes,})- : [];+ : Promise.resolve([]);+ extraction = deferredVideoFrames;const videoFrameLookup =planVideos && planVideos.extracted.length > 0- ? createFrameLookupTable(- planVideos.videos,- [- ...rebuildExtractedFramesFromPlanDir(- planDir,- planVideos.extracted.filter((video) => !video.deferredRange),- v2Manifest === null ? "dense-v1" : "sparse-v2",- ),- ...deferredVideoFrames,- ],- resolveRenderFpsConfig(job.config.fps).value,+ ? deferredVideoFrames.then((deferred) =>+ createFrameLookupTable(+ planVideos.videos,+ [+ ...rebuildExtractedFramesFromPlanDir(+ planDir,+ planVideos.extracted.filter((video) => !video.deferredRange),+ v2Manifest === null ? "dense-v1" : "sparse-v2",+ ),+ ...deferred,+ ],+ resolveRenderFpsConfig(job.config.fps).value,+ ),): null;+ // A capture can fail before any session reads it; the failure is rethrown below.+ videoFrameLookup?.catch(() => {});const createChunkVideoFrameInjector = createChunkVideoFrameInjectorFactory(videoFrameLookup);const captureStarted = Date.now();@@ -920,7 +926,11 @@ export async function renderChunk(},);},+ }).catch(async (captureError: unknown) => {+ await deferredVideoFrames;+ throw captureError;});+ await deferredVideoFrames;captureStageMs = Date.now() - captureStarted;applyRenderWarningPolicy(job,@@ -1018,6 +1028,8 @@ export async function renderChunk(}encodeStageMs = Date.now() - encodeStarted;} finally {+ // No ffmpeg left writing into a work dir the caller removes; its error surfaced above.+ await extraction?.catch(() => {});// Cleanest path: captureStage closed the session for us. The defensive// close handles error paths where we threw before delegating.if (session) {diff --git a/packages/producer/src/services/distributed/renderChunkFallback.test.ts b/packages/producer/src/services/distributed/renderChunkFallback.test.tsindex 71e1d89d3a..816f9e3ce9 100644--- a/packages/producer/src/services/distributed/renderChunkFallback.test.ts+++ b/packages/producer/src/services/distributed/renderChunkFallback.test.ts@@ -124,6 +124,26 @@ describe("distributed session isolation", () => {expect(screenshotRetry).not.toBeNull();expect(screenshotRetry).not.toBe(firstAttempt);});++ it("holds a session's first injection until its frames are extracted", async () => {+ let failExtraction: (error: Error) => void = () => {};+ const lookup = new Promise<ReturnType<typeof createFrameLookupTable>>((_, reject) => {+ failExtraction = reject;+ });+ const factory = createChunkVideoFrameInjectorFactory(lookup);+ const hook = factory();+ expect(hook).not.toBeNull();+ expect(factory()).not.toBe(hook);++ let settled = false;+ const injection = hook!({} as Parameters<NonNullable<typeof hook>>[0], 0).finally(() => {+ settled = true;+ });+ await Promise.resolve();+ expect(settled).toBe(false);+ failExtraction(new Error("ffmpeg could not read the clip"));+ await expect(injection).rejects.toThrow("ffmpeg could not read the clip");+ });});describe("shouldRetryChunkCaptureWithScreenshot", () => {