Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(producer): graded video renders split into chunks that finish inside the worker limit (#5883)
Miguel Ángel6c3cc7cbe91b
diff --git a/packages/producer/src/services/distributed/chunkFrames.ts b/packages/producer/src/services/distributed/chunkFrames.tsnew file mode 100644index 0000000000..440f164f90--- /dev/null+++ b/packages/producer/src/services/distributed/chunkFrames.ts@@ -0,0 +1,230 @@+// What a chunk capture session is built from, shared with the planner's frame-cost measurement.+import { existsSync, readdirSync } from "node:fs";+import { extname, join } from "node:path";+import {+ type BeforeCaptureHook,+ BROWSER_GPU_NOT_SOFTWARE,+ compositionRequiresWebGpu,+ type CaptureOptions,+ createFrameLookupTable,+ createVideoFrameInjector,+ type EngineConfig,+ type ExtractedFrames,+ extractVideoFramesRange,+ type FrameRange,+ type FrameLookupTable,+} from "@hyperframes/engine";+import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";+import { resolveCaptureJpegQuality } from "../render/captureImageFormat.js";+import type { LockedRenderConfig } from "../render/stages/freezePlan.js";+import { resolveRenderFpsConfig } from "../fileServer.js";+import { type DistributedFormat, INVALID_VIDEO_METADATA, type PlanVideosJson } from "./shared.js";++/** Distinct codes so adapters route retries per cause: a missing artifact may heal, a hash mismatch never. */+export const FFMPEG_VERSION_MISMATCH = "FFMPEG_VERSION_MISMATCH";+export const PLAN_HASH_MISMATCH = "PLAN_HASH_MISMATCH";+export const MISSING_PLAN_ARTIFACT = "MISSING_PLAN_ARTIFACT";+export const CHUNK_INDEX_OUT_OF_RANGE = "CHUNK_INDEX_OUT_OF_RANGE";+export const MISSING_RUNTIME_ENV_SNAPSHOT = "MISSING_RUNTIME_ENV_SNAPSHOT";+export { INVALID_VIDEO_METADATA };++export type RenderChunkValidationCode =+ | typeof FFMPEG_VERSION_MISMATCH+ | typeof PLAN_HASH_MISMATCH+ | typeof MISSING_PLAN_ARTIFACT+ | typeof CHUNK_INDEX_OUT_OF_RANGE+ | typeof MISSING_RUNTIME_ENV_SNAPSHOT+ | typeof INVALID_VIDEO_METADATA+ | typeof BROWSER_GPU_NOT_SOFTWARE;++/** A malformed planDir or a worker that does not match the plan's fingerprint; adapters key retries off `code`. */+export class RenderChunkValidationError extends Error {+ readonly code: RenderChunkValidationCode;+ constructor(code: RenderChunkValidationCode, message: string) {+ super(message);+ this.name = "RenderChunkValidationError";+ this.code = code;+ }+}++/** One injector per browser session: a hook reused on a fresh page can skip its first injection. */+export function createChunkVideoFrameInjectorFactory(+ frameLookup: FrameLookupTable | null,+): () => BeforeCaptureHook | null {+ return () => createVideoFrameInjector(frameLookup);+}++function frameNumberFromFileName(name: string): number | null {+ const match = /(\d+)(?=\.[^.]+$)/.exec(name);+ if (!match) return null;+ const frameNumber = Number(match[1]);+ return Number.isSafeInteger(frameNumber) ? frameNumber : null;+}++/**+ * Rebuilds `ExtractedFrames[]` from `<planDir>/video-frames/<videoId>/`. `framePaths` keys are 0-based+ * frame indexes (what `getFrameAtTime` reads); any other key drops every video's first-paint frame.+ */+export function rebuildExtractedFramesFromPlanDir(+ planDir: string,+ videos: PlanVideosJson["extracted"],+ indexMode: "dense-v1" | "sparse-v2" = "dense-v1",+): ExtractedFrames[] {+ const result: ExtractedFrames[] = [];+ for (const v of videos) {+ const outputDir = join(planDir, "video-frames", v.videoId);+ if (!existsSync(outputDir)) {+ throw new Error(+ `[renderChunk] planDir missing extracted video frames for ${JSON.stringify(v.videoId)}: ` ++ `${outputDir} not present. plan() should have written frames here; the planDir is malformed.`,+ );+ }+ // Numeric order: past the padding width, lexical order puts frame_100000 before frame_10001.+ const ext = (extname(v.framePattern) || ".jpg").toLowerCase();+ const frames = readdirSync(outputDir)+ .filter((name) => name.toLowerCase().endsWith(ext))+ .sort((left, right) => {+ const leftNumber = frameNumberFromFileName(left);+ const rightNumber = frameNumberFromFileName(right);+ if (leftNumber === null || rightNumber === null) return left.localeCompare(right);+ return leftNumber - rightNumber || left.localeCompare(right);+ });+ const framePaths = new Map<number, string>();+ for (let i = 0; i < frames.length; i++) {+ const frameName = frames[i];+ if (!frameName) continue;+ // v2 materialization is sparse, so only v2 reads the index from ffmpeg's 1-based name.+ const frameNumber = indexMode === "sparse-v2" ? frameNumberFromFileName(frameName) : null;+ const frameIndex = frameNumber === null ? i : frameNumber - 1;+ framePaths.set(frameIndex, join(outputDir, frameName));+ }+ result.push({+ videoId: v.videoId,+ srcPath: v.srcPath,+ outputDir,+ framePattern: v.framePattern,+ fps: v.fps,+ totalFrames: v.totalFrames,+ metadata: v.metadata,+ framePaths,+ // The planDir is shared read-only across workers; cleanup must not delete its frames.+ ownedByLookup: false,+ });+ }+ return result;+}++function contiguousRuns(sortedIndexes: readonly number[]): FrameRange[] {+ const runs: FrameRange[] = [];+ for (const index of sortedIndexes) {+ const last = runs[runs.length - 1];+ if (last && last.firstFrame + last.frames === index) last.frames++;+ else runs.push({ firstFrame: index, frames: 1 });+ }+ return runs;+}++/** Capture options a chunk session uses; the planner measures frame cost with the same ones. */+export function buildChunkCaptureOptions(input: {+ dimensions: {+ width: number;+ height: number;+ fpsNum: number;+ fpsDen: number;+ format: DistributedFormat;+ };+ encoder: Pick<LockedRenderConfig, "quality" | "deviceScaleFactor" | "variables">;+ videoCount: number;+ html: string;+}): CaptureOptions {+ const { dimensions, encoder } = input;+ const videoCaptureBeyondViewport = resolveVideoCaptureBeyondViewport(input.videoCount);+ return {+ width: dimensions.width,+ height: dimensions.height,+ fps: { num: dimensions.fpsNum, den: dimensions.fpsDen },+ format: dimensions.format === "mp4" ? "jpeg" : "png",+ quality: dimensions.format === "mp4" ? resolveCaptureJpegQuality(encoder.quality) : undefined,+ deviceScaleFactor: encoder.deviceScaleFactor,+ // The controller's snapshot, so the first capture sees the in-process renderer's `__hfVariables`.+ variables: encoder.variables,+ subTimelineWaitMemo: {},+ ...(videoCaptureBeyondViewport !== undefined+ ? { captureBeyondViewport: videoCaptureBeyondViewport }+ : {}),+ // A fixed warmup count keeps `beginFrameTimeTicks` host-independent.+ lockWarmupTicks: true,+ requiresWebGpu: compositionRequiresWebGpu(input.html),+ };+}++/** Extracts the given frames of deferred videos under the names a full extraction gives them. */+export async function extractDeferredVideoFramesForChunk(input: {+ planDir: string;+ planVideos: PlanVideosJson;+ frames: Iterable<number>;+ fps: { num: number; den: number };+ outputRoot: string;+ cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;+}): Promise<ExtractedFrames[]> {+ const deferred = input.planVideos.extracted.filter((video) => video.deferredRange);+ if (deferred.length === 0) return [];+ const table = createFrameLookupTable(+ input.planVideos.videos,+ deferred.map((video) => ({+ ...video,+ outputDir: input.outputRoot,+ framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),+ })),+ resolveRenderFpsConfig(input.fps).value,+ );+ const shown = new Map<string, Set<number>>();+ for (const frame of input.frames) {+ const globalTime = (frame * input.fps.den) / input.fps.num;+ for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {+ const indexes = shown.get(videoId) ?? new Set<number>();+ indexes.add(payload.frameIndex);+ shown.set(videoId, indexes);+ }+ }+ return Promise.all(+ deferred.map(async (video) => {+ const outputDir = join(input.outputRoot, video.videoId);+ const indexes = [...(shown.get(video.videoId) ?? [])].sort((a, b) => a - b);+ if (indexes.length === 0) {+ return { ...video, outputDir, framePaths: new Map<number, string>(), ownedByLookup: false };+ }+ const range = video.deferredRange!;+ const sourcePath = join(input.planDir, ...range.sourcePath.split("/"));+ if (!existsSync(sourcePath)) {+ throw new RenderChunkValidationError(+ MISSING_PLAN_ARTIFACT,+ `[renderChunk] planDir is missing the source for video ${JSON.stringify(video.videoId)}: ${range.sourcePath}`,+ );+ }+ const extracted = await extractVideoFramesRange(+ sourcePath,+ video.videoId,+ range.startTime,+ range.durationSeconds,+ {+ fps: input.fps,+ outputDir: input.outputRoot,+ format: range.format,+ frameRanges: contiguousRuns(indexes),+ },+ undefined,+ input.cfg,+ );+ const missing = indexes.find((index) => !extracted.framePaths.has(index));+ if (missing !== undefined) {+ throw new RenderChunkValidationError(+ INVALID_VIDEO_METADATA,+ `[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${missing}; ` ++ `the plan expected ${video.totalFrames} frames from ${range.sourcePath}.`,+ );+ }+ return { ...video, outputDir, framePaths: extracted.framePaths, ownedByLookup: false };+ }),+ );+}diff --git a/packages/producer/src/services/distributed/plan.test.ts b/packages/producer/src/services/distributed/plan.test.tsindex de14a45978..cb4d7bd33f 100644--- a/packages/producer/src/services/distributed/plan.test.ts+++ b/packages/producer/src/services/distributed/plan.test.ts@@ -15,7 +15,7 @@* inside the regression harness (`bun run --cwd packages/producer docker:test`).*/-import { afterAll, beforeAll, describe, expect, it } from "bun:test";+import { afterAll, beforeAll, describe, expect, it, mock } from "bun:test";import { spawnSync } from "node:child_process";import {chmodSync,@@ -46,11 +46,13 @@ import {ChunkExceedsWorkerTimeoutError,DEFAULT_CHUNK_SIZE,DEFAULT_MAX_PARALLEL_CHUNKS,+ chunkSizingUsesFrameCost,estimateChunkSeconds,MIN_CHUNK_SIZE,plan,resolveDistributedEngineConfig,resolveChunkPlan,+ planNeedsFrameCostMeasurement,resolveMaxParallelChunks,} from "./plan.js";import { buildSyntheticRenderJob } from "./shared.js";@@ -286,6 +288,45 @@ describe("resolveMaxParallelChunks", () => {expect(resolveMaxParallelChunks({ ...worker, targetChunkFrames: 300 }, ...args)).toBe(256);});+ it("sizes chunks from a measured frame cost so expensive frames fit the limit", () => {+ // 45 s of 1080x1920 at 24 fps whose frames measured ~10.5 s each (a graded video on software GL).+ const frames = 45 * 24;+ const mp = (1080 * 1920) / 1e6;+ const worker = { workerTimeoutSeconds: LAMBDA_LIMIT, workerConcurrency: 8 };+ const { log } = recordingLog();+ const priced = resolveChunkPlan(+ frames,+ undefined,+ resolveMaxParallelChunks(worker, frames, mp),+ );+ expect(estimateChunkSeconds(priced.effectiveChunkSize, mp, 10.5)).toBeGreaterThan(LAMBDA_LIMIT);++ const measured = resolveChunkPlan(+ frames,+ undefined,+ resolveMaxParallelChunks(worker, frames, mp, log, 10.5),+ );+ expect(estimateChunkSeconds(measured.effectiveChunkSize, mp, 10.5)).toBeLessThanOrEqual(+ LAMBDA_LIMIT * 0.75,+ );+ // A cheap measured frame keeps the per-pixel plan.+ expect(resolveMaxParallelChunks(worker, frames, mp, log, 0.07)).toBe(+ resolveMaxParallelChunks(worker, frames, mp, log),+ );+ });++ it("measures frame cost only for graded media and only when sizing uses it", () => {+ expect(planNeedsFrameCostMeasurement('<video data-color-grading=\'{"preset":"x"}\'>')).toBe(+ true,+ );+ expect(planNeedsFrameCostMeasurement("<video src=a.mp4>")).toBe(false);+ expect(chunkSizingUsesFrameCost({ workerTimeoutSeconds: LAMBDA_LIMIT })).toBe(true);+ expect(chunkSizingUsesFrameCost({ workerTimeoutSeconds: LAMBDA_LIMIT, chunkSize: 48 })).toBe(+ false,+ );+ expect(chunkSizingUsesFrameCost({})).toBe(false);+ });+it("fails fast with a typed error when even 256 chunks cannot fit the limit", () => {const eightHours = 8 * 60 * 60 * 30;let caught: unknown;@@ -525,6 +566,38 @@ describe("plan() defaults", () => {expect(sized.chunkCount).toBe(1);}, 30_000);+ it("splits graded media into more chunks from its measured frame cost", async () => {+ const measuredFor: string[] = [];+ mock.module("./planFrameCost.js", () => ({+ measurePlanSecondsPerFrame: async (input: { html: string }) => {+ measuredFor.push(input.html);+ return 10;+ },+ }));+ const graded = join(runRoot, "project-graded");+ mkdirSync(graded, { recursive: true });+ writeFileSync(+ join(graded, "index.html"),+ FIXTURE_HTML.replace("<p>", `<p data-color-grading='{"exposure":0.2}'>`),+ "utf-8",+ );+ const config = {+ fps: 30,+ width: 320,+ height: 240,+ format: "mp4",+ workerTimeoutSeconds: 900,+ workerConcurrency: 8,+ } as const;+ const cheap = await plan(projectDir, config, join(runRoot, "plan-cheap"));+ expect(measuredFor).toHaveLength(0);+ const costly = await plan(graded, config, join(runRoot, "plan-graded"));+ expect(measuredFor).toHaveLength(1);+ expect(cheap.chunkCount).toBe(1);+ // 30 frames at 10 s each split as far as the 10-frame chunk floor allows.+ expect(costly.chunkCount).toBe(3);+ }, 30_000);+it("rejects at plan time when no chunk can finish inside the worker limit", async () => {await expect(plan(diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.tsindex 96c178a87b..87fb1603b1 100644--- a/packages/producer/src/services/distributed/plan.ts+++ b/packages/producer/src/services/distributed/plan.ts@@ -38,7 +38,12 @@ import {writeFileSync,} from "node:fs";import { extname, isAbsolute, join, relative, sep } from "node:path";-import { type CanvasResolution, type DistributedFps, fpsToNumber } from "@hyperframes/core";+import {+ type CanvasResolution,+ type DistributedFps,+ fpsToNumber,+ HF_COLOR_GRADING_ATTR,+} from "@hyperframes/core";import {type EngineConfig,type VideoFrameFormat,@@ -54,6 +59,7 @@ import {type RenderStrictness,} from "../renderOrchestrator.js";import { closeFileServerSafely } from "../fileServer.js";+import { measurePlanSecondsPerFrame } from "./planFrameCost.js";import { constrainAacTruePeak } from "../render/audioPadTrim.js";import { encoderFailureError } from "../render/encoderInterruption.js";import { runAudioStage } from "../render/stages/audioStage.js";@@ -417,10 +423,44 @@ const ESTIMATED_CHUNK_OVERHEAD_SECONDS = 60;/** Share of the worker time limit a chunk's predicted time may use. */const WORKER_TIMEOUT_MARGIN = 0.75;+/** Seconds one frame is priced at: the per-pixel estimate, or the measured cost when it is higher. */+function estimateFrameSeconds(megapixels: number, measuredSecondsPerFrame = 0): number {+ return Math.max(ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME * megapixels, measuredSecondsPerFrame);+}+/** Predicted seconds for one chunk of `frames` frames at `megapixels` output pixels per frame. */-export function estimateChunkSeconds(frames: number, megapixels: number): number {+export function estimateChunkSeconds(+ frames: number,+ megapixels: number,+ measuredSecondsPerFrame?: number,+): number {return (- ESTIMATED_CHUNK_OVERHEAD_SECONDS + frames * ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME * megapixels+ ESTIMATED_CHUNK_OVERHEAD_SECONDS ++ frames * estimateFrameSeconds(megapixels, measuredSecondsPerFrame)+ );+}++/** Graded media runs a full-frame shader per frame, which the per-pixel estimate cannot see. */+export function planNeedsFrameCostMeasurement(html: string): boolean {+ return new RegExp(`\\b${HF_COLOR_GRADING_ATTR}=`, "i").test(html);+}++/** Whether {@link resolveMaxParallelChunks} sizes chunks from what a frame costs. */+export function chunkSizingUsesFrameCost(+ config: Pick<+ DistributedRenderConfig,+ | "maxParallelChunks"+ | "chunkSize"+ | "targetChunkFrames"+ | "workerTimeoutSeconds"+ | "workerConcurrency"+ >,+): boolean {+ return (+ config.maxParallelChunks === undefined &&+ config.chunkSize === undefined &&+ config.targetChunkFrames === undefined &&+ (config.workerTimeoutSeconds !== undefined || config.workerConcurrency !== undefined));}@@ -460,6 +500,8 @@ export function resolveMaxParallelChunks(/** Output pixels per frame, after supersampling. */megapixels: number,log: ProducerLogger = defaultLogger,+ /** Measured capture seconds per frame; frames are priced at it when it beats the estimate. */+ measuredSecondsPerFrame?: number,): number {if (config.maxParallelChunks !== undefined) return config.maxParallelChunks;const { workerTimeoutSeconds, workerConcurrency } = config;@@ -467,11 +509,8 @@ export function resolveMaxParallelChunks(workerConcurrency ?? DEFAULT_MAX_PARALLEL_CHUNKS,MAX_PARALLEL_CHUNKS_CEILING,);- const pinsChunking = config.chunkSize !== undefined || config.targetChunkFrames !== undefined;- if (pinsChunking || (workerTimeoutSeconds === undefined && workerConcurrency === undefined)) {- return wave;- }- const secondsPerFrame = ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME * megapixels;+ if (!chunkSizingUsesFrameCost(config)) return wave;+ const secondsPerFrame = estimateFrameSeconds(megapixels, measuredSecondsPerFrame);const minFramesPerChunk = Math.ceil(ESTIMATED_CHUNK_OVERHEAD_SECONDS / secondsPerFrame);const wide = Math.max(1, Math.min(wave, Math.floor(totalFrames / minFramesPerChunk)));if (workerTimeoutSeconds === undefined) return wide;@@ -484,7 +523,7 @@ export function resolveMaxParallelChunks(if (needed > MAX_PARALLEL_CHUNKS_CEILING) {const framesAtCeiling = Math.ceil(totalFrames / MAX_PARALLEL_CHUNKS_CEILING);throw new ChunkExceedsWorkerTimeoutError(- Math.ceil(estimateChunkSeconds(framesAtCeiling, megapixels)),+ Math.ceil(estimateChunkSeconds(framesAtCeiling, megapixels, measuredSecondsPerFrame)),workerTimeoutSeconds,);}@@ -1228,20 +1267,12 @@ export async function buildLocalExecutionPlan(}}- // ── Chunking decisions ── (before extraction so an impossible plan fails fast)- const maxParallel = resolveMaxParallelChunks(- config,- totalFrames,- (compileResult.outputWidth * compileResult.outputHeight) / 1e6,- log,- );- const { chunkCount, effectiveChunkSize } = resolveChunkPlan(- totalFrames,- config.chunkSize,- maxParallel,- config.targetChunkFrames,- );- const chunks = buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize);+ // ── Chunking decisions ── (before extraction so an impossible plan fails fast; graded+ // media is re-sized from its measured frame cost once its video frames exist)+ const megapixels = (compileResult.outputWidth * compileResult.outputHeight) / 1e6;+ let maxParallel = resolveMaxParallelChunks(config, totalFrames, megapixels, log);+ const measureFrameCost =+ chunkSizingUsesFrameCost(config) && planNeedsFrameCostMeasurement(compiled.html);// The compiled tree is now stable. If it already exceeds the final plan// budget, extraction/audio/freeze can only add retained bytes, so fail before@@ -1352,7 +1383,7 @@ export async function buildLocalExecutionPlan(// encode; the assembler pads or trims the track of any other plan.const audioSeconds = chunksCarryAudio ? totalFrames / config.fps : probeResult.duration;stageStarted = Date.now();- const [audioOutcome, overlapOutcome] = await Promise.allSettled([+ const [audioOutcome, overlapOutcome, frameCostOutcome] = await Promise.allSettled([runAudioStage({projectDir,workDir,@@ -1376,9 +1407,45 @@ export async function buildLocalExecutionPlan(logPlanStage(log, "prepublish", prepublishStarted);}})(),+ measureFrameCost+ ? measurePlanSecondsPerFrame({+ planDir,+ workDir,+ html: compiled.html,+ planVideos: planVideosJson,+ totalFrames,+ fps: config.fps,+ width,+ height,+ format: config.format,+ quality: config.quality ?? "standard",+ deviceScaleFactor,+ variables: config.variables,+ cfg: { ...cfg, forceScreenshot, forceScreenshotExplicitlyOptedOut: !forceScreenshot },+ log,+ }).finally(() => logPlanStage(log, "frame-cost", stageStarted))+ : undefined,]);if (audioOutcome.status === "rejected") throw audioOutcome.reason;if (overlapOutcome.status === "rejected") throw overlapOutcome.reason;+ if (frameCostOutcome.status === "rejected") throw frameCostOutcome.reason;+ if (frameCostOutcome.value !== undefined) {+ log.info("[plan] measured frame cost", { secondsPerFrame: frameCostOutcome.value });+ maxParallel = resolveMaxParallelChunks(+ config,+ totalFrames,+ megapixels,+ log,+ frameCostOutcome.value,+ );+ }+ const { chunkCount, effectiveChunkSize } = resolveChunkPlan(+ totalFrames,+ config.chunkSize,+ maxParallel,+ config.targetChunkFrames,+ );+ const chunks = buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize);const audioResult = audioOutcome.value;if (audioResult.audioError) {applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);diff --git a/packages/producer/src/services/distributed/planFrameCost.ts b/packages/producer/src/services/distributed/planFrameCost.tsnew file mode 100644index 0000000000..8089c82958--- /dev/null+++ b/packages/producer/src/services/distributed/planFrameCost.ts@@ -0,0 +1,100 @@+import { join } from "node:path";+import {+ closeCaptureSession,+ createCaptureSession,+ createFrameLookupTable,+ type EngineConfig,+ initializeSession,+} from "@hyperframes/engine";+import type { ProducerLogger } from "../../logger.js";+import {+ buildVirtualTimeShim,+ closeFileServerSafely,+ createFileServer,+ resolveRenderFpsConfig,+} from "../fileServer.js";+import {+ createCaptureCalibrationConfig,+ measureCaptureCostFromSession,+ selectCaptureCalibrationFrames,+} from "../render/captureCost.js";+import {+ buildChunkCaptureOptions,+ createChunkVideoFrameInjectorFactory,+ extractDeferredVideoFramesForChunk,+ rebuildExtractedFramesFromPlanDir,+} from "./chunkFrames.js";+import type { DistributedFormat, PlanVideosJson } from "./shared.js";++/**+ * Captures the renderer's calibration frames in a session set up like a chunk+ * worker's (same capture options, engine config and video frames) and returns+ * the p95 capture seconds per frame.+ */+export async function measurePlanSecondsPerFrame(input: {+ planDir: string;+ workDir: string;+ html: string;+ planVideos: PlanVideosJson;+ totalFrames: number;+ fps: number;+ width: number;+ height: number;+ format: DistributedFormat;+ quality: "draft" | "standard" | "high";+ deviceScaleFactor: number;+ variables?: Record<string, unknown>;+ cfg: EngineConfig;+ log: ProducerLogger;+}): Promise<number> {+ const { planDir, workDir, planVideos, fps, log } = input;+ const fpsRatio = { num: fps, den: 1 };+ const sampled = selectCaptureCalibrationFrames(input.totalFrames);+ const extracted = [+ ...rebuildExtractedFramesFromPlanDir(+ planDir,+ planVideos.extracted.filter((video) => !video.deferredRange),+ ),+ ...(await extractDeferredVideoFramesForChunk({+ planDir,+ planVideos,+ frames: sampled,+ fps: fpsRatio,+ outputRoot: join(workDir, "frame-cost-video-frames"),+ cfg: input.cfg,+ })),+ ];+ const lookup =+ extracted.length > 0+ ? createFrameLookupTable(planVideos.videos, extracted, resolveRenderFpsConfig(fpsRatio).value)+ : null;+ const compiledDir = join(planDir, "compiled");+ const fileServer = await createFileServer({+ projectDir: compiledDir,+ compiledDir,+ port: 0,+ preHeadScripts: [buildVirtualTimeShim({ seedRandomFromFrame: true })],+ fps: fpsRatio,+ });+ let session: Awaited<ReturnType<typeof createCaptureSession>> | null = null;+ try {+ session = await createCaptureSession(+ fileServer.url,+ join(workDir, "frame-cost"),+ buildChunkCaptureOptions({+ dimensions: { ...input, fpsNum: fps, fpsDen: 1 },+ encoder: input,+ videoCount: planVideos.videos.length,+ html: input.html,+ }),+ createChunkVideoFrameInjectorFactory(lookup)(),+ createCaptureCalibrationConfig(input.cfg),+ );+ await initializeSession(session);+ const { estimate } = await measureCaptureCostFromSession(session, input.totalFrames, fps, log);+ return (estimate.p95Ms ?? 0) / 1000;+ } finally {+ if (session) await closeCaptureSession(session).catch(() => {});+ closeFileServerSafely(fileServer, "plan frame cost", log);+ }+}diff --git a/packages/producer/src/services/distributed/renderChunk.test.ts b/packages/producer/src/services/distributed/renderChunk.test.tsindex ad1dff4c4b..69beb54408 100644--- a/packages/producer/src/services/distributed/renderChunk.test.ts+++ b/packages/producer/src/services/distributed/renderChunk.test.ts@@ -583,6 +583,8 @@ describe("resolveLockedVp9CpuUsed", () => {});describe("extractDeferredVideoFramesForChunk", () => {+ const frameSpan = (start: number, end: number) =>+ Array.from({ length: end - start }, (_, i) => start + i);const root = mkdtempSync(join(tmpdir(), "hf-deferred-chunk-"));const planDir = join(root, "plan");const range = {@@ -657,7 +659,7 @@ describe("extractDeferredVideoFramesForChunk", () => {const [chunk] = await extractDeferredVideoFramesForChunk({planDir,planVideos: planVideos(false),- slice: { index: 1, startFrame: 30, endFrame: 45 },+ frames: frameSpan(30, 45),fps: { num: 30, den: 1 },outputRoot: join(root, "chunk-1"),cfg: {},@@ -673,7 +675,7 @@ describe("extractDeferredVideoFramesForChunk", () => {const [chunk] = await extractDeferredVideoFramesForChunk({planDir,planVideos: { ...planVideos(false), videos: [{ ...planVideos(false).videos[0]!, end: 1 }] },- slice: { index: 2, startFrame: 60, endFrame: 90 },+ frames: frameSpan(60, 90),fps: { num: 30, den: 1 },outputRoot: join(root, "chunk-2"),cfg: {},@@ -685,7 +687,7 @@ describe("extractDeferredVideoFramesForChunk", () => {const [chunk] = await extractDeferredVideoFramesForChunk({planDir,planVideos: planVideos(true),- slice: { index: 3, startFrame: 70, endFrame: 80 },+ frames: frameSpan(70, 80),fps: { num: 30, den: 1 },outputRoot: join(root, "chunk-3"),cfg: {},@@ -706,7 +708,7 @@ describe("extractDeferredVideoFramesForChunk", () => {await extractDeferredVideoFramesForChunk({planDir,planVideos: missing,- slice: { index: 0, startFrame: 0, endFrame: 5 },+ frames: frameSpan(0, 5),fps: { num: 30, den: 1 },outputRoot: join(root, "chunk-missing"),cfg: {},diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.tsindex 86d580d07d..0abc78de07 100644--- a/packages/producer/src/services/distributed/renderChunk.ts+++ b/packages/producer/src/services/distributed/renderChunk.ts@@ -36,14 +36,12 @@import { randomBytes } from "node:crypto";import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";-import { extname, join } from "node:path";+import { join } from "node:path";import {assertSwiftShader,- type BeforeCaptureHook,BROWSER_GPU_NOT_SOFTWARE,calculateOptimalWorkers,classifyCaptureFailure,- compositionRequiresWebGpu,type CaptureOptions,type CaptureMode,type CapturePerfSummary,@@ -51,13 +49,8 @@ import {closeCaptureSession,createCaptureSession,createFrameLookupTable,- createVideoFrameInjector,deriveBeginFrameProbeTimeTicks,type EngineConfig,- type ExtractedFrames,- extractVideoFramesRange,- type FrameRange,- type FrameLookupTable,getEncoderPreset,initializeSession,probeBeginFrameLiveness,@@ -68,8 +61,6 @@ import { defaultLogger } from "../../logger.js";import { applyRenderWarningPolicy } from "../renderOrchestrator.js";import { runEncodeStage } from "../render/stages/encodeStage.js";import { runCaptureStage } from "../render/stages/captureStage.js";-import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";-import { resolveCaptureJpegQuality } from "../render/captureImageFormat.js";import { createCapturePlan } from "../render/capturePlan.js";import {type ChunkSliceJson,@@ -98,48 +89,34 @@ import {import { muxChunkWithAudio } from "./chunkAudio.js";import { DISTRIBUTED_RENDER_CAPABILITIES, readPlanProtocolV1 } from "./planProtocol.js";import { validatePlanV2MaterializedTarget } from "./planV2.js";+import {+ buildChunkCaptureOptions,+ createChunkVideoFrameInjectorFactory,+ extractDeferredVideoFramesForChunk,+ MISSING_PLAN_ARTIFACT,+ PLAN_HASH_MISMATCH,+ FFMPEG_VERSION_MISMATCH,+ CHUNK_INDEX_OUT_OF_RANGE,+ MISSING_RUNTIME_ENV_SNAPSHOT,+ rebuildExtractedFramesFromPlanDir,+ RenderChunkValidationError,+} from "./chunkFrames.js";++export {+ CHUNK_INDEX_OUT_OF_RANGE,+ createChunkVideoFrameInjectorFactory,+ extractDeferredVideoFramesForChunk,+ FFMPEG_VERSION_MISMATCH,+ INVALID_VIDEO_METADATA,+ MISSING_PLAN_ARTIFACT,+ MISSING_RUNTIME_ENV_SNAPSHOT,+ PLAN_HASH_MISMATCH,+ rebuildExtractedFramesFromPlanDir,+ RenderChunkValidationError,+} from "./chunkFrames.js";-/**- * Non-retryable error codes raised when the planDir is structurally- * malformed, semantically out of range, or fingerprints differently from- * what the controller wrote. Each is distinct so adapter retry policies- * can route them independently — e.g. `MISSING_PLAN_ARTIFACT` may point- * to a partial S3 download that a retry could heal, while- * `PLAN_HASH_MISMATCH` strictly indicates cross-version drift that- * retries won't fix.- */-export const FFMPEG_VERSION_MISMATCH = "FFMPEG_VERSION_MISMATCH";-export const PLAN_HASH_MISMATCH = "PLAN_HASH_MISMATCH";-export const MISSING_PLAN_ARTIFACT = "MISSING_PLAN_ARTIFACT";-export const CHUNK_INDEX_OUT_OF_RANGE = "CHUNK_INDEX_OUT_OF_RANGE";-export const MISSING_RUNTIME_ENV_SNAPSHOT = "MISSING_RUNTIME_ENV_SNAPSHOT";-export { INVALID_VIDEO_METADATA };const LEGACY_DISTRIBUTED_VP9_CPU_USED = 2;-export type RenderChunkValidationCode =- | typeof FFMPEG_VERSION_MISMATCH- | typeof PLAN_HASH_MISMATCH- | typeof MISSING_PLAN_ARTIFACT- | typeof CHUNK_INDEX_OUT_OF_RANGE- | typeof MISSING_RUNTIME_ENV_SNAPSHOT- | typeof INVALID_VIDEO_METADATA- | typeof BROWSER_GPU_NOT_SOFTWARE;--/**- * Typed non-retryable error raised by `renderChunk` when the planDir is- * malformed or the worker's runtime doesn't match the planDir's- * controller-side fingerprint. Workflow adapters key retry policies off- * `code` — most of these failures will not heal on retry.- */-export class RenderChunkValidationError extends Error {- readonly code: RenderChunkValidationCode;- constructor(code: RenderChunkValidationCode, message: string) {- super(message);- this.name = "RenderChunkValidationError";- this.code = code;- }-}-/** Validate the shared video contract before any v1 chunk can inject frames. */export function validatePlanVideosForChunk(value: unknown): PlanVideosJson {try {@@ -260,17 +237,6 @@ export async function createVerifiedDistributedCaptureSession(}}-/**- * Build the immutable lookup table once, but keep injector state scoped to a- * browser session. Reusing one hook across workers or a whole-chunk retry can- * incorrectly suppress injection on a fresh page.- */-export function createChunkVideoFrameInjectorFactory(- frameLookup: FrameLookupTable | null,-): () => BeforeCaptureHook | null {- return () => createVideoFrameInjector(frameLookup);-}-function isCaptureMode(value: string): value is CaptureMode {return value === "beginframe" || value === "screenshot" || value === "drawelement";}@@ -333,173 +299,6 @@ export async function beginFrameSessionNeedsScreenshotFallback(return !(await probe(session.page, timeoutMs, probeTick, session.beginFrameIntervalMs));}-function frameNumberFromFileName(name: string): number | null {- const match = /(\d+)(?=\.[^.]+$)/.exec(name);- if (!match) return null;- const frameNumber = Number(match[1]);- return Number.isSafeInteger(frameNumber) ? frameNumber : null;-}--/**- * Rebuild the engine's in-memory `ExtractedFrames[]` from the on-disk- * planDir layout. `<planDir>/video-frames/<videoId>/` holds the numbered- * frame files plan() extracted; this lists each dir and rebuilds the- * 0-based `framePaths` Map that `FrameLookupTable` / `videoFrameInjector`- * both index against — the consumer is- * `videoFrameExtractor.ts:getFrameAtTime`, which floors `localTime * fps`- * to a 0-based index and reads `framePaths.get(frameIndex)`. Any drift- * from that key convention silently drops every `<video>`'s first-paint- * frame; see HF#1731 / HF#1730.- *- * Exported so a unit test can pin the 0-based contract without spinning- * up the heavyweight Docker fixture — the bug surfaces only under- * distributed mode and only at video first-paint, so this primitive is- * the right granularity to guard.- */-export function rebuildExtractedFramesFromPlanDir(- planDir: string,- videos: PlanVideosJson["extracted"],- indexMode: "dense-v1" | "sparse-v2" = "dense-v1",-): ExtractedFrames[] {- const result: ExtractedFrames[] = [];- for (const v of videos) {- const outputDir = join(planDir, "video-frames", v.videoId);- if (!existsSync(outputDir)) {- throw new Error(- `[renderChunk] planDir missing extracted video frames for ${JSON.stringify(v.videoId)}: ` +- `${outputDir} not present. plan() should have written frames here; the planDir is malformed.`,- );- }- // framePattern looks like `frame_%05d.jpg`; sprintf isn't available at- // runtime so list the directory and order numeric names by their ordinal.- // Width changes once FFmpeg passes the padding minimum, so lexical order- // would interleave frame_100000 before frame_10001.- const ext = (extname(v.framePattern) || ".jpg").toLowerCase();- const frames = readdirSync(outputDir)- .filter((name) => name.toLowerCase().endsWith(ext))- .sort((left, right) => {- const leftNumber = frameNumberFromFileName(left);- const rightNumber = frameNumberFromFileName(right);- if (leftNumber === null || rightNumber === null) return left.localeCompare(right);- return leftNumber - rightNumber || left.localeCompare(right);- });- const framePaths = new Map<number, string>();- for (let i = 0; i < frames.length; i++) {- const frameName = frames[i];- if (!frameName) continue;- // V1 plans preserve the historical sorted-position behavior even for- // unusual zero-based filenames. V2 materialization is sparse, so only- // that mode derives the original index from ffmpeg's 1-based filename.- const frameNumber = indexMode === "sparse-v2" ? frameNumberFromFileName(frameName) : null;- const frameIndex = frameNumber === null ? i : frameNumber - 1;- framePaths.set(frameIndex, join(outputDir, frameName));- }- result.push({- videoId: v.videoId,- srcPath: v.srcPath,- outputDir,- framePattern: v.framePattern,- fps: v.fps,- totalFrames: v.totalFrames,- metadata: v.metadata,- framePaths,- // The chunk worker doesn't own the planDir's video-frames/ directory- // (the controller does — adapters that fan out chunks across machines- // share the planDir as read-only). Mark ownership as false so the- // injector's eventual cleanup doesn't rm bytes another worker may- // still be reading.- ownedByLookup: false,- });- }- return result;-}--function contiguousRuns(sortedIndexes: readonly number[]): FrameRange[] {- const runs: FrameRange[] = [];- for (const index of sortedIndexes) {- const last = runs[runs.length - 1];- if (last && last.firstFrame + last.frames === index) last.frames++;- else runs.push({ firstFrame: index, frames: 1 });- }- return runs;-}--/**- * Extract the frames this chunk shows from videos the planner left- * unextracted. Frames land in `outputRoot/<videoId>/` under the names a full- * extraction gives them, so the lookup indexes them exactly as it would- * frames the planner wrote.- */-export async function extractDeferredVideoFramesForChunk(input: {- planDir: string;- planVideos: PlanVideosJson;- slice: ChunkSliceJson;- fps: { num: number; den: number };- outputRoot: string;- cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;-}): Promise<ExtractedFrames[]> {- const deferred = input.planVideos.extracted.filter((video) => video.deferredRange);- if (deferred.length === 0) return [];- const table = createFrameLookupTable(- input.planVideos.videos,- deferred.map((video) => ({- ...video,- outputDir: input.outputRoot,- framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),- })),- resolveRenderFpsConfig(input.fps).value,- );- const shown = new Map<string, Set<number>>();- for (let frame = input.slice.startFrame; frame < input.slice.endFrame; frame++) {- const globalTime = (frame * input.fps.den) / input.fps.num;- for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {- const indexes = shown.get(videoId) ?? new Set<number>();- indexes.add(payload.frameIndex);- shown.set(videoId, indexes);- }- }- return Promise.all(- deferred.map(async (video) => {- const outputDir = join(input.outputRoot, video.videoId);- const indexes = [...(shown.get(video.videoId) ?? [])].sort((a, b) => a - b);- if (indexes.length === 0) {- return { ...video, outputDir, framePaths: new Map<number, string>(), ownedByLookup: false };- }- const range = video.deferredRange!;- const sourcePath = join(input.planDir, ...range.sourcePath.split("/"));- if (!existsSync(sourcePath)) {- throw new RenderChunkValidationError(- MISSING_PLAN_ARTIFACT,- `[renderChunk] planDir is missing the source for video ${JSON.stringify(video.videoId)}: ${range.sourcePath}`,- );- }- const extracted = await extractVideoFramesRange(- sourcePath,- video.videoId,- range.startTime,- range.durationSeconds,- {- fps: input.fps,- outputDir: input.outputRoot,- format: range.format,- frameRanges: contiguousRuns(indexes),- },- undefined,- input.cfg,- );- const missing = indexes.find((index) => !extracted.framePaths.has(index));- if (missing !== undefined) {- throw new RenderChunkValidationError(- INVALID_VIDEO_METADATA,- `[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${missing}; ` +- `the plan expected ${video.totalFrames} frames from ${range.sourcePath}.`,- );- }- return { ...video, outputDir, framePaths: extracted.framePaths, ownedByLookup: false };- }),- );-}-/** Plan-time JSON manifest written by `freezePlan`. */interface PlanJson {protocol?: unknown;@@ -789,7 +588,10 @@ export async function renderChunk(...(await extractDeferredVideoFramesForChunk({planDir,planVideos,- slice,+ frames: Array.from(+ { length: slice.endFrame - slice.startFrame },+ (_, i) => slice.startFrame + i,+ ),fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },outputRoot: videoFramesDir,cfg,@@ -800,10 +602,6 @@ export async function renderChunk(: null;const createChunkVideoFrameInjector = createChunkVideoFrameInjectorFactory(videoFrameLookup);- const videoCaptureBeyondViewport = resolveVideoCaptureBeyondViewport(- planVideos?.videos.length ?? 0,- );-// ── File server with the seeded-random shim ──// `Math.random` / `crypto.getRandomValues` are seeded from virtual// time so retries are pixel-identical. Only distributed renders flip this.@@ -817,33 +615,12 @@ export async function renderChunk(fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },});- const captureOptions: CaptureOptions = {- width: plan.dimensions.width,- height: plan.dimensions.height,- fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },- format: plan.dimensions.format === "mp4" ? "jpeg" : "png",- quality:- plan.dimensions.format === "mp4"- ? resolveCaptureJpegQuality(job.config.quality)- : undefined,- deviceScaleFactor: encoder.deviceScaleFactor,- // Re-inject the controller's snapshotted variables so the chunk's- // first capture sees the same `window.__hfVariables` the in-process- // renderer would have seen. Optional — compositions that don't- // declare `data-composition-variables` leave this undefined and the- // engine skips the `evaluateOnNewDocument` injection.- variables: encoder.variables,- subTimelineWaitMemo: {},- ...(videoCaptureBeyondViewport !== undefined- ? { captureBeyondViewport: videoCaptureBeyondViewport }- : {}),- // lock the BeginFrame warmup loop to a fixed iteration count so- // `beginFrameTimeTicks` is host-independent. Only chunks ever set this.- lockWarmupTicks: true,- requiresWebGpu: compositionRequiresWebGpu(- readFileSync(join(compiledDir, "index.html"), "utf-8"),- ),- };+ const captureOptions = buildChunkCaptureOptions({+ dimensions: plan.dimensions,+ encoder,+ videoCount: planVideos?.videos.length ?? 0,+ html: readFileSync(join(compiledDir, "index.html"), "utf-8"),+ });// Resolve worker count up-front. Sequential capture reuses the initialized// session below. Parallel BeginFrame capture pays one bounded preflight