Skip to content

Write HTML. Render video. Built for agents.

fix(producer): graded video renders split into chunks that finish inside the worker limit (#5883)

Miguel Ángel6c3cc7cbe91b
diff --git a/packages/producer/src/services/distributed/chunkFrames.ts b/packages/producer/src/services/distributed/chunkFrames.ts
new file mode 100644
index 0000000000..440f164f90
--- /dev/null
+++ b/packages/producer/src/services/distributed/chunkFrames.ts
@@ -0,0 +1,230 @@
+// What a chunk capture session is built from, shared with the planner's frame-cost measurement.
+import { existsSync, readdirSync } from "node:fs";
+import { extname, join } from "node:path";
+import {
+ type BeforeCaptureHook,
+ BROWSER_GPU_NOT_SOFTWARE,
+ compositionRequiresWebGpu,
+ type CaptureOptions,
+ createFrameLookupTable,
+ createVideoFrameInjector,
+ type EngineConfig,
+ type ExtractedFrames,
+ extractVideoFramesRange,
+ type FrameRange,
+ type FrameLookupTable,
+} from "@hyperframes/engine";
+import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";
+import { resolveCaptureJpegQuality } from "../render/captureImageFormat.js";
+import type { LockedRenderConfig } from "../render/stages/freezePlan.js";
+import { resolveRenderFpsConfig } from "../fileServer.js";
+import { type DistributedFormat, INVALID_VIDEO_METADATA, type PlanVideosJson } from "./shared.js";
+
+/** Distinct codes so adapters route retries per cause: a missing artifact may heal, a hash mismatch never. */
+export const FFMPEG_VERSION_MISMATCH = "FFMPEG_VERSION_MISMATCH";
+export const PLAN_HASH_MISMATCH = "PLAN_HASH_MISMATCH";
+export const MISSING_PLAN_ARTIFACT = "MISSING_PLAN_ARTIFACT";
+export const CHUNK_INDEX_OUT_OF_RANGE = "CHUNK_INDEX_OUT_OF_RANGE";
+export const MISSING_RUNTIME_ENV_SNAPSHOT = "MISSING_RUNTIME_ENV_SNAPSHOT";
+export { INVALID_VIDEO_METADATA };
+
+export type RenderChunkValidationCode =
+ | typeof FFMPEG_VERSION_MISMATCH
+ | typeof PLAN_HASH_MISMATCH
+ | typeof MISSING_PLAN_ARTIFACT
+ | typeof CHUNK_INDEX_OUT_OF_RANGE
+ | typeof MISSING_RUNTIME_ENV_SNAPSHOT
+ | typeof INVALID_VIDEO_METADATA
+ | typeof BROWSER_GPU_NOT_SOFTWARE;
+
+/** A malformed planDir or a worker that does not match the plan's fingerprint; adapters key retries off `code`. */
+export class RenderChunkValidationError extends Error {
+ readonly code: RenderChunkValidationCode;
+ constructor(code: RenderChunkValidationCode, message: string) {
+ super(message);
+ this.name = "RenderChunkValidationError";
+ this.code = code;
+ }
+}
+
+/** One injector per browser session: a hook reused on a fresh page can skip its first injection. */
+export function createChunkVideoFrameInjectorFactory(
+ frameLookup: FrameLookupTable | null,
+): () => BeforeCaptureHook | null {
+ return () => createVideoFrameInjector(frameLookup);
+}
+
+function frameNumberFromFileName(name: string): number | null {
+ const match = /(\d+)(?=\.[^.]+$)/.exec(name);
+ if (!match) return null;
+ const frameNumber = Number(match[1]);
+ return Number.isSafeInteger(frameNumber) ? frameNumber : null;
+}
+
+/**
+ * Rebuilds `ExtractedFrames[]` from `<planDir>/video-frames/<videoId>/`. `framePaths` keys are 0-based
+ * frame indexes (what `getFrameAtTime` reads); any other key drops every video's first-paint frame.
+ */
+export function rebuildExtractedFramesFromPlanDir(
+ planDir: string,
+ videos: PlanVideosJson["extracted"],
+ indexMode: "dense-v1" | "sparse-v2" = "dense-v1",
+): ExtractedFrames[] {
+ const result: ExtractedFrames[] = [];
+ for (const v of videos) {
+ const outputDir = join(planDir, "video-frames", v.videoId);
+ if (!existsSync(outputDir)) {
+ throw new Error(
+ `[renderChunk] planDir missing extracted video frames for ${JSON.stringify(v.videoId)}: ` +
+ `${outputDir} not present. plan() should have written frames here; the planDir is malformed.`,
+ );
+ }
+ // Numeric order: past the padding width, lexical order puts frame_100000 before frame_10001.
+ const ext = (extname(v.framePattern) || ".jpg").toLowerCase();
+ const frames = readdirSync(outputDir)
+ .filter((name) => name.toLowerCase().endsWith(ext))
+ .sort((left, right) => {
+ const leftNumber = frameNumberFromFileName(left);
+ const rightNumber = frameNumberFromFileName(right);
+ if (leftNumber === null || rightNumber === null) return left.localeCompare(right);
+ return leftNumber - rightNumber || left.localeCompare(right);
+ });
+ const framePaths = new Map<number, string>();
+ for (let i = 0; i < frames.length; i++) {
+ const frameName = frames[i];
+ if (!frameName) continue;
+ // v2 materialization is sparse, so only v2 reads the index from ffmpeg's 1-based name.
+ const frameNumber = indexMode === "sparse-v2" ? frameNumberFromFileName(frameName) : null;
+ const frameIndex = frameNumber === null ? i : frameNumber - 1;
+ framePaths.set(frameIndex, join(outputDir, frameName));
+ }
+ result.push({
+ videoId: v.videoId,
+ srcPath: v.srcPath,
+ outputDir,
+ framePattern: v.framePattern,
+ fps: v.fps,
+ totalFrames: v.totalFrames,
+ metadata: v.metadata,
+ framePaths,
+ // The planDir is shared read-only across workers; cleanup must not delete its frames.
+ ownedByLookup: false,
+ });
+ }
+ return result;
+}
+
+function contiguousRuns(sortedIndexes: readonly number[]): FrameRange[] {
+ const runs: FrameRange[] = [];
+ for (const index of sortedIndexes) {
+ const last = runs[runs.length - 1];
+ if (last && last.firstFrame + last.frames === index) last.frames++;
+ else runs.push({ firstFrame: index, frames: 1 });
+ }
+ return runs;
+}
+
+/** Capture options a chunk session uses; the planner measures frame cost with the same ones. */
+export function buildChunkCaptureOptions(input: {
+ dimensions: {
+ width: number;
+ height: number;
+ fpsNum: number;
+ fpsDen: number;
+ format: DistributedFormat;
+ };
+ encoder: Pick<LockedRenderConfig, "quality" | "deviceScaleFactor" | "variables">;
+ videoCount: number;
+ html: string;
+}): CaptureOptions {
+ const { dimensions, encoder } = input;
+ const videoCaptureBeyondViewport = resolveVideoCaptureBeyondViewport(input.videoCount);
+ return {
+ width: dimensions.width,
+ height: dimensions.height,
+ fps: { num: dimensions.fpsNum, den: dimensions.fpsDen },
+ format: dimensions.format === "mp4" ? "jpeg" : "png",
+ quality: dimensions.format === "mp4" ? resolveCaptureJpegQuality(encoder.quality) : undefined,
+ deviceScaleFactor: encoder.deviceScaleFactor,
+ // The controller's snapshot, so the first capture sees the in-process renderer's `__hfVariables`.
+ variables: encoder.variables,
+ subTimelineWaitMemo: {},
+ ...(videoCaptureBeyondViewport !== undefined
+ ? { captureBeyondViewport: videoCaptureBeyondViewport }
+ : {}),
+ // A fixed warmup count keeps `beginFrameTimeTicks` host-independent.
+ lockWarmupTicks: true,
+ requiresWebGpu: compositionRequiresWebGpu(input.html),
+ };
+}
+
+/** Extracts the given frames of deferred videos under the names a full extraction gives them. */
+export async function extractDeferredVideoFramesForChunk(input: {
+ planDir: string;
+ planVideos: PlanVideosJson;
+ frames: Iterable<number>;
+ fps: { num: number; den: number };
+ outputRoot: string;
+ cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;
+}): Promise<ExtractedFrames[]> {
+ const deferred = input.planVideos.extracted.filter((video) => video.deferredRange);
+ if (deferred.length === 0) return [];
+ const table = createFrameLookupTable(
+ input.planVideos.videos,
+ deferred.map((video) => ({
+ ...video,
+ outputDir: input.outputRoot,
+ framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),
+ })),
+ resolveRenderFpsConfig(input.fps).value,
+ );
+ const shown = new Map<string, Set<number>>();
+ for (const frame of input.frames) {
+ const globalTime = (frame * input.fps.den) / input.fps.num;
+ for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {
+ const indexes = shown.get(videoId) ?? new Set<number>();
+ indexes.add(payload.frameIndex);
+ shown.set(videoId, indexes);
+ }
+ }
+ return Promise.all(
+ deferred.map(async (video) => {
+ const outputDir = join(input.outputRoot, video.videoId);
+ const indexes = [...(shown.get(video.videoId) ?? [])].sort((a, b) => a - b);
+ if (indexes.length === 0) {
+ return { ...video, outputDir, framePaths: new Map<number, string>(), ownedByLookup: false };
+ }
+ const range = video.deferredRange!;
+ const sourcePath = join(input.planDir, ...range.sourcePath.split("/"));
+ if (!existsSync(sourcePath)) {
+ throw new RenderChunkValidationError(
+ MISSING_PLAN_ARTIFACT,
+ `[renderChunk] planDir is missing the source for video ${JSON.stringify(video.videoId)}: ${range.sourcePath}`,
+ );
+ }
+ const extracted = await extractVideoFramesRange(
+ sourcePath,
+ video.videoId,
+ range.startTime,
+ range.durationSeconds,
+ {
+ fps: input.fps,
+ outputDir: input.outputRoot,
+ format: range.format,
+ frameRanges: contiguousRuns(indexes),
+ },
+ undefined,
+ input.cfg,
+ );
+ const missing = indexes.find((index) => !extracted.framePaths.has(index));
+ if (missing !== undefined) {
+ throw new RenderChunkValidationError(
+ INVALID_VIDEO_METADATA,
+ `[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${missing}; ` +
+ `the plan expected ${video.totalFrames} frames from ${range.sourcePath}.`,
+ );
+ }
+ return { ...video, outputDir, framePaths: extracted.framePaths, ownedByLookup: false };
+ }),
+ );
+}
diff --git a/packages/producer/src/services/distributed/plan.test.ts b/packages/producer/src/services/distributed/plan.test.ts
index de14a45978..cb4d7bd33f 100644
--- a/packages/producer/src/services/distributed/plan.test.ts
+++ b/packages/producer/src/services/distributed/plan.test.ts
@@ -15,7 +15,7 @@
* inside the regression harness (`bun run --cwd packages/producer docker:test`).
*/
-import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import { afterAll, beforeAll, describe, expect, it, mock } from "bun:test";
import { spawnSync } from "node:child_process";
import {
chmodSync,
@@ -46,11 +46,13 @@ import {
ChunkExceedsWorkerTimeoutError,
DEFAULT_CHUNK_SIZE,
DEFAULT_MAX_PARALLEL_CHUNKS,
+ chunkSizingUsesFrameCost,
estimateChunkSeconds,
MIN_CHUNK_SIZE,
plan,
resolveDistributedEngineConfig,
resolveChunkPlan,
+ planNeedsFrameCostMeasurement,
resolveMaxParallelChunks,
} from "./plan.js";
import { buildSyntheticRenderJob } from "./shared.js";
@@ -286,6 +288,45 @@ describe("resolveMaxParallelChunks", () => {
expect(resolveMaxParallelChunks({ ...worker, targetChunkFrames: 300 }, ...args)).toBe(256);
});
+ it("sizes chunks from a measured frame cost so expensive frames fit the limit", () => {
+ // 45 s of 1080x1920 at 24 fps whose frames measured ~10.5 s each (a graded video on software GL).
+ const frames = 45 * 24;
+ const mp = (1080 * 1920) / 1e6;
+ const worker = { workerTimeoutSeconds: LAMBDA_LIMIT, workerConcurrency: 8 };
+ const { log } = recordingLog();
+ const priced = resolveChunkPlan(
+ frames,
+ undefined,
+ resolveMaxParallelChunks(worker, frames, mp),
+ );
+ expect(estimateChunkSeconds(priced.effectiveChunkSize, mp, 10.5)).toBeGreaterThan(LAMBDA_LIMIT);
+
+ const measured = resolveChunkPlan(
+ frames,
+ undefined,
+ resolveMaxParallelChunks(worker, frames, mp, log, 10.5),
+ );
+ expect(estimateChunkSeconds(measured.effectiveChunkSize, mp, 10.5)).toBeLessThanOrEqual(
+ LAMBDA_LIMIT * 0.75,
+ );
+ // A cheap measured frame keeps the per-pixel plan.
+ expect(resolveMaxParallelChunks(worker, frames, mp, log, 0.07)).toBe(
+ resolveMaxParallelChunks(worker, frames, mp, log),
+ );
+ });
+
+ it("measures frame cost only for graded media and only when sizing uses it", () => {
+ expect(planNeedsFrameCostMeasurement('<video data-color-grading=\'{"preset":"x"}\'>')).toBe(
+ true,
+ );
+ expect(planNeedsFrameCostMeasurement("<video src=a.mp4>")).toBe(false);
+ expect(chunkSizingUsesFrameCost({ workerTimeoutSeconds: LAMBDA_LIMIT })).toBe(true);
+ expect(chunkSizingUsesFrameCost({ workerTimeoutSeconds: LAMBDA_LIMIT, chunkSize: 48 })).toBe(
+ false,
+ );
+ expect(chunkSizingUsesFrameCost({})).toBe(false);
+ });
+
it("fails fast with a typed error when even 256 chunks cannot fit the limit", () => {
const eightHours = 8 * 60 * 60 * 30;
let caught: unknown;
@@ -525,6 +566,38 @@ describe("plan() defaults", () => {
expect(sized.chunkCount).toBe(1);
}, 30_000);
+ it("splits graded media into more chunks from its measured frame cost", async () => {
+ const measuredFor: string[] = [];
+ mock.module("./planFrameCost.js", () => ({
+ measurePlanSecondsPerFrame: async (input: { html: string }) => {
+ measuredFor.push(input.html);
+ return 10;
+ },
+ }));
+ const graded = join(runRoot, "project-graded");
+ mkdirSync(graded, { recursive: true });
+ writeFileSync(
+ join(graded, "index.html"),
+ FIXTURE_HTML.replace("<p>", `<p data-color-grading='{"exposure":0.2}'>`),
+ "utf-8",
+ );
+ const config = {
+ fps: 30,
+ width: 320,
+ height: 240,
+ format: "mp4",
+ workerTimeoutSeconds: 900,
+ workerConcurrency: 8,
+ } as const;
+ const cheap = await plan(projectDir, config, join(runRoot, "plan-cheap"));
+ expect(measuredFor).toHaveLength(0);
+ const costly = await plan(graded, config, join(runRoot, "plan-graded"));
+ expect(measuredFor).toHaveLength(1);
+ expect(cheap.chunkCount).toBe(1);
+ // 30 frames at 10 s each split as far as the 10-frame chunk floor allows.
+ expect(costly.chunkCount).toBe(3);
+ }, 30_000);
+
it("rejects at plan time when no chunk can finish inside the worker limit", async () => {
await expect(
plan(
diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts
index 96c178a87b..87fb1603b1 100644
--- a/packages/producer/src/services/distributed/plan.ts
+++ b/packages/producer/src/services/distributed/plan.ts
@@ -38,7 +38,12 @@ import {
writeFileSync,
} from "node:fs";
import { extname, isAbsolute, join, relative, sep } from "node:path";
-import { type CanvasResolution, type DistributedFps, fpsToNumber } from "@hyperframes/core";
+import {
+ type CanvasResolution,
+ type DistributedFps,
+ fpsToNumber,
+ HF_COLOR_GRADING_ATTR,
+} from "@hyperframes/core";
import {
type EngineConfig,
type VideoFrameFormat,
@@ -54,6 +59,7 @@ import {
type RenderStrictness,
} from "../renderOrchestrator.js";
import { closeFileServerSafely } from "../fileServer.js";
+import { measurePlanSecondsPerFrame } from "./planFrameCost.js";
import { constrainAacTruePeak } from "../render/audioPadTrim.js";
import { encoderFailureError } from "../render/encoderInterruption.js";
import { runAudioStage } from "../render/stages/audioStage.js";
@@ -417,10 +423,44 @@ const ESTIMATED_CHUNK_OVERHEAD_SECONDS = 60;
/** Share of the worker time limit a chunk's predicted time may use. */
const WORKER_TIMEOUT_MARGIN = 0.75;
+/** Seconds one frame is priced at: the per-pixel estimate, or the measured cost when it is higher. */
+function estimateFrameSeconds(megapixels: number, measuredSecondsPerFrame = 0): number {
+ return Math.max(ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME * megapixels, measuredSecondsPerFrame);
+}
+
/** Predicted seconds for one chunk of `frames` frames at `megapixels` output pixels per frame. */
-export function estimateChunkSeconds(frames: number, megapixels: number): number {
+export function estimateChunkSeconds(
+ frames: number,
+ megapixels: number,
+ measuredSecondsPerFrame?: number,
+): number {
return (
- ESTIMATED_CHUNK_OVERHEAD_SECONDS + frames * ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME * megapixels
+ ESTIMATED_CHUNK_OVERHEAD_SECONDS +
+ frames * estimateFrameSeconds(megapixels, measuredSecondsPerFrame)
+ );
+}
+
+/** Graded media runs a full-frame shader per frame, which the per-pixel estimate cannot see. */
+export function planNeedsFrameCostMeasurement(html: string): boolean {
+ return new RegExp(`\\b${HF_COLOR_GRADING_ATTR}=`, "i").test(html);
+}
+
+/** Whether {@link resolveMaxParallelChunks} sizes chunks from what a frame costs. */
+export function chunkSizingUsesFrameCost(
+ config: Pick<
+ DistributedRenderConfig,
+ | "maxParallelChunks"
+ | "chunkSize"
+ | "targetChunkFrames"
+ | "workerTimeoutSeconds"
+ | "workerConcurrency"
+ >,
+): boolean {
+ return (
+ config.maxParallelChunks === undefined &&
+ config.chunkSize === undefined &&
+ config.targetChunkFrames === undefined &&
+ (config.workerTimeoutSeconds !== undefined || config.workerConcurrency !== undefined)
);
}
@@ -460,6 +500,8 @@ export function resolveMaxParallelChunks(
/** Output pixels per frame, after supersampling. */
megapixels: number,
log: ProducerLogger = defaultLogger,
+ /** Measured capture seconds per frame; frames are priced at it when it beats the estimate. */
+ measuredSecondsPerFrame?: number,
): number {
if (config.maxParallelChunks !== undefined) return config.maxParallelChunks;
const { workerTimeoutSeconds, workerConcurrency } = config;
@@ -467,11 +509,8 @@ export function resolveMaxParallelChunks(
workerConcurrency ?? DEFAULT_MAX_PARALLEL_CHUNKS,
MAX_PARALLEL_CHUNKS_CEILING,
);
- const pinsChunking = config.chunkSize !== undefined || config.targetChunkFrames !== undefined;
- if (pinsChunking || (workerTimeoutSeconds === undefined && workerConcurrency === undefined)) {
- return wave;
- }
- const secondsPerFrame = ESTIMATED_SECONDS_PER_MEGAPIXEL_FRAME * megapixels;
+ if (!chunkSizingUsesFrameCost(config)) return wave;
+ const secondsPerFrame = estimateFrameSeconds(megapixels, measuredSecondsPerFrame);
const minFramesPerChunk = Math.ceil(ESTIMATED_CHUNK_OVERHEAD_SECONDS / secondsPerFrame);
const wide = Math.max(1, Math.min(wave, Math.floor(totalFrames / minFramesPerChunk)));
if (workerTimeoutSeconds === undefined) return wide;
@@ -484,7 +523,7 @@ export function resolveMaxParallelChunks(
if (needed > MAX_PARALLEL_CHUNKS_CEILING) {
const framesAtCeiling = Math.ceil(totalFrames / MAX_PARALLEL_CHUNKS_CEILING);
throw new ChunkExceedsWorkerTimeoutError(
- Math.ceil(estimateChunkSeconds(framesAtCeiling, megapixels)),
+ Math.ceil(estimateChunkSeconds(framesAtCeiling, megapixels, measuredSecondsPerFrame)),
workerTimeoutSeconds,
);
}
@@ -1228,20 +1267,12 @@ export async function buildLocalExecutionPlan(
}
}
- // ── Chunking decisions ── (before extraction so an impossible plan fails fast)
- const maxParallel = resolveMaxParallelChunks(
- config,
- totalFrames,
- (compileResult.outputWidth * compileResult.outputHeight) / 1e6,
- log,
- );
- const { chunkCount, effectiveChunkSize } = resolveChunkPlan(
- totalFrames,
- config.chunkSize,
- maxParallel,
- config.targetChunkFrames,
- );
- const chunks = buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize);
+ // ── Chunking decisions ── (before extraction so an impossible plan fails fast; graded
+ // media is re-sized from its measured frame cost once its video frames exist)
+ const megapixels = (compileResult.outputWidth * compileResult.outputHeight) / 1e6;
+ let maxParallel = resolveMaxParallelChunks(config, totalFrames, megapixels, log);
+ const measureFrameCost =
+ chunkSizingUsesFrameCost(config) && planNeedsFrameCostMeasurement(compiled.html);
// The compiled tree is now stable. If it already exceeds the final plan
// budget, extraction/audio/freeze can only add retained bytes, so fail before
@@ -1352,7 +1383,7 @@ export async function buildLocalExecutionPlan(
// encode; the assembler pads or trims the track of any other plan.
const audioSeconds = chunksCarryAudio ? totalFrames / config.fps : probeResult.duration;
stageStarted = Date.now();
- const [audioOutcome, overlapOutcome] = await Promise.allSettled([
+ const [audioOutcome, overlapOutcome, frameCostOutcome] = await Promise.allSettled([
runAudioStage({
projectDir,
workDir,
@@ -1376,9 +1407,45 @@ export async function buildLocalExecutionPlan(
logPlanStage(log, "prepublish", prepublishStarted);
}
})(),
+ measureFrameCost
+ ? measurePlanSecondsPerFrame({
+ planDir,
+ workDir,
+ html: compiled.html,
+ planVideos: planVideosJson,
+ totalFrames,
+ fps: config.fps,
+ width,
+ height,
+ format: config.format,
+ quality: config.quality ?? "standard",
+ deviceScaleFactor,
+ variables: config.variables,
+ cfg: { ...cfg, forceScreenshot, forceScreenshotExplicitlyOptedOut: !forceScreenshot },
+ log,
+ }).finally(() => logPlanStage(log, "frame-cost", stageStarted))
+ : undefined,
]);
if (audioOutcome.status === "rejected") throw audioOutcome.reason;
if (overlapOutcome.status === "rejected") throw overlapOutcome.reason;
+ if (frameCostOutcome.status === "rejected") throw frameCostOutcome.reason;
+ if (frameCostOutcome.value !== undefined) {
+ log.info("[plan] measured frame cost", { secondsPerFrame: frameCostOutcome.value });
+ maxParallel = resolveMaxParallelChunks(
+ config,
+ totalFrames,
+ megapixels,
+ log,
+ frameCostOutcome.value,
+ );
+ }
+ const { chunkCount, effectiveChunkSize } = resolveChunkPlan(
+ totalFrames,
+ config.chunkSize,
+ maxParallel,
+ config.targetChunkFrames,
+ );
+ const chunks = buildChunkSlices(totalFrames, chunkCount, effectiveChunkSize);
const audioResult = audioOutcome.value;
if (audioResult.audioError) {
applyDistributedAudioWarningPolicy(job, audioResult.audioError, audioResult.audioFailures, log);
diff --git a/packages/producer/src/services/distributed/planFrameCost.ts b/packages/producer/src/services/distributed/planFrameCost.ts
new file mode 100644
index 0000000000..8089c82958
--- /dev/null
+++ b/packages/producer/src/services/distributed/planFrameCost.ts
@@ -0,0 +1,100 @@
+import { join } from "node:path";
+import {
+ closeCaptureSession,
+ createCaptureSession,
+ createFrameLookupTable,
+ type EngineConfig,
+ initializeSession,
+} from "@hyperframes/engine";
+import type { ProducerLogger } from "../../logger.js";
+import {
+ buildVirtualTimeShim,
+ closeFileServerSafely,
+ createFileServer,
+ resolveRenderFpsConfig,
+} from "../fileServer.js";
+import {
+ createCaptureCalibrationConfig,
+ measureCaptureCostFromSession,
+ selectCaptureCalibrationFrames,
+} from "../render/captureCost.js";
+import {
+ buildChunkCaptureOptions,
+ createChunkVideoFrameInjectorFactory,
+ extractDeferredVideoFramesForChunk,
+ rebuildExtractedFramesFromPlanDir,
+} from "./chunkFrames.js";
+import type { DistributedFormat, PlanVideosJson } from "./shared.js";
+
+/**
+ * Captures the renderer's calibration frames in a session set up like a chunk
+ * worker's (same capture options, engine config and video frames) and returns
+ * the p95 capture seconds per frame.
+ */
+export async function measurePlanSecondsPerFrame(input: {
+ planDir: string;
+ workDir: string;
+ html: string;
+ planVideos: PlanVideosJson;
+ totalFrames: number;
+ fps: number;
+ width: number;
+ height: number;
+ format: DistributedFormat;
+ quality: "draft" | "standard" | "high";
+ deviceScaleFactor: number;
+ variables?: Record<string, unknown>;
+ cfg: EngineConfig;
+ log: ProducerLogger;
+}): Promise<number> {
+ const { planDir, workDir, planVideos, fps, log } = input;
+ const fpsRatio = { num: fps, den: 1 };
+ const sampled = selectCaptureCalibrationFrames(input.totalFrames);
+ const extracted = [
+ ...rebuildExtractedFramesFromPlanDir(
+ planDir,
+ planVideos.extracted.filter((video) => !video.deferredRange),
+ ),
+ ...(await extractDeferredVideoFramesForChunk({
+ planDir,
+ planVideos,
+ frames: sampled,
+ fps: fpsRatio,
+ outputRoot: join(workDir, "frame-cost-video-frames"),
+ cfg: input.cfg,
+ })),
+ ];
+ const lookup =
+ extracted.length > 0
+ ? createFrameLookupTable(planVideos.videos, extracted, resolveRenderFpsConfig(fpsRatio).value)
+ : null;
+ const compiledDir = join(planDir, "compiled");
+ const fileServer = await createFileServer({
+ projectDir: compiledDir,
+ compiledDir,
+ port: 0,
+ preHeadScripts: [buildVirtualTimeShim({ seedRandomFromFrame: true })],
+ fps: fpsRatio,
+ });
+ let session: Awaited<ReturnType<typeof createCaptureSession>> | null = null;
+ try {
+ session = await createCaptureSession(
+ fileServer.url,
+ join(workDir, "frame-cost"),
+ buildChunkCaptureOptions({
+ dimensions: { ...input, fpsNum: fps, fpsDen: 1 },
+ encoder: input,
+ videoCount: planVideos.videos.length,
+ html: input.html,
+ }),
+ createChunkVideoFrameInjectorFactory(lookup)(),
+ createCaptureCalibrationConfig(input.cfg),
+ );
+ await initializeSession(session);
+ const { estimate } = await measureCaptureCostFromSession(session, input.totalFrames, fps, log);
+ return (estimate.p95Ms ?? 0) / 1000;
+ } finally {
+ if (session) await closeCaptureSession(session).catch(() => {});
+ closeFileServerSafely(fileServer, "plan frame cost", log);
+ }
+}
diff --git a/packages/producer/src/services/distributed/renderChunk.test.ts b/packages/producer/src/services/distributed/renderChunk.test.ts
index ad1dff4c4b..69beb54408 100644
--- a/packages/producer/src/services/distributed/renderChunk.test.ts
+++ b/packages/producer/src/services/distributed/renderChunk.test.ts
@@ -583,6 +583,8 @@ describe("resolveLockedVp9CpuUsed", () => {
});
describe("extractDeferredVideoFramesForChunk", () => {
+ const frameSpan = (start: number, end: number) =>
+ Array.from({ length: end - start }, (_, i) => start + i);
const root = mkdtempSync(join(tmpdir(), "hf-deferred-chunk-"));
const planDir = join(root, "plan");
const range = {
@@ -657,7 +659,7 @@ describe("extractDeferredVideoFramesForChunk", () => {
const [chunk] = await extractDeferredVideoFramesForChunk({
planDir,
planVideos: planVideos(false),
- slice: { index: 1, startFrame: 30, endFrame: 45 },
+ frames: frameSpan(30, 45),
fps: { num: 30, den: 1 },
outputRoot: join(root, "chunk-1"),
cfg: {},
@@ -673,7 +675,7 @@ describe("extractDeferredVideoFramesForChunk", () => {
const [chunk] = await extractDeferredVideoFramesForChunk({
planDir,
planVideos: { ...planVideos(false), videos: [{ ...planVideos(false).videos[0]!, end: 1 }] },
- slice: { index: 2, startFrame: 60, endFrame: 90 },
+ frames: frameSpan(60, 90),
fps: { num: 30, den: 1 },
outputRoot: join(root, "chunk-2"),
cfg: {},
@@ -685,7 +687,7 @@ describe("extractDeferredVideoFramesForChunk", () => {
const [chunk] = await extractDeferredVideoFramesForChunk({
planDir,
planVideos: planVideos(true),
- slice: { index: 3, startFrame: 70, endFrame: 80 },
+ frames: frameSpan(70, 80),
fps: { num: 30, den: 1 },
outputRoot: join(root, "chunk-3"),
cfg: {},
@@ -706,7 +708,7 @@ describe("extractDeferredVideoFramesForChunk", () => {
await extractDeferredVideoFramesForChunk({
planDir,
planVideos: missing,
- slice: { index: 0, startFrame: 0, endFrame: 5 },
+ frames: frameSpan(0, 5),
fps: { num: 30, den: 1 },
outputRoot: join(root, "chunk-missing"),
cfg: {},
diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.ts
index 86d580d07d..0abc78de07 100644
--- a/packages/producer/src/services/distributed/renderChunk.ts
+++ b/packages/producer/src/services/distributed/renderChunk.ts
@@ -36,14 +36,12 @@
import { randomBytes } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
-import { extname, join } from "node:path";
+import { join } from "node:path";
import {
assertSwiftShader,
- type BeforeCaptureHook,
BROWSER_GPU_NOT_SOFTWARE,
calculateOptimalWorkers,
classifyCaptureFailure,
- compositionRequiresWebGpu,
type CaptureOptions,
type CaptureMode,
type CapturePerfSummary,
@@ -51,13 +49,8 @@ import {
closeCaptureSession,
createCaptureSession,
createFrameLookupTable,
- createVideoFrameInjector,
deriveBeginFrameProbeTimeTicks,
type EngineConfig,
- type ExtractedFrames,
- extractVideoFramesRange,
- type FrameRange,
- type FrameLookupTable,
getEncoderPreset,
initializeSession,
probeBeginFrameLiveness,
@@ -68,8 +61,6 @@ import { defaultLogger } from "../../logger.js";
import { applyRenderWarningPolicy } from "../renderOrchestrator.js";
import { runEncodeStage } from "../render/stages/encodeStage.js";
import { runCaptureStage } from "../render/stages/captureStage.js";
-import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";
-import { resolveCaptureJpegQuality } from "../render/captureImageFormat.js";
import { createCapturePlan } from "../render/capturePlan.js";
import {
type ChunkSliceJson,
@@ -98,48 +89,34 @@ import {
import { muxChunkWithAudio } from "./chunkAudio.js";
import { DISTRIBUTED_RENDER_CAPABILITIES, readPlanProtocolV1 } from "./planProtocol.js";
import { validatePlanV2MaterializedTarget } from "./planV2.js";
+import {
+ buildChunkCaptureOptions,
+ createChunkVideoFrameInjectorFactory,
+ extractDeferredVideoFramesForChunk,
+ MISSING_PLAN_ARTIFACT,
+ PLAN_HASH_MISMATCH,
+ FFMPEG_VERSION_MISMATCH,
+ CHUNK_INDEX_OUT_OF_RANGE,
+ MISSING_RUNTIME_ENV_SNAPSHOT,
+ rebuildExtractedFramesFromPlanDir,
+ RenderChunkValidationError,
+} from "./chunkFrames.js";
+
+export {
+ CHUNK_INDEX_OUT_OF_RANGE,
+ createChunkVideoFrameInjectorFactory,
+ extractDeferredVideoFramesForChunk,
+ FFMPEG_VERSION_MISMATCH,
+ INVALID_VIDEO_METADATA,
+ MISSING_PLAN_ARTIFACT,
+ MISSING_RUNTIME_ENV_SNAPSHOT,
+ PLAN_HASH_MISMATCH,
+ rebuildExtractedFramesFromPlanDir,
+ RenderChunkValidationError,
+} from "./chunkFrames.js";
-/**
- * Non-retryable error codes raised when the planDir is structurally
- * malformed, semantically out of range, or fingerprints differently from
- * what the controller wrote. Each is distinct so adapter retry policies
- * can route them independently — e.g. `MISSING_PLAN_ARTIFACT` may point
- * to a partial S3 download that a retry could heal, while
- * `PLAN_HASH_MISMATCH` strictly indicates cross-version drift that
- * retries won't fix.
- */
-export const FFMPEG_VERSION_MISMATCH = "FFMPEG_VERSION_MISMATCH";
-export const PLAN_HASH_MISMATCH = "PLAN_HASH_MISMATCH";
-export const MISSING_PLAN_ARTIFACT = "MISSING_PLAN_ARTIFACT";
-export const CHUNK_INDEX_OUT_OF_RANGE = "CHUNK_INDEX_OUT_OF_RANGE";
-export const MISSING_RUNTIME_ENV_SNAPSHOT = "MISSING_RUNTIME_ENV_SNAPSHOT";
-export { INVALID_VIDEO_METADATA };
const LEGACY_DISTRIBUTED_VP9_CPU_USED = 2;
-export type RenderChunkValidationCode =
- | typeof FFMPEG_VERSION_MISMATCH
- | typeof PLAN_HASH_MISMATCH
- | typeof MISSING_PLAN_ARTIFACT
- | typeof CHUNK_INDEX_OUT_OF_RANGE
- | typeof MISSING_RUNTIME_ENV_SNAPSHOT
- | typeof INVALID_VIDEO_METADATA
- | typeof BROWSER_GPU_NOT_SOFTWARE;
-
-/**
- * Typed non-retryable error raised by `renderChunk` when the planDir is
- * malformed or the worker's runtime doesn't match the planDir's
- * controller-side fingerprint. Workflow adapters key retry policies off
- * `code` — most of these failures will not heal on retry.
- */
-export class RenderChunkValidationError extends Error {
- readonly code: RenderChunkValidationCode;
- constructor(code: RenderChunkValidationCode, message: string) {
- super(message);
- this.name = "RenderChunkValidationError";
- this.code = code;
- }
-}
-
/** Validate the shared video contract before any v1 chunk can inject frames. */
export function validatePlanVideosForChunk(value: unknown): PlanVideosJson {
try {
@@ -260,17 +237,6 @@ export async function createVerifiedDistributedCaptureSession(
}
}
-/**
- * Build the immutable lookup table once, but keep injector state scoped to a
- * browser session. Reusing one hook across workers or a whole-chunk retry can
- * incorrectly suppress injection on a fresh page.
- */
-export function createChunkVideoFrameInjectorFactory(
- frameLookup: FrameLookupTable | null,
-): () => BeforeCaptureHook | null {
- return () => createVideoFrameInjector(frameLookup);
-}
-
function isCaptureMode(value: string): value is CaptureMode {
return value === "beginframe" || value === "screenshot" || value === "drawelement";
}
@@ -333,173 +299,6 @@ export async function beginFrameSessionNeedsScreenshotFallback(
return !(await probe(session.page, timeoutMs, probeTick, session.beginFrameIntervalMs));
}
-function frameNumberFromFileName(name: string): number | null {
- const match = /(\d+)(?=\.[^.]+$)/.exec(name);
- if (!match) return null;
- const frameNumber = Number(match[1]);
- return Number.isSafeInteger(frameNumber) ? frameNumber : null;
-}
-
-/**
- * Rebuild the engine's in-memory `ExtractedFrames[]` from the on-disk
- * planDir layout. `<planDir>/video-frames/<videoId>/` holds the numbered
- * frame files plan() extracted; this lists each dir and rebuilds the
- * 0-based `framePaths` Map that `FrameLookupTable` / `videoFrameInjector`
- * both index against — the consumer is
- * `videoFrameExtractor.ts:getFrameAtTime`, which floors `localTime * fps`
- * to a 0-based index and reads `framePaths.get(frameIndex)`. Any drift
- * from that key convention silently drops every `<video>`'s first-paint
- * frame; see HF#1731 / HF#1730.
- *
- * Exported so a unit test can pin the 0-based contract without spinning
- * up the heavyweight Docker fixture — the bug surfaces only under
- * distributed mode and only at video first-paint, so this primitive is
- * the right granularity to guard.
- */
-export function rebuildExtractedFramesFromPlanDir(
- planDir: string,
- videos: PlanVideosJson["extracted"],
- indexMode: "dense-v1" | "sparse-v2" = "dense-v1",
-): ExtractedFrames[] {
- const result: ExtractedFrames[] = [];
- for (const v of videos) {
- const outputDir = join(planDir, "video-frames", v.videoId);
- if (!existsSync(outputDir)) {
- throw new Error(
- `[renderChunk] planDir missing extracted video frames for ${JSON.stringify(v.videoId)}: ` +
- `${outputDir} not present. plan() should have written frames here; the planDir is malformed.`,
- );
- }
- // framePattern looks like `frame_%05d.jpg`; sprintf isn't available at
- // runtime so list the directory and order numeric names by their ordinal.
- // Width changes once FFmpeg passes the padding minimum, so lexical order
- // would interleave frame_100000 before frame_10001.
- const ext = (extname(v.framePattern) || ".jpg").toLowerCase();
- const frames = readdirSync(outputDir)
- .filter((name) => name.toLowerCase().endsWith(ext))
- .sort((left, right) => {
- const leftNumber = frameNumberFromFileName(left);
- const rightNumber = frameNumberFromFileName(right);
- if (leftNumber === null || rightNumber === null) return left.localeCompare(right);
- return leftNumber - rightNumber || left.localeCompare(right);
- });
- const framePaths = new Map<number, string>();
- for (let i = 0; i < frames.length; i++) {
- const frameName = frames[i];
- if (!frameName) continue;
- // V1 plans preserve the historical sorted-position behavior even for
- // unusual zero-based filenames. V2 materialization is sparse, so only
- // that mode derives the original index from ffmpeg's 1-based filename.
- const frameNumber = indexMode === "sparse-v2" ? frameNumberFromFileName(frameName) : null;
- const frameIndex = frameNumber === null ? i : frameNumber - 1;
- framePaths.set(frameIndex, join(outputDir, frameName));
- }
- result.push({
- videoId: v.videoId,
- srcPath: v.srcPath,
- outputDir,
- framePattern: v.framePattern,
- fps: v.fps,
- totalFrames: v.totalFrames,
- metadata: v.metadata,
- framePaths,
- // The chunk worker doesn't own the planDir's video-frames/ directory
- // (the controller does — adapters that fan out chunks across machines
- // share the planDir as read-only). Mark ownership as false so the
- // injector's eventual cleanup doesn't rm bytes another worker may
- // still be reading.
- ownedByLookup: false,
- });
- }
- return result;
-}
-
-function contiguousRuns(sortedIndexes: readonly number[]): FrameRange[] {
- const runs: FrameRange[] = [];
- for (const index of sortedIndexes) {
- const last = runs[runs.length - 1];
- if (last && last.firstFrame + last.frames === index) last.frames++;
- else runs.push({ firstFrame: index, frames: 1 });
- }
- return runs;
-}
-
-/**
- * Extract the frames this chunk shows from videos the planner left
- * unextracted. Frames land in `outputRoot/<videoId>/` under the names a full
- * extraction gives them, so the lookup indexes them exactly as it would
- * frames the planner wrote.
- */
-export async function extractDeferredVideoFramesForChunk(input: {
- planDir: string;
- planVideos: PlanVideosJson;
- slice: ChunkSliceJson;
- fps: { num: number; den: number };
- outputRoot: string;
- cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;
-}): Promise<ExtractedFrames[]> {
- const deferred = input.planVideos.extracted.filter((video) => video.deferredRange);
- if (deferred.length === 0) return [];
- const table = createFrameLookupTable(
- input.planVideos.videos,
- deferred.map((video) => ({
- ...video,
- outputDir: input.outputRoot,
- framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),
- })),
- resolveRenderFpsConfig(input.fps).value,
- );
- const shown = new Map<string, Set<number>>();
- for (let frame = input.slice.startFrame; frame < input.slice.endFrame; frame++) {
- const globalTime = (frame * input.fps.den) / input.fps.num;
- for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {
- const indexes = shown.get(videoId) ?? new Set<number>();
- indexes.add(payload.frameIndex);
- shown.set(videoId, indexes);
- }
- }
- return Promise.all(
- deferred.map(async (video) => {
- const outputDir = join(input.outputRoot, video.videoId);
- const indexes = [...(shown.get(video.videoId) ?? [])].sort((a, b) => a - b);
- if (indexes.length === 0) {
- return { ...video, outputDir, framePaths: new Map<number, string>(), ownedByLookup: false };
- }
- const range = video.deferredRange!;
- const sourcePath = join(input.planDir, ...range.sourcePath.split("/"));
- if (!existsSync(sourcePath)) {
- throw new RenderChunkValidationError(
- MISSING_PLAN_ARTIFACT,
- `[renderChunk] planDir is missing the source for video ${JSON.stringify(video.videoId)}: ${range.sourcePath}`,
- );
- }
- const extracted = await extractVideoFramesRange(
- sourcePath,
- video.videoId,
- range.startTime,
- range.durationSeconds,
- {
- fps: input.fps,
- outputDir: input.outputRoot,
- format: range.format,
- frameRanges: contiguousRuns(indexes),
- },
- undefined,
- input.cfg,
- );
- const missing = indexes.find((index) => !extracted.framePaths.has(index));
- if (missing !== undefined) {
- throw new RenderChunkValidationError(
- INVALID_VIDEO_METADATA,
- `[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${missing}; ` +
- `the plan expected ${video.totalFrames} frames from ${range.sourcePath}.`,
- );
- }
- return { ...video, outputDir, framePaths: extracted.framePaths, ownedByLookup: false };
- }),
- );
-}
-
/** Plan-time JSON manifest written by `freezePlan`. */
interface PlanJson {
protocol?: unknown;
@@ -789,7 +588,10 @@ export async function renderChunk(
...(await extractDeferredVideoFramesForChunk({
planDir,
planVideos,
- slice,
+ frames: Array.from(
+ { length: slice.endFrame - slice.startFrame },
+ (_, i) => slice.startFrame + i,
+ ),
fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },
outputRoot: videoFramesDir,
cfg,
@@ -800,10 +602,6 @@ export async function renderChunk(
: null;
const createChunkVideoFrameInjector = createChunkVideoFrameInjectorFactory(videoFrameLookup);
- const videoCaptureBeyondViewport = resolveVideoCaptureBeyondViewport(
- planVideos?.videos.length ?? 0,
- );
-
// ── File server with the seeded-random shim ──
// `Math.random` / `crypto.getRandomValues` are seeded from virtual
// time so retries are pixel-identical. Only distributed renders flip this.
@@ -817,33 +615,12 @@ export async function renderChunk(
fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },
});
- const captureOptions: CaptureOptions = {
- width: plan.dimensions.width,
- height: plan.dimensions.height,
- fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },
- format: plan.dimensions.format === "mp4" ? "jpeg" : "png",
- quality:
- plan.dimensions.format === "mp4"
- ? resolveCaptureJpegQuality(job.config.quality)
- : undefined,
- deviceScaleFactor: encoder.deviceScaleFactor,
- // Re-inject the controller's snapshotted variables so the chunk's
- // first capture sees the same `window.__hfVariables` the in-process
- // renderer would have seen. Optional — compositions that don't
- // declare `data-composition-variables` leave this undefined and the
- // engine skips the `evaluateOnNewDocument` injection.
- variables: encoder.variables,
- subTimelineWaitMemo: {},
- ...(videoCaptureBeyondViewport !== undefined
- ? { captureBeyondViewport: videoCaptureBeyondViewport }
- : {}),
- // lock the BeginFrame warmup loop to a fixed iteration count so
- // `beginFrameTimeTicks` is host-independent. Only chunks ever set this.
- lockWarmupTicks: true,
- requiresWebGpu: compositionRequiresWebGpu(
- readFileSync(join(compiledDir, "index.html"), "utf-8"),
- ),
- };
+ const captureOptions = buildChunkCaptureOptions({
+ dimensions: plan.dimensions,
+ encoder,
+ videoCount: planVideos?.videos.length ?? 0,
+ html: readFileSync(join(compiledDir, "index.html"), "utf-8"),
+ });
// Resolve worker count up-front. Sequential capture reuses the initialized
// session below. Parallel BeginFrame capture pays one bounded preflight