Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(producer): extract distributed video frames in each chunk (#5290)
James Russo04f2bcd67a6f
diff --git a/docs/deploy/aws-lambda.mdx b/docs/deploy/aws-lambda.mdxindex 3a35e805cf..dc59d80158 100644--- a/docs/deploy/aws-lambda.mdx+++ b/docs/deploy/aws-lambda.mdx@@ -223,14 +223,16 @@ The render bucket is created with CloudFormation `Retain` on delete — `hyperfr- Each `--region` is an independent stack. Cross-region failover is not builtin.- The distributed AWS path is SDR-only.-- Video length is bounded by the plan step. One Lambda invocation extracts- every frame of every video and uploads them before any chunk starts, and it- must finish inside Lambda's 15-minute limit. On a 10,240 MB function, a- 26-minute 576×1024 24 fps H.264 clip plans in about 3.5 minutes. Larger- frames, high-bitrate footage and more videos take proportionally longer.- ffmpeg extracts long clips in 2-minute segments, so `FFMPEG_PROCESS_TIMEOUT_MS`- applies to each segment, not to the whole clip. Split edits that approach the- limit into separate renders.+- Each render chunk extracts only the video frames it shows, so the plan step+ no longer grows with video length. On a 10,240 MB function, a 26-minute+ 576×1024 24 fps H.264 clip plans in about 20 seconds. Each chunk downloads+ the whole source file of every video it shows. Variable-frame-rate and HDR+ sources are the exception: the plan step still extracts every frame of those,+ inside Lambda's 15-minute limit, in 2-minute segments, so+ `FFMPEG_PROCESS_TIMEOUT_MS` applies to each segment, not to the whole clip.+- A chunk must also finish inside 15 minutes. For long renders, raise+ `--max-parallel-chunks` (`maxParallelChunks` in the SDK) so each chunk covers+ fewer frames.## Related topicsdiff --git a/packages/engine/src/index.ts b/packages/engine/src/index.tsindex bb01945821..0ff6cbf4af 100644--- a/packages/engine/src/index.ts+++ b/packages/engine/src/index.ts@@ -221,6 +221,8 @@ export {type VideoElement,type ImageElement,type ExtractedFrames,+ type DeferredFrameRange,+ type FrameRange,type ExtractionOptions,type ExtractionResult,type ExtractionPhaseBreakdown,diff --git a/packages/engine/src/services/extractedFrameIndex.ts b/packages/engine/src/services/extractedFrameIndex.tsindex af6e1bde54..a041033ac1 100644--- a/packages/engine/src/services/extractedFrameIndex.ts+++ b/packages/engine/src/services/extractedFrameIndex.ts@@ -27,6 +27,7 @@ export function extractedFrameIndex(file: string, format: ExtractedFrameFormat):export function framePathsFromDirectory(outputDir: string,format: ExtractedFrameFormat,+ allowGaps = false,): Map<number, string> {const suffix = `.${format}`;const indexed = new Map<number, string>();@@ -40,6 +41,7 @@ export function framePathsFromDirectory(}indexed.set(index, join(outputDir, file));}+ if (allowGaps) return new Map([...indexed].sort(([a], [b]) => a - b));const ordered = new Map<number, string>();for (let index = 0; index < indexed.size; index += 1) {diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.tsindex ad0f9c6dc1..2f917e9a3d 100644--- a/packages/engine/src/services/videoFrameExtractor.test.ts+++ b/packages/engine/src/services/videoFrameExtractor.test.ts@@ -1926,6 +1926,112 @@ describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => {},60_000,);++ // A distributed chunk extracts only the frames it renders; they must be the full extraction's.+ describe("partial extraction", () => {+ const gopSource = join(FIXTURE_DIR, "gop-30fps.mp4");++ beforeAll(async () => {+ // Real motion and long GOPs, so range seeks land mid-GOP.+ const synth = await runFfmpeg([+ "-y",+ "-hide_banner",+ "-loglevel",+ "error",+ "-f",+ "lavfi",+ "-i",+ "testsrc2=s=64x36:r=30:d=20",+ "-c:v",+ "libx264",+ "-g",+ "250",+ "-pix_fmt",+ "yuv420p",+ gopSource,+ ]);+ if (!synth.success) throw new Error(`gop fixture synthesis failed: ${synth.stderr}`);+ }, 30_000);++ function frameBytes(dir: string): Map<string, Buffer> {+ return new Map(readdirSync(dir).map((name) => [name, readFileSync(join(dir, name))]));+ }++ it.each([+ { fps: 24, startTime: 1.3, duration: 17 },+ { fps: 60, startTime: 0, duration: 20 },+ ])(+ "frameRanges at $fps fps write the same files as a full extraction",+ async (c) => {+ const full = await extractVideoFramesRange(+ gopSource,+ `full-${c.fps}`,+ c.startTime,+ c.duration,+ { fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg" },+ );+ const last = full.totalFrames;+ const ranges = [+ { firstFrame: 0, frames: 5 },+ { firstFrame: 97, frames: 40 },+ { firstFrame: last - 3, frames: 3 },+ ];+ const partial = await extractVideoFramesRange(+ gopSource,+ `partial-${c.fps}`,+ c.startTime,+ c.duration,+ { fps: c.fps, outputDir: FIXTURE_DIR, format: "jpg", frameRanges: ranges },+ );+ const fullFrames = frameBytes(full.outputDir);+ const partialFrames = frameBytes(partial.outputDir);+ expect(partialFrames.size).toBe(48);+ for (const [name, bytes] of partialFrames) {+ expect(fullFrames.get(name)?.equals(bytes), name).toBe(true);+ }+ },+ 60_000,+ );++ it.each([+ { fps: 24, mediaStart: 3.3 },+ { fps: 30, mediaStart: 0 },+ { fps: 60, mediaStart: 7.77 },+ ])(+ "a deferred window to the source end at $fps fps reports the full frame count",+ async (c) => {+ const video = (): VideoElement => ({+ id: `defer-${c.fps}`,+ src: gopSource,+ start: 0,+ end: Infinity,+ mediaStart: c.mediaStart,+ loop: false,+ hasAudio: false,+ });+ const full = await extractAllVideoFrames([video()], FIXTURE_DIR, {+ fps: c.fps,+ outputDir: join(FIXTURE_DIR, `defer-full-${c.fps}`),+ });+ const deferred = await extractAllVideoFrames([video()], FIXTURE_DIR, {+ fps: c.fps,+ outputDir: join(FIXTURE_DIR, `defer-${c.fps}`),+ deferRangeExtraction: true,+ });+ expect(deferred.errors).toEqual([]);+ const result = deferred.extracted[0]!;+ expect(result.totalFrames).toBe(full.extracted[0]!.totalFrames);+ expect(result.framePaths.size).toBe(0);+ expect(readdirSync(result.outputDir)).toEqual([]);+ expect(result.deferredRange).toEqual({+ startTime: c.mediaStart,+ durationSeconds: expect.any(Number),+ format: "jpg",+ });+ },+ 60_000,+ );+ });});describe.skipIf(!HAS_FFMPEG)("held tails on sparse-timestamp sources", () => {diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.tsindex 4e196c1707..1dfee36d99 100644--- a/packages/engine/src/services/videoFrameExtractor.ts+++ b/packages/engine/src/services/videoFrameExtractor.ts@@ -97,6 +97,7 @@ export interface ExtractedFrames {totalFrames: number;metadata: VideoMetadata;framePaths: Map<number, string>;+ deferredRange?: DeferredFrameRange;/*** True when the extractor owns `outputDir` and cleanup should rm it when* the render ends. Cache hits set this to false so the shared entry isn't@@ -194,6 +195,18 @@ export function extractionFrameCountForDuration(return Math.max(1, Number.isSafeInteger(frames) ? frames : Number.MAX_SAFE_INTEGER);}+export interface FrameRange {+ firstFrame: number;+ frames: number;+}++/** The window a deferred extraction samples; frame k is the source frame on screen at `startTime + k / fps`. */+export interface DeferredFrameRange {+ startTime: number;+ durationSeconds: number;+ format: "jpg" | "png";+}+export interface ExtractionOptions {/** Exact configured rate. Rational rates are passed to FFmpeg verbatim. */fps: FpsInput;@@ -222,6 +235,13 @@ export interface ExtractionOptions {* candidate enforce lane may opt into typed aggregation.*/collectProbeFailures?: boolean;+ /** Extract only these runs of output frames, named as a full extraction names them. CFR only. */+ frameRanges?: readonly FrameRange[];+ /**+ * Leave CFR windows longer than a second unextracted, returning `deferredRange` and an exact+ * `totalFrames` so another process can extract just the frames it needs with `frameRanges`.+ */+ deferRangeExtraction?: boolean;}export const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;@@ -870,16 +890,28 @@ interface SegmentedExtraction {filterAndEncodeArgs: string[];videoPath: string;startTime: number;- totalFrames: number;fps: number;- segmentFrames: number;+ segments: readonly FrameRange[];runOptions: { signal?: AbortSignal; timeout: number };}+/** Split runs so no single ffmpeg process extracts more than `segmentFrames` frames. */+function splitFrameRanges(ranges: readonly FrameRange[], segmentFrames: number): FrameRange[] {+ const segments: FrameRange[] = [];+ for (const range of ranges) {+ for (let offset = 0; offset < range.frames; offset += segmentFrames) {+ segments.push({+ firstFrame: range.firstFrame + offset,+ frames: Math.min(segmentFrames, range.frames - offset),+ });+ }+ }+ return segments;+}+async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmpegResult> {- const { decodeArgs, filterAndEncodeArgs, videoPath, startTime, totalFrames, fps, segmentFrames } =- job;- const segmentCount = Math.ceil(totalFrames / segmentFrames);+ const { decodeArgs, filterAndEncodeArgs, videoPath, startTime, fps, segments } = job;+ const segmentCount = segments.length;// One failed segment fails the range, so stop the others instead of finishing them.const failed = new AbortController();const signal = job.runOptions.signal@@ -890,8 +922,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmpconst worker = async () => {while (next < segmentCount && !signal.aborted) {const index = next++;- const firstFrame = index * segmentFrames;- const frames = Math.min(segmentFrames, totalFrames - firstFrame);+ const { firstFrame, frames } = segments[index]!;const result = await runFfmpeg([...decodeArgs,@@ -991,6 +1022,9 @@ export async function extractVideoFramesRange(const decodeWithVideoToolbox = isHdr && isMacOS && !toneMappedToSdr;const sampleCfrAtOutputFps = !options.finalFrameOnly && !metadata.isVFR;+ if (options.frameRanges && !sampleCfrAtOutputFps) {+ throw new Error("frameRanges extraction needs a constant-frame-rate source");+ }const args: string[] = [];if (decodeWithVideoToolbox) {@@ -1092,15 +1126,17 @@ export async function extractVideoFramesRange(const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps);const totalFrames = extractionFrameCountForDuration(duration, normalizedFps, false);processResult =- sampleCfrAtOutputFps && totalFrames > segmentFrames+ sampleCfrAtOutputFps && (options.frameRanges || totalFrames > segmentFrames)? await runSegmentedExtraction({decodeArgs,filterAndEncodeArgs: [...filterArgs, ...encodeArgs],videoPath,startTime,- totalFrames,fps,- segmentFrames,+ segments: splitFrameRanges(+ options.frameRanges ?? [{ firstFrame: 0, frames: totalFrames }],+ segmentFrames,+ ),runOptions,}): await runFfmpeg([...args, ...filterArgs, ...encodeArgs], runOptions);@@ -1154,7 +1190,7 @@ export async function extractVideoFramesRange();}- const framePaths = framePathsFromDirectory(videoOutputDir, format);+ const framePaths = framePathsFromDirectory(videoOutputDir, format, !!options.frameRanges);if (framePaths.size === 0 && duration > 0) {throw new VideoSourceExtractionError("zero_output",@@ -1168,7 +1204,7 @@ export async function extractVideoFramesRange(// frames predicted by the probed duration, the duration is likely// inflated (container duration includes a longer audio track). Correct// it so downstream coverage accounting uses the actual video extent.- if (framePaths.size > 0 && duration > 0 && !options.finalFrameOnly) {+ if (framePaths.size > 0 && duration > 0 && !options.finalFrameOnly && !options.frameRanges) {const expectedAtFps = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);if (expectedAtFps > 0 && framePaths.size <= expectedAtFps * 0.6) {const correctedDuration = startTime + framePaths.size / fps;@@ -2186,6 +2222,68 @@ export async function extractAllVideoFrames(return rehydratePublishedCache(work, cacheTarget);}+ /**+ * Every output slot before the source's end holds a frame, so decoding the last second gives+ * the exact count. Anything unusual returns null and takes the normal path.+ */+ async function deferExtraction(work: PreparedExtraction): Promise<ExtractedFrames | null> {+ if (+ work.finalFrameOnly ||+ work.metadata.isVFR ||+ work.sdrToHdrTransfer !== undefined ||+ work.hdrToSdrTransformKey !== undefined+ ) {+ return null;+ }+ const expectedFrames = extractionFrameCountForDuration(+ work.videoDuration,+ configuredFps,+ false,+ );+ const tailFrames = Math.min(expectedFrames, Math.ceil(fps));+ if (expectedFrames <= tailFrames) return null;+ const probeDir = join(options.outputDir, `${work.video.id}.tail-probe`);+ try {+ const tail = await extractVideoFramesRange(+ work.videoPath,+ work.video.id,+ work.extractionMediaStart,+ work.videoDuration,+ {+ ...scopedExtractionOptions(work),+ frameRanges: [{ firstFrame: expectedFrames - tailFrames, frames: tailFrames }],+ },+ signal,+ config,+ probeDir,+ );+ const totalFrames = expectedFrames - tailFrames + tail.totalFrames;+ // A full extraction this short rewrites the probed duration; let it.+ if (totalFrames <= expectedFrames * 0.6) return null;+ const outputDir = join(options.outputDir, work.video.id);+ mkdirSync(outputDir, { recursive: true });+ return {+ videoId: work.video.id,+ srcPath: work.videoPath,+ outputDir,+ framePattern: `${FRAME_FILENAME_PREFIX}%05d.${work.format}`,+ fps,+ totalFrames,+ metadata: work.metadata,+ framePaths: new Map(),+ deferredRange: {+ startTime: work.extractionMediaStart,+ durationSeconds: work.videoDuration,+ format: work.format,+ },+ };+ } catch {+ return null;+ } finally {+ rmSync(probeDir, { recursive: true, force: true });+ }+ }+async function executeDirectMiss(miss: UniqueExtractionMiss,maxTransientRetries = options.maxTransientRetries ?? 0,@@ -2350,8 +2448,17 @@ export async function extractAllVideoFrames(}const uniqueOutcomes = new Map<string, ExtractionOutcome>();+ if (options.deferRangeExtraction) {+ const works = [...uniqueWorks.values()];+ const deferred = await Promise.all(works.map(deferExtraction));+ works.forEach((work, i) => {+ const result = deferred[i];+ if (result) uniqueOutcomes.set(work.dedupeKey, { result });+ });+ }const cacheMisses: UniqueExtractionMiss[] = [];for (const work of uniqueWorks.values()) {+ if (uniqueOutcomes.has(work.dedupeKey)) continue;const lookup = lookupCacheFor(work);if ("work" in lookup) {cacheMisses.push(lookup);@@ -2419,7 +2526,7 @@ export async function extractAllVideoFrames(errors.push(item.error);} else if ("result" in item) {extracted.push(item.result);- totalFramesExtracted += item.result.totalFrames;+ if (!item.result.deferredRange) totalFramesExtracted += item.result.totalFrames;}}diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.tsindex 93edeafa31..fe7a6db4b8 100644--- a/packages/producer/src/services/distributed/plan.ts+++ b/packages/producer/src/services/distributed/plan.ts@@ -26,8 +26,17 @@* never have to handle them.*/-import { cpSync, existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";-import { join, relative, sep } from "node:path";+import {+ copyFileSync,+ cpSync,+ existsSync,+ linkSync,+ mkdirSync,+ renameSync,+ rmSync,+ writeFileSync,+} from "node:fs";+import { extname, isAbsolute, join, relative, sep } from "node:path";import { type CanvasResolution, fpsToNumber } from "@hyperframes/core";import {type EngineConfig,@@ -276,6 +285,12 @@ export interface BuildLocalExecutionPlanOptions {* transport cap because it publishes role-scoped content-addressed objects.*/readonly executionPlanSizeLimitBytes?: number;+ /**+ * Leave long constant-frame-rate video windows unextracted and ship their+ * sources, so each chunk extracts only the frames it renders. Plan v2 only:+ * chunks write those frames into their own work directory.+ */+ readonly deferVideoExtraction?: boolean;}/**@@ -825,6 +840,53 @@ export function resolveDistributedEngineConfig(config: DistributedRenderConfig):};}+function pathInside(root: string, path: string): string | null {+ const rel = relative(root, path);+ return rel === "" || rel.startsWith("..") || isAbsolute(rel) ? null : rel;+}++/**+ * Return a planDir-relative path chunks can extract a deferred video from.+ * Sources the plan already ships under `compiled/` are reused; anything else+ * (downloads, files outside the project) is linked into `video-sources/`.+ * Call after the staged `compiled/` and `video-frames/` trees are promoted.+ */+function createVideoSourceShipper(input: {+ planDir: string;+ projectDir: string;+ workCompiledDir: string;+ workVideoFramesDir: string;+}): (srcPath: string) => string {+ const shipped = new Map<string, string>();+ return (srcPath) => {+ const known = shipped.get(srcPath);+ if (known) return known;+ // Downloads were staged under the frame tree, which now lives at video-frames/.+ const stagedRel = pathInside(input.workVideoFramesDir, srcPath);+ const current = stagedRel === null ? srcPath : join(input.planDir, "video-frames", stagedRel);+ let result: string | null = null;+ for (const root of stagedRel === null ? [input.workCompiledDir, input.projectDir] : []) {+ const rel = pathInside(root, srcPath);+ if (rel !== null && existsSync(join(input.planDir, "compiled", rel))) {+ result = ["compiled", ...rel.split(sep)].join("/");+ break;+ }+ }+ if (result === null) {+ result = `video-sources/${shipped.size}${extname(current)}`;+ const destination = join(input.planDir, ...result.split("/"));+ mkdirSync(join(input.planDir, "video-sources"), { recursive: true });+ try {+ linkSync(current, destination);+ } catch {+ copyFileSync(current, destination);+ }+ }+ shipped.set(srcPath, result);+ return result;+ };+}+/*** Build the shared local execution representation used by both transport* protocols. See the module docstring for the directory layout.@@ -1043,6 +1105,7 @@ export async function buildLocalExecutionPlan(abortSignal,assertNotAborted,materializeSymlinks: true,+ deferRangeExtraction: options.deferVideoExtraction,});if (extractResult.failureToEnforce) throw extractResult.failureToEnforce;if (extractResult.extractionResult) {@@ -1082,6 +1145,7 @@ export async function buildLocalExecutionPlan(const stagedVideoFrames = join(compiledDir, "__hyperframes_video_frames");const videoFramesDst = join(planDir, "video-frames");if (existsSync(videoFramesDst)) rmSync(videoFramesDst, { recursive: true, force: true });+ rmSync(join(planDir, "video-sources"), { recursive: true, force: true });if (existsSync(stagedVideoFrames)) {renameSync(stagedVideoFrames, videoFramesDst);} else {@@ -1097,6 +1161,12 @@ export async function buildLocalExecutionPlan(// page's native `<video>` element decodes the source mp4 ~1 frame// off the pre-extracted images the in-process baseline was captured// from.+ const shipVideoSource = createVideoSourceShipper({+ planDir,+ projectDir,+ workCompiledDir: compiledDir,+ workVideoFramesDir: stagedVideoFrames,+ });const planVideosJson: PlanVideosJson = buildPlanVideosJson({videos: composition.videos,compositionEnd: job.duration ?? Number.NaN,@@ -1107,6 +1177,9 @@ export async function buildLocalExecutionPlan(fps: ext.fps,totalFrames: ext.totalFrames,metadata: ext.metadata,+ ...(ext.deferredRange+ ? { deferredRange: { ...ext.deferredRange, sourcePath: shipVideoSource(ext.srcPath) } }+ : {}),})),});mkdirSync(join(planDir, "meta"), { recursive: true });diff --git a/packages/producer/src/services/distributed/planV2.test.ts b/packages/producer/src/services/distributed/planV2.test.tsindex 11fbe1ec85..287e4a51cf 100644--- a/packages/producer/src/services/distributed/planV2.test.ts+++ b/packages/producer/src/services/distributed/planV2.test.ts@@ -11,7 +11,7 @@ import {writeFileSync,} from "node:fs";import { tmpdir } from "node:os";-import { join } from "node:path";+import { dirname, join } from "node:path";import { recomputePlanHashFromPlanDir } from "../render/stages/freezePlan.js";import { canonicalJsonStringify, sha256Hex } from "../render/stages/planHash.js";import { CURRENT_PLAN_PROTOCOL } from "./planProtocol.js";@@ -65,6 +65,8 @@ function createV1Plan(framePattern?: string;videoStart?: number;videoEnd?: number;+ /** Ship this planDir-relative source instead of extracted frames. */+ deferredSource?: string;},): string {const {@@ -79,6 +81,7 @@ function createV1Plan(framePattern = "frame_%05d.jpg",videoStart = 0,videoEnd = 1,+ deferredSource,} = options ?? {};const planDir = join(root, "v1");mkdirSync(join(planDir, "compiled"), { recursive: true });@@ -97,9 +100,14 @@ function createV1Plan(if (video) {const framesDir = join(planDir, "video-frames", "hero");mkdirSync(framesDir, { recursive: true });- writeFileSync(join(framesDir, "frame_00001.jpg"), "frame zero");- writeFileSync(join(framesDir, "frame_00002.jpg"), "frame one");- writeFileSync(join(framesDir, "frame_00003.jpg"), "never rendered");+ if (deferredSource === undefined) {+ writeFileSync(join(framesDir, "frame_00001.jpg"), "frame zero");+ writeFileSync(join(framesDir, "frame_00002.jpg"), "frame one");+ writeFileSync(join(framesDir, "frame_00003.jpg"), "never rendered");+ } else if (!deferredSource.startsWith("..")) {+ mkdirSync(dirname(join(planDir, deferredSource)), { recursive: true });+ writeFileSync(join(planDir, deferredSource), "source video");+ }writeFileSync(join(planDir, "meta", "videos.json"),JSON.stringify({@@ -133,6 +141,16 @@ function createV1Plan(hasAlpha: false,colorSpace: videoColorSpace,},+ ...(deferredSource === undefined+ ? {}+ : {+ deferredRange: {+ sourcePath: deferredSource,+ startTime: 0,+ durationSeconds: 0.1,+ format: "jpg",+ },+ }),},],}),@@ -409,6 +427,49 @@ describe("Plan v2 manifest", () => {);});+ it("ships a deferred video's source only to the chunks that show it", () => {+ const root = tempPath("hf-plan-v2-deferred-source-");+ const v1 = createV1Plan(root, {+ video: true,+ videoStart: 1 / 30,+ deferredSource: "video-sources/0.mp4",+ });+ const manifest = readPlanV2Manifest(createPlanV2FromV1(v1, join(root, "v2")).planDir);+ const chunk0 = listPlanV2ArtifactsForTarget(manifest, { role: "chunk", chunkIndex: 0 });+ const chunk1 = listPlanV2ArtifactsForTarget(manifest, { role: "chunk", chunkIndex: 1 });+ const assembler = listPlanV2ArtifactsForTarget(manifest, { role: "assembler" });++ expect(manifest.limitations.videoDependencyMode).toBe("source-extract");+ expect(chunk0.some((artifact) => artifact.path === "video-sources/0.mp4")).toBe(false);+ expect(chunk1.some((artifact) => artifact.path === "video-sources/0.mp4")).toBe(true);+ expect(assembler.some((artifact) => artifact.path === "video-sources/0.mp4")).toBe(false);+ expect(manifest.artifacts.some((artifact) => artifact.path.startsWith("video-frames/"))).toBe(+ false,+ );+ });++ it("keeps a deferred source the page itself loads in every chunk", () => {+ const root = tempPath("hf-plan-v2-deferred-compiled-source-");+ const v1 = createV1Plan(root, {+ video: true,+ videoStart: 1 / 30,+ deferredSource: "compiled/hero.mp4",+ });+ const manifest = readPlanV2Manifest(createPlanV2FromV1(v1, join(root, "v2")).planDir);++ expect(manifest.limitations.videoDependencyMode).toBe("source-extract");+ for (const chunkIndex of [0, 1]) {+ const artifacts = listPlanV2ArtifactsForTarget(manifest, { role: "chunk", chunkIndex });+ expect(artifacts.some((artifact) => artifact.path === "compiled/hero.mp4")).toBe(true);+ }+ });++ it("rejects a deferred video source outside the plan directory", () => {+ const root = tempPath("hf-plan-v2-deferred-escape-");+ const v1 = createV1Plan(root, { video: true, deferredSource: "../outside.mp4" });+ expect(() => createPlanV2FromV1(v1, join(root, "v2"))).toThrow(PlanV2IntegrityError);+ });+it("gives a chunk the frames of a video starting a hair after its frame, as export shows it", () => {const root = tempPath("hf-plan-v2-snapped-start-");const v1 = createV1Plan(root, { video: true, videoStart: 1 / 30 + 1e-6, videoEnd: 1 });diff --git a/packages/producer/src/services/distributed/planV2.ts b/packages/producer/src/services/distributed/planV2.tsindex 6233e383bd..730daa9593 100644--- a/packages/producer/src/services/distributed/planV2.ts+++ b/packages/producer/src/services/distributed/planV2.ts@@ -103,7 +103,7 @@ export interface PlanV2Manifest {}export interface PlanV2Limitations {- readonly videoDependencyMode: "exact-rendered-frames" | "full-source-pack";+ readonly videoDependencyMode: "exact-rendered-frames" | "full-source-pack" | "source-extract";}export interface PlanV2Result {@@ -288,6 +288,8 @@ function artifactTargets(if (path === "plan.json" || path === "meta/chunks.json" || path === "meta/encoder.json") {return { chunks: "all", assembler: true };}+ // Each chunk's page loads every <video> src at init (a missing one fails the+ // chunk), so deferred sources under compiled/ still go to every chunk.if (path === "meta/composition.json" ||path === "meta/videos.json" ||@@ -295,7 +297,7 @@ function artifactTargets() {return { chunks: "all", assembler: false };}- if (path.startsWith("video-frames/")) {+ if (path.startsWith("video-frames/") || path.startsWith("video-sources/")) {return {chunks: videoDependencies === null ? "all" : (videoDependencies.get(path) ?? []),assembler: false,@@ -309,6 +311,13 @@ function artifactTargets(function listVideoFramePaths(executionPlanDir: string, videos: PlanVideosJson): ExtractedFrames[] {return videos.extracted.map((video) => {const outputDir = resolveExtractedVideoOutputDir(executionPlanDir, video.videoId);+ if (video.deferredRange) {+ const framePaths = new Map<number, string>();+ for (let index = 0; index < video.totalFrames; index++) {+ framePaths.set(index, join(outputDir, String(index)));+ }+ return { ...video, outputDir, framePaths, ownedByLookup: false };+ }const frameNames = readdirSync(outputDir).sort();const framePaths = new Map<number, string>();for (const frameName of frameNames) {@@ -391,7 +400,7 @@ function buildVideoChunkDependencies(executionPlanDir: string,dimensions: Record<string, unknown>,): {- mode: "exact-rendered-frames" | "full-source-pack";+ mode: PlanV2Limitations["videoDependencyMode"];dependencies: ReadonlyMap<string, readonly number[]> | null;} {const videoRoot = join(executionPlanDir, "video-frames");@@ -415,13 +424,41 @@ function buildVideoChunkDependencies(extracted,resolveRenderFpsConfig({ num: fpsNum, den: fpsDen }).value,);- const mutable = new Map<string, Set<number>>();+ const deferredSources = new Map(+ parsedVideos.extracted.flatMap((video) =>+ video.deferredRange ? [[video.videoId, video.deferredRange.sourcePath] as const] : [],+ ),+ );+ const dependencies = collectChunkVideoDependencies({+ executionPlanDir,+ table,+ chunks: parsedChunks,+ fpsNum,+ fpsDen,+ deferredSources,+ });+ return {+ mode: deferredSources.size > 0 ? "source-extract" : "exact-rendered-frames",+ dependencies,+ };+}- for (const chunk of parsedChunks) {+function collectChunkVideoDependencies(input: {+ executionPlanDir: string;+ table: ReturnType<typeof createFrameLookupTable>;+ chunks: readonly ChunkSliceJson[];+ fpsNum: number;+ fpsDen: number;+ deferredSources: ReadonlyMap<string, string>;+}): ReadonlyMap<string, readonly number[]> {+ const mutable = new Map<string, Set<number>>();+ for (const chunk of input.chunks) {for (let frame = chunk.startFrame; frame < chunk.endFrame; frame++) {- const globalTime = (frame * fpsDen) / fpsNum;- for (const payload of table.getActiveFramePayloads(globalTime).values()) {- const path = relative(resolve(executionPlanDir), payload.framePath).split(sep).join("/");+ const globalTime = (frame * input.fpsDen) / input.fpsNum;+ for (const [videoId, payload] of input.table.getActiveFramePayloads(globalTime)) {+ const path =+ input.deferredSources.get(videoId) ??+ relative(resolve(input.executionPlanDir), payload.framePath).split(sep).join("/");assertSafeRelativePath(path);const indexes = mutable.get(path) ?? new Set<number>();indexes.add(chunk.index);@@ -429,12 +466,7 @@ function buildVideoChunkDependencies(}}}- return {- mode: "exact-rendered-frames",- dependencies: new Map(- [...mutable].map(([path, indexes]) => [path, [...indexes].sort((a, b) => a - b)]),- ),- };+ return new Map([...mutable].map(([path, indexes]) => [path, [...indexes].sort((a, b) => a - b)]));}function manifestPayload(@@ -504,7 +536,7 @@ function buildPlanV2Publication(executionPlanDir: string): PlanV2Publication {if (isExtractionCacheCompleteSentinelPath(file.path)) continue;const targets = artifactTargets(file.path, videoDependencyPlan.dependencies);if (- file.path.startsWith("video-frames/") &&+ (file.path.startsWith("video-frames/") || file.path.startsWith("video-sources/")) &&targets.chunks !== "all" &&targets.chunks.length === 0 &&!targets.assembler@@ -648,6 +680,7 @@ export async function planV2WithPublisher(try {await buildLocalExecutionPlan(projectDir, config, stagingRoot, {executionPlanSizeLimitBytes: Number.MAX_SAFE_INTEGER,+ deferVideoExtraction: true,});return await publishPlanV2FromExecutionPlan(stagingRoot, publisher);} catch (error) {@@ -814,7 +847,8 @@ function parsePlanV2Manifest(value: unknown): Readonly<PlanV2Manifest> {if (!isRecord(value.limitations) ||(value.limitations.videoDependencyMode !== "exact-rendered-frames" &&- value.limitations.videoDependencyMode !== "full-source-pack")+ value.limitations.videoDependencyMode !== "full-source-pack" &&+ value.limitations.videoDependencyMode !== "source-extract")) {throw new PlanV2IntegrityError("unsupported video dependency mode");}diff --git a/packages/producer/src/services/distributed/renderChunk.test.ts b/packages/producer/src/services/distributed/renderChunk.test.tsindex f4ab4a3795..ad1dff4c4b 100644--- a/packages/producer/src/services/distributed/renderChunk.test.ts+++ b/packages/producer/src/services/distributed/renderChunk.test.ts@@ -19,16 +19,18 @@*/import { afterAll, beforeAll, describe, expect, it } from "bun:test";-import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { spawnSync } from "node:child_process";+import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";-import type { CaptureOptions } from "@hyperframes/engine";+import { type CaptureOptions, extractVideoFramesRange } from "@hyperframes/engine";import { HOST_CHROME_FAILURE_PATTERNS } from "./__test_utils__/hostChromeFailures.js";import { RenderQualityError } from "../renderOrchestrator.js";import { plan } from "./plan.js";import {CHUNK_INDEX_OUT_OF_RANGE,distributedCaptureSessionDependencies,+ extractDeferredVideoFramesForChunk,MISSING_PLAN_ARTIFACT,MISSING_RUNTIME_ENV_SNAPSHOT,PLAN_HASH_MISMATCH,@@ -579,3 +581,140 @@ describe("resolveLockedVp9CpuUsed", () => {expect(resolveLockedVp9CpuUsed({ encoder: "libx264-software" })).toBeUndefined();});});++describe("extractDeferredVideoFramesForChunk", () => {+ const root = mkdtempSync(join(tmpdir(), "hf-deferred-chunk-"));+ const planDir = join(root, "plan");+ const range = {+ sourcePath: "video-sources/0.mp4",+ startTime: 1.5,+ durationSeconds: 2.5,+ format: "jpg" as const,+ };+ const metadata = {+ durationSeconds: 4,+ videoStreamDurationSeconds: 4,+ videoStreamStartSeconds: 0,+ width: 64,+ height: 36,+ fps: 30,+ videoCodec: "h264",+ hasAudio: false,+ isVFR: false,+ hasAlpha: false,+ colorSpace: null,+ };+ const planVideos = (loop: boolean) => ({+ videos: [+ { id: "clip", src: "clip.mp4", start: 0, end: 4, mediaStart: 1.5, loop, hasAudio: false },+ ],+ extracted: [+ {+ videoId: "clip",+ srcPath: "/planner/clip.mp4",+ framePattern: "frame_%05d.jpg",+ fps: 30,+ totalFrames: 75,+ metadata,+ deferredRange: range,+ },+ ],+ });++ beforeAll(() => {+ mkdirSync(join(planDir, "video-sources"), { recursive: true });+ const synth = spawnSync("ffmpeg", [+ "-v",+ "error",+ "-y",+ "-f",+ "lavfi",+ "-i",+ "testsrc2=s=64x36:r=30:d=4",+ "-c:v",+ "libx264",+ "-g",+ "60",+ "-pix_fmt",+ "yuv420p",+ join(planDir, range.sourcePath),+ ]);+ if (synth.status !== 0) throw new Error(synth.stderr.toString());+ });++ afterAll(() => {+ rmSync(root, { recursive: true, force: true });+ });++ it("extracts exactly the frames the chunk shows, identical to the planner's", async () => {+ const full = await extractVideoFramesRange(+ join(planDir, range.sourcePath),+ "clip",+ range.startTime,+ range.durationSeconds,+ { fps: 30, outputDir: join(root, "full"), format: "jpg" },+ );+ const [chunk] = await extractDeferredVideoFramesForChunk({+ planDir,+ planVideos: planVideos(false),+ slice: { index: 1, startFrame: 30, endFrame: 45 },+ fps: { num: 30, den: 1 },+ outputRoot: join(root, "chunk-1"),+ cfg: {},+ });++ expect([...chunk!.framePaths.keys()]).toEqual(Array.from({ length: 15 }, (_, i) => 30 + i));+ for (const [index, path] of chunk!.framePaths) {+ expect(readFileSync(path).equals(readFileSync(full.framePaths.get(index)!))).toBe(true);+ }+ });++ it("extracts nothing for a chunk past the clip, and holds no frames", async () => {+ const [chunk] = await extractDeferredVideoFramesForChunk({+ planDir,+ planVideos: { ...planVideos(false), videos: [{ ...planVideos(false).videos[0]!, end: 1 }] },+ slice: { index: 2, startFrame: 60, endFrame: 90 },+ fps: { num: 30, den: 1 },+ outputRoot: join(root, "chunk-2"),+ cfg: {},+ });+ expect(chunk!.framePaths.size).toBe(0);+ });++ it("wraps a looping clip back to its first frames", async () => {+ const [chunk] = await extractDeferredVideoFramesForChunk({+ planDir,+ planVideos: planVideos(true),+ slice: { index: 3, startFrame: 70, endFrame: 80 },+ fps: { num: 30, den: 1 },+ outputRoot: join(root, "chunk-3"),+ cfg: {},+ });+ expect([...chunk!.framePaths.keys()].sort((a, b) => a - b)).toEqual([+ 0, 1, 2, 3, 4, 70, 71, 72, 73, 74,+ ]);+ expect(readdirSync(join(root, "chunk-3", "clip"))).toHaveLength(10);+ });++ it("fails closed when the plan lacks the source", async () => {+ const missing = { ...planVideos(false) };+ missing.extracted = [+ { ...missing.extracted[0]!, deferredRange: { ...range, sourcePath: "video-sources/9.mp4" } },+ ];+ let caught: unknown;+ try {+ await extractDeferredVideoFramesForChunk({+ planDir,+ planVideos: missing,+ slice: { index: 0, startFrame: 0, endFrame: 5 },+ fps: { num: 30, den: 1 },+ outputRoot: join(root, "chunk-missing"),+ cfg: {},+ });+ } catch (error) {+ caught = error;+ }+ expect(caught).toBeInstanceOf(RenderChunkValidationError);+ expect((caught as RenderChunkValidationError).code).toBe(MISSING_PLAN_ARTIFACT);+ });+});diff --git a/packages/producer/src/services/distributed/renderChunk.ts b/packages/producer/src/services/distributed/renderChunk.tsindex 12d52af801..0c20ee79ff 100644--- a/packages/producer/src/services/distributed/renderChunk.ts+++ b/packages/producer/src/services/distributed/renderChunk.ts@@ -55,6 +55,8 @@ import {deriveBeginFrameProbeTimeTicks,type EngineConfig,type ExtractedFrames,+ extractVideoFramesRange,+ type FrameRange,type FrameLookupTable,getEncoderPreset,initializeSession,@@ -409,6 +411,92 @@ export function rebuildExtractedFramesFromPlanDir(return result;}+function contiguousRuns(sortedIndexes: readonly number[]): FrameRange[] {+ const runs: FrameRange[] = [];+ for (const index of sortedIndexes) {+ const last = runs[runs.length - 1];+ if (last && last.firstFrame + last.frames === index) last.frames++;+ else runs.push({ firstFrame: index, frames: 1 });+ }+ return runs;+}++/**+ * Extract the frames this chunk shows from videos the planner left+ * unextracted. Frames land in `outputRoot/<videoId>/` under the names a full+ * extraction gives them, so the lookup indexes them exactly as it would+ * frames the planner wrote.+ */+export async function extractDeferredVideoFramesForChunk(input: {+ planDir: string;+ planVideos: PlanVideosJson;+ slice: ChunkSliceJson;+ fps: { num: number; den: number };+ outputRoot: string;+ cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;+}): Promise<ExtractedFrames[]> {+ const deferred = input.planVideos.extracted.filter((video) => video.deferredRange);+ if (deferred.length === 0) return [];+ const table = createFrameLookupTable(+ input.planVideos.videos,+ deferred.map((video) => ({+ ...video,+ outputDir: input.outputRoot,+ framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),+ })),+ resolveRenderFpsConfig(input.fps).value,+ );+ const shown = new Map<string, Set<number>>();+ for (let frame = input.slice.startFrame; frame < input.slice.endFrame; frame++) {+ const globalTime = (frame * input.fps.den) / input.fps.num;+ for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {+ const indexes = shown.get(videoId) ?? new Set<number>();+ indexes.add(payload.frameIndex);+ shown.set(videoId, indexes);+ }+ }+ return Promise.all(+ deferred.map(async (video) => {+ const outputDir = join(input.outputRoot, video.videoId);+ const indexes = [...(shown.get(video.videoId) ?? [])].sort((a, b) => a - b);+ if (indexes.length === 0) {+ return { ...video, outputDir, framePaths: new Map<number, string>(), ownedByLookup: false };+ }+ const range = video.deferredRange!;+ const sourcePath = join(input.planDir, ...range.sourcePath.split("/"));+ if (!existsSync(sourcePath)) {+ throw new RenderChunkValidationError(+ MISSING_PLAN_ARTIFACT,+ `[renderChunk] planDir is missing the source for video ${JSON.stringify(video.videoId)}: ${range.sourcePath}`,+ );+ }+ const extracted = await extractVideoFramesRange(+ sourcePath,+ video.videoId,+ range.startTime,+ range.durationSeconds,+ {+ fps: input.fps,+ outputDir: input.outputRoot,+ format: range.format,+ frameRanges: contiguousRuns(indexes),+ },+ undefined,+ input.cfg,+ );+ const missing = indexes.find((index) => !extracted.framePaths.has(index));+ if (missing !== undefined) {+ throw new RenderChunkValidationError(+ INVALID_VIDEO_METADATA,+ `[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${missing}; ` ++ `the plan expected ${video.totalFrames} frames from ${range.sourcePath}.`,+ );+ }+ return { ...video, outputDir, framePaths: extracted.framePaths, ownedByLookup: false };+ }),+ );+}+/** Plan-time JSON manifest written by `freezePlan`. */interface PlanJson {protocol?: unknown;@@ -667,6 +755,20 @@ export async function renderChunk(forceScreenshotExplicitlyOptedOut: !encoder.forceScreenshot,};+ // ── Per-chunk work + frames directories ──+ // Suffix workDir with pid + random bytes so concurrent invocations on+ // the SAME `(planDir, chunkIndex)` (e.g. a scheduler that double-fires+ // due to heartbeat skew) don't race on the same tmp tree. The output+ // path itself is still the caller's contract — concurrent writers to+ // `outputChunkPath` produce undefined bytes, but we don't make it worse+ // by also deleting their workDirs out from under them.+ const workDir = `${outputChunkPath}.work.${process.pid}.${randomBytes(4).toString("hex")}`;+ mkdirSync(workDir, { recursive: true });+ const framesDir = join(workDir, "captured-frames");+ mkdirSync(framesDir, { recursive: true });+ // Beside workDir, not in it: a screenshot retry wipes workDir.+ const videoFramesDir = `${workDir}.video-frames`;+// Build the immutable frame lookup once. Each browser session/worker gets// its own injector hook because the hook remembers the last injected frame// per video; sharing that state across a fresh retry page can suppress the@@ -675,11 +777,21 @@ export async function renderChunk(planVideos && planVideos.extracted.length > 0? createFrameLookupTable(planVideos.videos,- rebuildExtractedFramesFromPlanDir(- planDir,- planVideos.extracted,- v2Manifest === null ? "dense-v1" : "sparse-v2",- ),+ [+ ...rebuildExtractedFramesFromPlanDir(+ planDir,+ planVideos.extracted.filter((video) => !video.deferredRange),+ v2Manifest === null ? "dense-v1" : "sparse-v2",+ ),+ ...(await extractDeferredVideoFramesForChunk({+ planDir,+ planVideos,+ slice,+ fps: { num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },+ outputRoot: videoFramesDir,+ cfg,+ })),+ ],resolveRenderFpsConfig(job.config.fps).value,): null;@@ -689,18 +801,6 @@ export async function renderChunk(planVideos?.videos.length ?? 0,);- // ── Per-chunk work + frames directories ──- // Suffix workDir with pid + random bytes so concurrent invocations on- // the SAME `(planDir, chunkIndex)` (e.g. a scheduler that double-fires- // due to heartbeat skew) don't race on the same tmp tree. The output- // path itself is still the caller's contract — concurrent writers to- // `outputChunkPath` produce undefined bytes, but we don't make it worse- // by also deleting their workDirs out from under them.- const workDir = `${outputChunkPath}.work.${process.pid}.${randomBytes(4).toString("hex")}`;- mkdirSync(workDir, { recursive: true });- const framesDir = join(workDir, "captured-frames");- mkdirSync(framesDir, { recursive: true });-// ── File server with the seeded-random shim ──// `Math.random` / `crypto.getRandomValues` are seeded from virtual// time so retries are pixel-identical. Only distributed renders flip this.@@ -1048,6 +1148,7 @@ export async function renderChunk(// leaves the framesDir in place for inspection.try {rmSync(workDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });+ rmSync(videoFramesDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });} catch (err) {log.warn("[renderChunk] failed to remove work dir", {workDir,diff --git a/packages/producer/src/services/distributed/shared.ts b/packages/producer/src/services/distributed/shared.tsindex e14c013b88..ef574f8d8f 100644--- a/packages/producer/src/services/distributed/shared.ts+++ b/packages/producer/src/services/distributed/shared.ts@@ -97,9 +97,18 @@ export interface PlanVideosJson {fps: number;totalFrames: number;metadata: VideoMetadata;+ /** Present when chunks extract their own frames from `sourcePath` (relative to the planDir). */+ deferredRange?: PlanDeferredFrameRange;}>;}+export interface PlanDeferredFrameRange {+ sourcePath: string;+ startTime: number;+ durationSeconds: number;+ format: "jpg" | "png";+}+export const INVALID_VIDEO_METADATA = "INVALID_VIDEO_METADATA" as const;/**@@ -173,6 +182,25 @@ function readNonNegativeInteger(value: unknown, field: string): number {return value;}+function readDeferredFrameRange(value: unknown, field: string): PlanDeferredFrameRange {+ const range = readRecord(value, field);+ const sourcePath = readNonEmptyString(range.sourcePath, `${field}.sourcePath`);+ if (sourcePath.startsWith("/") || sourcePath.split(/[\\/]/).some((part) => part === "..")) {+ metadataError(`${field}.sourcePath`, "must stay inside the plan directory");+ }+ const durationSeconds = readFiniteNumber(range.durationSeconds, `${field}.durationSeconds`);+ if (durationSeconds <= 0) metadataError(`${field}.durationSeconds`, "must be positive");+ if (range.format !== "jpg" && range.format !== "png") {+ metadataError(`${field}.format`, 'must be "jpg" or "png"');+ }+ return {+ sourcePath,+ startTime: readFiniteNumber(range.startTime, `${field}.startTime`),+ durationSeconds,+ format: range.format,+ };+}+function readRateSpec(value: unknown, field: string): RateSpec | undefined {if (value === undefined) return undefined;if (typeof value === "number") return readFiniteNumber(value, field);@@ -260,6 +288,9 @@ export function parsePlanVideosJson(value: unknown): PlanVideosJson {fps: readFiniteNumber(entry.fps, `${field}.fps`),totalFrames: readNonNegativeInteger(entry.totalFrames, `${field}.totalFrames`),metadata: readVideoMetadata(entry.metadata, `${field}.metadata`),+ ...(entry.deferredRange === undefined+ ? {}+ : { deferredRange: readDeferredFrameRange(entry.deferredRange, `${field}.deferredRange`) }),};});diff --git a/packages/producer/src/services/render/stages/extractVideosStage.ts b/packages/producer/src/services/render/stages/extractVideosStage.tsindex adb435b972..994249f572 100644--- a/packages/producer/src/services/render/stages/extractVideosStage.ts+++ b/packages/producer/src/services/render/stages/extractVideosStage.ts@@ -83,6 +83,7 @@ export interface ExtractVideosStageInput {* passes `true` so the planDir is self-contained.*/materializeSymlinks?: boolean;+ deferRangeExtraction?: boolean;}export interface ExtractVideosStageResult {@@ -382,6 +383,7 @@ export async function runExtractVideosStage(abortSignal,assertNotAborted,materializeSymlinks,+ deferRangeExtraction,} = input;const stage2Start = Date.now();@@ -501,6 +503,7 @@ export async function runExtractVideosStage(timelineEnd: composition.duration,maxTransientRetries: extractionPolicy.maxTransientRetries,collectProbeFailures: extractionPolicy.failureMode === "enforce",+ deferRangeExtraction,},abortSignal,cfg,