Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): a 29.97 fps camera clip exports smoothly at 30 fps (#5849)
Miguel Ángelb46ebedeae95
diff --git a/packages/engine/src/services/extractionCache.test.ts b/packages/engine/src/services/extractionCache.test.tsindex 81b5ca54d3..baecc882d7 100644--- a/packages/engine/src/services/extractionCache.test.ts+++ b/packages/engine/src/services/extractionCache.test.ts@@ -68,7 +68,7 @@ function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: sdescribe("extractionCache constants", () => {it("exposes the v7 schema prefix", () => {- expect(SCHEMA_PREFIX).toBe("hfcache-v7-");+ expect(SCHEMA_PREFIX).toBe("hfcache-v8-");});it("exposes the frame filename prefix shared with the extractor", () => {diff --git a/packages/engine/src/services/extractionCache.ts b/packages/engine/src/services/extractionCache.tsindex 5990992af6..a5d05c7876 100644--- a/packages/engine/src/services/extractionCache.ts+++ b/packages/engine/src/services/extractionCache.ts@@ -62,7 +62,7 @@ export const GC_MARKER = ".hf-last-gc";* Current schema version. Bump it whenever extraction writes different frames for the same key,* or warm entries keep serving the old frames across a deploy. Each bump's commit says why.*/-export const SCHEMA_PREFIX = "hfcache-v7-";+export const SCHEMA_PREFIX = "hfcache-v8-";/** Truncated hex chars of SHA-256 used for the entry directory name. */const KEY_HEX_CHARS = 16;diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.tsindex 3f85c31cdc..b670a14c53 100644--- a/packages/engine/src/services/videoFrameExtractor.test.ts+++ b/packages/engine/src/services/videoFrameExtractor.test.ts@@ -1653,6 +1653,101 @@ describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {60_000,);+ it("samples a 29.97 fps source with millisecond timestamps at 30 fps without skipping a frame", async () => {+ // Each frame writes its index as eight black or white blocks; the mp4 keeps 1/1000 s ticks.+ const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks.mp4");+ const synth = await runFfmpeg([+ ...["-y", "-v", "error", "-f", "lavfi", "-i"],+ "color=c=black:s=64x8:r=30000/1001:d=2,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",+ ...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],+ fixture,+ ]);+ if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);++ const result = await extractVideoFramesRange(fixture, "ntsc-ms-ticks", 0, 1.5, {+ fps: 30,+ outputDir: join(FIXTURE_DIR, "out-ntsc-ms"),+ format: "png",+ });+ const indexOf = (file: string): number => {+ const row = spawnSync("ffmpeg", [+ "-v",+ "error",+ "-i",+ file,+ "-vf",+ "format=gray,crop=64:1:0:4",+ "-f",+ "rawvideo",+ "-",+ ]).stdout;+ return [...Array(8).keys()].reduce(+ (n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),+ 0,+ );+ };+ const shown = [...result.framePaths.keys()]+ .sort((a, b) => a - b)+ .map((i) => indexOf(result.framePaths.get(i)!));+ const steps = shown.slice(1).map((n, i) => n - shown[i]!);++ expect(shown.length).toBeGreaterThanOrEqual(44);+ expect(+ steps.filter((step) => step > 1),+ `source frames ${shown.join(",")}`,+ ).toEqual([]);+ expect(steps.filter((step) => step === 0).length).toBeLessThanOrEqual(1);+ }, 60_000);++ it("samples a millisecond-tick 29.97 fps source the same in segments as in one pass", async () => {+ const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks-8s.mp4");+ const synth = await runFfmpeg([+ ...["-y", "-v", "error", "-f", "lavfi", "-i"],+ "color=c=black:s=64x8:r=30000/1001:d=8,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",+ ...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],+ fixture,+ ]);+ if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);+ const indexOf = (file: string): number => {+ const row = spawnSync("ffmpeg", [+ "-v",+ "error",+ "-i",+ file,+ "-vf",+ "format=gray,crop=64:1:0:4",+ "-f",+ "rawvideo",+ "-",+ ]).stdout;+ return [...Array(8).keys()].reduce(+ (n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),+ 0,+ );+ };+ const options = {+ fps: 30,+ outputDir: join(FIXTURE_DIR, "out-ntsc-seams"),+ format: "png" as const,+ };+ const single = await extractVideoFramesRange(fixture, "ntsc-single", 0, 7.5, options);++ const seams = [4, 17, 29, 41, 58, 73, 90, 107, 121, 139, 156, 170, 188, 201];+ const differing: string[] = [];+ for (const firstFrame of seams) {+ const segment = await extractVideoFramesRange(fixture, `ntsc-seam-${firstFrame}`, 0, 7.5, {+ ...options,+ frameRanges: [{ firstFrame, frames: 12 }],+ });+ for (const [slot, path] of segment.framePaths) {+ const expected = indexOf(single.framePaths.get(slot)!);+ const shown = indexOf(path);+ if (shown !== expected) differing.push(`slot ${slot}: ${shown} vs ${expected}`);+ }+ }+ expect(differing).toEqual([]);+ }, 120_000);+// ffmpeg < 6.1 drops frame durations from CFR resampling once any -vf is set, so the colour// filters would cut the still that ends this window short.it.each([diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.tsindex b2b0d71e51..03038a5f5d 100644--- a/packages/engine/src/services/videoFrameExtractor.ts+++ b/packages/engine/src/services/videoFrameExtractor.ts@@ -836,6 +836,14 @@ function chromeGuessForUntaggedMatrix(metadata: VideoMetadata): string[] {* same absolute times as a single pass, so their frames are byte-identical to it.*/const EXTRACTION_SEGMENT_SECONDS = 120;+/** Frame times since file start are `PTS*TB + seek`; snapping them to `lead + k/FR` gives every process one grid. */+function undoTickRoundingOnTheSourceFrameGrid(seek: number, lead: number): string[] {+ const sinceFileStart = `(PTS*TB+${seek}-${lead})`;+ return [+ "settb=AVTB",+ `setpts=if(isnan(FR)\\,PTS\\,(${lead}+round(${sinceFileStart}*FR)/FR-${seek})/TB)`,+ ];+}// Caps extraction ffmpeg runs across all clips; uncapped, dozens of clips starved each other into timeouts.const EXTRACTION_SLOTS = Math.max(1, Math.floor(cpus().length / 2));@@ -843,7 +851,7 @@ const inExtractionSlot = createConcurrencyLimit(EXTRACTION_SLOTS);interface SegmentedExtraction {decodeArgs: string[];- filterAndEncodeArgs: string[];+ filterAndEncodeArgs: (seek: number) => string[];videoPath: string;startTime: number;fps: number;@@ -879,6 +887,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmpwhile (next < segmentCount && !signal.aborted) {const index = next++;const { firstFrame, frames } = segments[index]!;+ const seek = startTime + firstFrame / fps;const result = await inExtractionSlot(() =>runFfmpeg(@@ -886,7 +895,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmp...decodeArgs,"-noaccurate_seek","-ss",- String(startTime + firstFrame / fps),+ String(seek),"-i",videoPath,// Read one frame past the segment and keep exactly its own frames.@@ -896,7 +905,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmpString(frames),"-start_number",String(firstFrame + 1),- ...filterAndEncodeArgs,+ ...filterAndEncodeArgs(seek),],{ ...job.runOptions, signal },),@@ -1019,15 +1028,20 @@ export async function extractVideoFramesRange(if (decodeWithVideoToolbox) {vfFilters.push("format=nv12");}- if (sampleAtOutputFps) {- // Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer- // first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).- vfFilters.push(- "settb=intb/2",- "setpts=PTS-lte(TB\\,0.001)",- `fps=${ffmpegFps}:start_time=0:round=up`,- );- }+ const samplingAt = vfFilters.length;+ // Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer+ // first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).+ const samplingFilters = (seek: number): string[] =>+ sampleAtOutputFps+ ? [+ ...(metadata.isVFR+ ? []+ : undoTickRoundingOnTheSourceFrameGrid(seek, metadata.videoStreamLeadSeconds ?? 0)),+ "settb=intb/2",+ "setpts=PTS-lte(TB\\,0.001)",+ `fps=${ffmpegFps}:start_time=0:round=up`,+ ]+ : [];if (options.sdrToHdrTransfer) {// Ordering intent: fps sampling runs BEFORE the colorspace remap so only// kept frames are converted. The remap is pointwise per-frame, so the@@ -1055,7 +1069,11 @@ export async function extractVideoFramesRange(encodeArgs.push("-y", outputPattern);const runOptions = { signal, timeout: ffmpegProcessTimeout };- const filterArgs = vfFilters.length > 0 ? ["-vf", vfFilters.join(",")] : [];+ const filterArgsFor = (seek: number): string[] => {+ const filters = [...vfFilters];+ filters.splice(samplingAt, 0, ...samplingFilters(seek));+ return filters.length > 0 ? ["-vf", filters.join(",")] : [];+ };const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps);const totalFrames = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);// A VFR window rounds its frame count up, and only the segments' `-frames:v` bound@@ -1066,7 +1084,7 @@ export async function extractVideoFramesRange(sampleAtOutputFps && (options.frameRanges || boundVfrFrameCount || totalFrames > segmentFrames)? await runSegmentedExtraction({decodeArgs,- filterAndEncodeArgs: [...filterArgs, ...encodeArgs],+ filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs],videoPath,startTime,fps,@@ -1077,7 +1095,7 @@ export async function extractVideoFramesRange(runOptions,}): await inExtractionSlot(- () => runFfmpeg([...args, ...filterArgs, ...encodeArgs], runOptions),+ () => runFfmpeg([...args, ...filterArgsFor(startTime), ...encodeArgs], runOptions),signal,);if (processResult.failureReason === "external_interruption") {