Skip to content

Write HTML. Render video. Built for agents.

fix(engine): a 29.97 fps camera clip exports smoothly at 30 fps (#5849)

Miguel Ángelb46ebedeae95
diff --git a/packages/engine/src/services/extractionCache.test.ts b/packages/engine/src/services/extractionCache.test.ts
index 81b5ca54d3..baecc882d7 100644
--- a/packages/engine/src/services/extractionCache.test.ts
+++ b/packages/engine/src/services/extractionCache.test.ts
@@ -68,7 +68,7 @@ function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: s
describe("extractionCache constants", () => {
it("exposes the v7 schema prefix", () => {
- expect(SCHEMA_PREFIX).toBe("hfcache-v7-");
+ expect(SCHEMA_PREFIX).toBe("hfcache-v8-");
});
it("exposes the frame filename prefix shared with the extractor", () => {
diff --git a/packages/engine/src/services/extractionCache.ts b/packages/engine/src/services/extractionCache.ts
index 5990992af6..a5d05c7876 100644
--- a/packages/engine/src/services/extractionCache.ts
+++ b/packages/engine/src/services/extractionCache.ts
@@ -62,7 +62,7 @@ export const GC_MARKER = ".hf-last-gc";
* Current schema version. Bump it whenever extraction writes different frames for the same key,
* or warm entries keep serving the old frames across a deploy. Each bump's commit says why.
*/
-export const SCHEMA_PREFIX = "hfcache-v7-";
+export const SCHEMA_PREFIX = "hfcache-v8-";
/** Truncated hex chars of SHA-256 used for the entry directory name. */
const KEY_HEX_CHARS = 16;
diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts
index 3f85c31cdc..b670a14c53 100644
--- a/packages/engine/src/services/videoFrameExtractor.test.ts
+++ b/packages/engine/src/services/videoFrameExtractor.test.ts
@@ -1653,6 +1653,101 @@ describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
60_000,
);
+ it("samples a 29.97 fps source with millisecond timestamps at 30 fps without skipping a frame", async () => {
+ // Each frame writes its index as eight black or white blocks; the mp4 keeps 1/1000 s ticks.
+ const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks.mp4");
+ const synth = await runFfmpeg([
+ ...["-y", "-v", "error", "-f", "lavfi", "-i"],
+ "color=c=black:s=64x8:r=30000/1001:d=2,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
+ ...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
+ fixture,
+ ]);
+ if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
+
+ const result = await extractVideoFramesRange(fixture, "ntsc-ms-ticks", 0, 1.5, {
+ fps: 30,
+ outputDir: join(FIXTURE_DIR, "out-ntsc-ms"),
+ format: "png",
+ });
+ const indexOf = (file: string): number => {
+ const row = spawnSync("ffmpeg", [
+ "-v",
+ "error",
+ "-i",
+ file,
+ "-vf",
+ "format=gray,crop=64:1:0:4",
+ "-f",
+ "rawvideo",
+ "-",
+ ]).stdout;
+ return [...Array(8).keys()].reduce(
+ (n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
+ 0,
+ );
+ };
+ const shown = [...result.framePaths.keys()]
+ .sort((a, b) => a - b)
+ .map((i) => indexOf(result.framePaths.get(i)!));
+ const steps = shown.slice(1).map((n, i) => n - shown[i]!);
+
+ expect(shown.length).toBeGreaterThanOrEqual(44);
+ expect(
+ steps.filter((step) => step > 1),
+ `source frames ${shown.join(",")}`,
+ ).toEqual([]);
+ expect(steps.filter((step) => step === 0).length).toBeLessThanOrEqual(1);
+ }, 60_000);
+
+ it("samples a millisecond-tick 29.97 fps source the same in segments as in one pass", async () => {
+ const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks-8s.mp4");
+ const synth = await runFfmpeg([
+ ...["-y", "-v", "error", "-f", "lavfi", "-i"],
+ "color=c=black:s=64x8:r=30000/1001:d=8,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
+ ...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
+ fixture,
+ ]);
+ if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
+ const indexOf = (file: string): number => {
+ const row = spawnSync("ffmpeg", [
+ "-v",
+ "error",
+ "-i",
+ file,
+ "-vf",
+ "format=gray,crop=64:1:0:4",
+ "-f",
+ "rawvideo",
+ "-",
+ ]).stdout;
+ return [...Array(8).keys()].reduce(
+ (n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
+ 0,
+ );
+ };
+ const options = {
+ fps: 30,
+ outputDir: join(FIXTURE_DIR, "out-ntsc-seams"),
+ format: "png" as const,
+ };
+ const single = await extractVideoFramesRange(fixture, "ntsc-single", 0, 7.5, options);
+
+ const seams = [4, 17, 29, 41, 58, 73, 90, 107, 121, 139, 156, 170, 188, 201];
+ const differing: string[] = [];
+ for (const firstFrame of seams) {
+ const segment = await extractVideoFramesRange(fixture, `ntsc-seam-${firstFrame}`, 0, 7.5, {
+ ...options,
+ frameRanges: [{ firstFrame, frames: 12 }],
+ });
+ for (const [slot, path] of segment.framePaths) {
+ const expected = indexOf(single.framePaths.get(slot)!);
+ const shown = indexOf(path);
+ if (shown !== expected) differing.push(`slot ${slot}: ${shown} vs ${expected}`);
+ }
+ }
+ expect(differing).toEqual([]);
+ }, 120_000);
+
// ffmpeg < 6.1 drops frame durations from CFR resampling once any -vf is set, so the colour
// filters would cut the still that ends this window short.
it.each([
diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts
index b2b0d71e51..03038a5f5d 100644
--- a/packages/engine/src/services/videoFrameExtractor.ts
+++ b/packages/engine/src/services/videoFrameExtractor.ts
@@ -836,6 +836,14 @@ function chromeGuessForUntaggedMatrix(metadata: VideoMetadata): string[] {
* same absolute times as a single pass, so their frames are byte-identical to it.
*/
const EXTRACTION_SEGMENT_SECONDS = 120;
+/** Frame times since file start are `PTS*TB + seek`; snapping them to `lead + k/FR` gives every process one grid. */
+function undoTickRoundingOnTheSourceFrameGrid(seek: number, lead: number): string[] {
+ const sinceFileStart = `(PTS*TB+${seek}-${lead})`;
+ return [
+ "settb=AVTB",
+ `setpts=if(isnan(FR)\\,PTS\\,(${lead}+round(${sinceFileStart}*FR)/FR-${seek})/TB)`,
+ ];
+}
// Caps extraction ffmpeg runs across all clips; uncapped, dozens of clips starved each other into timeouts.
const EXTRACTION_SLOTS = Math.max(1, Math.floor(cpus().length / 2));
@@ -843,7 +851,7 @@ const inExtractionSlot = createConcurrencyLimit(EXTRACTION_SLOTS);
interface SegmentedExtraction {
decodeArgs: string[];
- filterAndEncodeArgs: string[];
+ filterAndEncodeArgs: (seek: number) => string[];
videoPath: string;
startTime: number;
fps: number;
@@ -879,6 +887,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmp
while (next < segmentCount && !signal.aborted) {
const index = next++;
const { firstFrame, frames } = segments[index]!;
+ const seek = startTime + firstFrame / fps;
const result = await inExtractionSlot(
() =>
runFfmpeg(
@@ -886,7 +895,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmp
...decodeArgs,
"-noaccurate_seek",
"-ss",
- String(startTime + firstFrame / fps),
+ String(seek),
"-i",
videoPath,
// Read one frame past the segment and keep exactly its own frames.
@@ -896,7 +905,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmp
String(frames),
"-start_number",
String(firstFrame + 1),
- ...filterAndEncodeArgs,
+ ...filterAndEncodeArgs(seek),
],
{ ...job.runOptions, signal },
),
@@ -1019,15 +1028,20 @@ export async function extractVideoFramesRange(
if (decodeWithVideoToolbox) {
vfFilters.push("format=nv12");
}
- if (sampleAtOutputFps) {
- // Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer
- // first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).
- vfFilters.push(
- "settb=intb/2",
- "setpts=PTS-lte(TB\\,0.001)",
- `fps=${ffmpegFps}:start_time=0:round=up`,
- );
- }
+ const samplingAt = vfFilters.length;
+ // Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer
+ // first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).
+ const samplingFilters = (seek: number): string[] =>
+ sampleAtOutputFps
+ ? [
+ ...(metadata.isVFR
+ ? []
+ : undoTickRoundingOnTheSourceFrameGrid(seek, metadata.videoStreamLeadSeconds ?? 0)),
+ "settb=intb/2",
+ "setpts=PTS-lte(TB\\,0.001)",
+ `fps=${ffmpegFps}:start_time=0:round=up`,
+ ]
+ : [];
if (options.sdrToHdrTransfer) {
// Ordering intent: fps sampling runs BEFORE the colorspace remap so only
// kept frames are converted. The remap is pointwise per-frame, so the
@@ -1055,7 +1069,11 @@ export async function extractVideoFramesRange(
encodeArgs.push("-y", outputPattern);
const runOptions = { signal, timeout: ffmpegProcessTimeout };
- const filterArgs = vfFilters.length > 0 ? ["-vf", vfFilters.join(",")] : [];
+ const filterArgsFor = (seek: number): string[] => {
+ const filters = [...vfFilters];
+ filters.splice(samplingAt, 0, ...samplingFilters(seek));
+ return filters.length > 0 ? ["-vf", filters.join(",")] : [];
+ };
const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps);
const totalFrames = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
// A VFR window rounds its frame count up, and only the segments' `-frames:v` bound
@@ -1066,7 +1084,7 @@ export async function extractVideoFramesRange(
sampleAtOutputFps && (options.frameRanges || boundVfrFrameCount || totalFrames > segmentFrames)
? await runSegmentedExtraction({
decodeArgs,
- filterAndEncodeArgs: [...filterArgs, ...encodeArgs],
+ filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs],
videoPath,
startTime,
fps,
@@ -1077,7 +1095,7 @@ export async function extractVideoFramesRange(
runOptions,
})
: await inExtractionSlot(
- () => runFfmpeg([...args, ...filterArgs, ...encodeArgs], runOptions),
+ () => runFfmpeg([...args, ...filterArgsFor(startTime), ...encodeArgs], runOptions),
signal,
);
if (processResult.failureReason === "external_interruption") {