Skip to content

Write HTML. Render video. Built for agents.

fix(engine): long phone-video exports neither time out nor stall before the first frame (#5921)

Miguel Ángel472ee43c3a26
diff --git a/packages/engine/src/config.ts b/packages/engine/src/config.ts
index 9829642d83..053889ab67 100644
--- a/packages/engine/src/config.ts
+++ b/packages/engine/src/config.ts
@@ -185,7 +185,7 @@ export interface EngineConfig {
// ── FFmpeg timeouts ──────────────────────────────────────────────────
/** Timeout for FFmpeg frame encoding (ms). Default: 600_000 */
ffmpegEncodeTimeout: number;
- /** Timeout for FFmpeg mux/faststart processes (ms). Default: 300_000 */
+ /** FFmpeg mux/faststart timeout (ms); extraction waits this long for each new frame. Default: 300_000 */
ffmpegProcessTimeout: number;
/**
* Inactivity timeout for FFmpeg streaming encode (ms). The timer resets on
diff --git a/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts b/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts
index a7323debb7..90a78b87a0 100644
--- a/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts
+++ b/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts
@@ -1,3 +1,4 @@
+import { spawnSync } from "node:child_process";
import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -104,6 +105,48 @@ describe.skipIf(process.platform === "win32")("extractAllVideoFrames ffmpeg conc
expect(Math.max(...alive)).toBe(1);
}, 60_000);
+ const HAS_ZSCALE = /\szscale\s/.test(
+ spawnSync(getFfmpegBinary(), ["-hide_banner", "-filters"]).stdout.toString(),
+ );
+
+ it.skipIf(!HAS_ZSCALE)(
+ "finishes tone-mapped cuts of shared sources in the one slot, probing each source once",
+ async () => {
+ const hlg = [
+ ...[
+ "-vf",
+ "zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=arib-std-b67:m=bt2020nc:r=tv,format=yuv420p",
+ ],
+ ...["-color_primaries", "bt2020", "-color_trc", "arib-std-b67", "-colorspace", "bt2020nc"],
+ ];
+ const sources = await Promise.all(
+ [0, 1].map((i) => synth(`hlg-${i}`, `testsrc=s=32x32:d=4:r=10,hue=h=${i * 90}`, hlg)),
+ );
+ const run = join(dir, "tonemap");
+ mkdirSync(join(run, "running"), { recursive: true });
+ process.env.HF_TEST_SHIM_DIR = run;
+ process.env[FFMPEG_PATH_ENV] = shim;
+ try {
+ const cuts = Array.from({ length: 8 }, (_, cut) =>
+ extractVideoFramesRange(sources[cut % 2]!, `tonemap-${cut}`, cut * 0.4, 0.4, {
+ fps: 10,
+ outputDir: run,
+ toneMapHdrToSdr: true,
+ }),
+ );
+ expect((await Promise.all(cuts)).map((cut) => cut.totalFrames)).toEqual(Array(8).fill(4));
+ } finally {
+ if (previousFfmpeg === undefined) delete process.env[FFMPEG_PATH_ENV];
+ else process.env[FFMPEG_PATH_ENV] = previousFfmpeg;
+ delete process.env.HF_TEST_SHIM_DIR;
+ }
+ const alive = readFileSync(join(run, "alive"), "utf8").trim().split(/\s+/).map(Number);
+ expect(alive).toHaveLength(8 + 2);
+ expect(Math.max(...alive)).toBe(1);
+ },
+ 60_000,
+ );
+
it("lets a cancelled extraction leave the queue while another holds the slot", async () => {
const [held, queued] = await Promise.all([
synth("held-0", "testsrc=s=32x32:d=1:r=10"),
diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts
index b670a14c53..7e886d8a77 100644
--- a/packages/engine/src/services/videoFrameExtractor.test.ts
+++ b/packages/engine/src/services/videoFrameExtractor.test.ts
@@ -2082,6 +2082,44 @@ describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => {
60_000,
);
+ it("keeps a run that outlasts the timeout while it still writes frames", async () => {
+ const source = join(FIXTURE_DIR, "paced-5s.mp4");
+ const synth = await runFfmpeg([
+ ...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"],
+ ...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=5`, "-pix_fmt", "yuv420p", source],
+ ]);
+ if (!synth.success) throw new Error(`paced fixture synthesis failed: ${synth.stderr}`);
+ const { runFfmpeg: real } =
+ await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
+ // Real-time pacing makes 5 s of source take 5 s against a 2 s timeout.
+ vi.mocked(runFfmpeg).mockImplementation((args, opts) =>
+ real(["-stats_period", "0.1", "-re", ...args], opts),
+ );
+ try {
+ const extracted = await extractVideoFramesRange(
+ source,
+ "paced-past-timeout",
+ 0,
+ 5,
+ { fps: 10, outputDir: FIXTURE_DIR, format: "png" },
+ undefined,
+ { ffmpegProcessTimeout: 2000 },
+ );
+ expect(extracted.totalFrames).toBe(50);
+ } finally {
+ vi.mocked(runFfmpeg).mockImplementation(real);
+ }
+ }, 30_000);
+
+ it("stops a run that writes no new frame for the stall timeout", async () => {
+ // stdin stays open and empty, so ffmpeg waits for its first frame forever.
+ const stalled = await runFfmpeg(
+ ["-f", "rawvideo", "-pix_fmt", "gray", "-s", "2x2", "-i", "pipe:0", "-f", "null", "-"],
+ { stallTimeout: 500 },
+ );
+ expect(stalled.terminationReason).toBe("inactivity");
+ }, 30_000);
+
// A distributed chunk extracts only the frames it renders; they must be the full extraction's.
describe("partial extraction", () => {
const gopSource = join(FIXTURE_DIR, "gop-30fps.mp4");
@@ -3628,6 +3666,60 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {
60_000,
);
+ it("probes each source's first frame once, however many cuts of it extract together", async () => {
+ const sources = ["a", "b"].map((name) => join(fixtureDir, `cuts-${name}.mp4`));
+ for (const source of sources) await synthesizeHlgClip(source);
+ const probes = () =>
+ vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length;
+ const before = probes();
+ await Promise.all(
+ Array.from({ length: 12 }, (_, cut) =>
+ extractVideoFramesRange(sources[cut % 2]!, `cut-${cut}`, 0, 1, {
+ fps: 1,
+ outputDir: join(fixtureDir, "cuts"),
+ format: "png",
+ toneMapHdrToSdr: true,
+ }),
+ ),
+ );
+ expect(probes() - before).toBe(2);
+ }, 60_000);
+
+ it("probes a source again after its first-frame probe failed", async () => {
+ const source = join(fixtureDir, "probe-retry.mp4");
+ await synthesizeHlgClip(source);
+ const probes = () =>
+ vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length;
+ const { runFfmpeg: real } =
+ await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");
+ vi.mocked(runFfmpeg).mockImplementation((args, opts) =>
+ args.includes("showinfo")
+ ? Promise.resolve({
+ success: false,
+ exitCode: 1,
+ stderr: "",
+ durationMs: 0,
+ terminationReason: "exit",
+ })
+ : real(args, opts),
+ );
+ const extract = (id: string) =>
+ extractVideoFramesRange(source, id, 0, 1, {
+ fps: 1,
+ outputDir: join(fixtureDir, "retry"),
+ format: "png",
+ toneMapHdrToSdr: true,
+ });
+ const before = probes();
+ try {
+ await extract("retry-failed");
+ } finally {
+ vi.mocked(runFfmpeg).mockImplementation(real);
+ }
+ await extract("retry-again");
+ expect(probes() - before).toBe(2);
+ }, 60_000);
+
// One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer.
it("tone-maps each frame of a mixed HLG and PQ stream with its own transfer", async () => {
const segment = async (name: string, transfer: number) => {
diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts
index 282a6959d4..2705cbcc26 100644
--- a/packages/engine/src/services/videoFrameExtractor.ts
+++ b/packages/engine/src/services/videoFrameExtractor.ts
@@ -42,6 +42,10 @@ import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";
import {
extractFinalVideoFrameTimestamp,
extractMediaMetadata,
+ mediaFileIdentity,
+ readMetadataCache,
+ writeMetadataCache,
+ type MediaProbeCacheEntry,
type VideoMetadata,
} from "../utils/ffprobe.js";
import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";
@@ -815,12 +819,28 @@ export function parseImageElements(html: string): ImageElement[] {
return images;
}
-/** The first shown frame's colour tags; see {@link firstFrameColourArgs}. */
-async function readFirstFrameColour(videoPath: string, signal?: AbortSignal) {
- const result = await runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)], {
- signal,
+const firstFrameColourCache = new Map<
+ string,
+ MediaProbeCacheEntry<ReturnType<typeof parseFirstFrameColour>>
+>();
+
+/** The first shown frame's colour tags (see {@link firstFrameColourArgs}), probed once per source file. */
+function readFirstFrameColour(videoPath: string) {
+ const identity = mediaFileIdentity(videoPath);
+ const cached = readMetadataCache(firstFrameColourCache, videoPath, identity);
+ if (cached) return cached;
+ // In an extraction slot, so it never adds a decoder; unsignalled, so one cancel cannot cut it short for others.
+ const probe = inExtractionSlot(() =>
+ runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)]),
+ ).then((result) => {
+ // A failed probe answers this cut only; the next one probes again.
+ if (!result.success && firstFrameColourCache.get(videoPath)?.promise === probe) {
+ firstFrameColourCache.delete(videoPath);
+ }
+ return parseFirstFrameColour(result.stderr);
});
- return parseFirstFrameColour(result.stderr);
+ writeMetadataCache(firstFrameColourCache, videoPath, identity, probe);
+ return probe;
}
/** Chrome plays untagged VP9 and AV1 as BT.601, H.264 and VP8 as BT.709 from 720 coded lines (assumed for the rest). */
@@ -834,10 +854,9 @@ function chromeGuessForUntaggedMatrix(metadata: VideoMetadata): string[] {
}
/**
- * Longest stretch of output one ffmpeg process extracts. `ffmpegProcessTimeout` bounds a
- * single process, so one process over a long clip timed out however much time the render
- * had left. Longer ranges split into segments that run side by side. Segments sample the
- * same absolute times as a single pass, so their frames are byte-identical to it.
+ * Longest stretch of output one ffmpeg process extracts; longer ranges split into segments
+ * that run side by side. Segments sample the same absolute times as a single pass, so their
+ * frames are byte-identical to it.
*/
const EXTRACTION_SEGMENT_SECONDS = 120;
/** Frame times since file start are `PTS*TB + seek`; snapping them to `lead + k/FR` gives every process one grid. */
@@ -860,7 +879,7 @@ interface SegmentedExtraction {
startTime: number;
fps: number;
segments: readonly FrameRange[];
- runOptions: { signal?: AbortSignal; timeout: number };
+ runOptions: { signal?: AbortSignal; stallTimeout: number };
}
/** Split runs so no single ffmpeg process extracts more than `segmentFrames` frames. */
@@ -1067,7 +1086,7 @@ async function extractVideoFramesRangeUnredacted(
vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER);
}
if (toneMappedToSdr) {
- const frameColour = await readFirstFrameColour(videoPath, signal);
+ const frameColour = await readFirstFrameColour(videoPath);
vfFilters.push(hdrToSdrToneMapFilter(metadata.colorSpace ?? {}, frameColour));
}
if (toneMappedToSdr || (!isHdr && !options.sdrToHdrTransfer)) {
@@ -1082,7 +1101,8 @@ async function extractVideoFramesRangeUnredacted(
if (format === "png") encodeArgs.push("-compression_level", "1");
encodeArgs.push("-y", outputPattern);
- const runOptions = { signal, timeout: ffmpegProcessTimeout };
+ // A long 4K run can take far longer than the timeout and still be healthy; only a stall is a hang.
+ const runOptions = { signal, stallTimeout: ffmpegProcessTimeout };
const filterArgsFor = (seek: number): string[] => {
const filters = [...vfFilters];
filters.splice(samplingAt, 0, ...samplingFilters(seek));
@@ -1134,8 +1154,8 @@ async function extractVideoFramesRangeUnredacted(
const hdrPrefix = options.sdrToHdrTransfer
? `SDR→HDR conversion failed (colorspace filter in extract pass, target ${options.sdrToHdrTransfer}): `
: "";
- const timedOut = processResult.terminationReason === "deadline";
- const timeoutSuffix = timedOut ? ` (timed out after ${ffmpegProcessTimeout} ms)` : "";
+ const timedOut = processResult.terminationReason === "inactivity";
+ const timeoutSuffix = timedOut ? ` (no new frame for ${ffmpegProcessTimeout} ms)` : "";
const diagnostic =
`${hdrPrefix}FFmpeg exited with code ${processResult.exitCode}${timeoutSuffix}: ` +
processResult.stderr.slice(-500);
@@ -2349,7 +2369,7 @@ export async function extractAllVideoFrames(
try {
rmSync(tempDir, { recursive: true, force: true });
- // A long union can hit the fixed FFmpeg deadline even when each shorter
+ // A long union can fail even when each shorter
// member range succeeds. Do not retry the optimization itself; preserve
// the established grouped→direct fallback and apply bounded retries only
// to the individual source ranges below.
diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts
index 7659580761..d45f64c15a 100644
--- a/packages/engine/src/utils/ffprobe.ts
+++ b/packages/engine/src/utils/ffprobe.ts
@@ -158,7 +158,7 @@ const finalVideoFrameTimestampSignalCaches = new WeakMap<
Map<string, MediaProbeCacheEntry<number>>
>();
const audioMetadataCache = new Map<string, MediaProbeCacheEntry<AudioMetadata>>();
-interface MediaProbeCacheEntry<T = FFProbeOutput> {
+export interface MediaProbeCacheEntry<T = FFProbeOutput> {
identity: string | null;
promise: Promise<T>;
}
@@ -341,7 +341,7 @@ function isPngImageProbe(output: FFProbeOutput): boolean {
);
}
-function mediaFileIdentity(filePath: string): string | null {
+export function mediaFileIdentity(filePath: string): string | null {
try {
const stat = statSync(filePath, { bigint: true });
return [stat.dev, stat.ino, stat.size, stat.mtimeNs, stat.ctimeNs].join(":");
@@ -350,7 +350,7 @@ function mediaFileIdentity(filePath: string): string | null {
}
}
-function readMetadataCache<T>(
+export function readMetadataCache<T>(
cache: Map<string, MediaProbeCacheEntry<T>>,
key: string,
identity: string | null,
@@ -359,7 +359,7 @@ function readMetadataCache<T>(
return cached?.identity === identity ? cached.promise : undefined;
}
-function writeMetadataCache<T>(
+export function writeMetadataCache<T>(
cache: Map<string, MediaProbeCacheEntry<T>>,
key: string,
identity: string | null,
diff --git a/packages/engine/src/utils/runFfmpeg.ts b/packages/engine/src/utils/runFfmpeg.ts
index 75a5836f1f..81ab66bb37 100644
--- a/packages/engine/src/utils/runFfmpeg.ts
+++ b/packages/engine/src/utils/runFfmpeg.ts
@@ -18,6 +18,8 @@ import {
export interface RunFfmpegOptions {
signal?: AbortSignal;
timeout?: number;
+ /** Replaces `timeout`: stop only once this long passes without a new output frame. */
+ stallTimeout?: number;
onStderr?: (line: string) => void;
}
@@ -200,11 +202,25 @@ const ABORTED_BEFORE_START: RunFfmpegResult = {
export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promise<RunFfmpegResult> {
if (opts?.signal?.aborted) return { ...ABORTED_BEFORE_START };
- const timeout = opts?.timeout ?? DEFAULT_TIMEOUT;
+ const stallTimeout = opts?.stallTimeout;
+ let framesWritten = 0;
+ const readStats = ffmpegStatsReader(({ frames }) => {
+ if (frames === undefined || frames <= framesWritten) return;
+ framesWritten = frames;
+ managed.markActivity();
+ });
const managed = new ManagedChildProcess(spawnFfmpeg(args), {
signal: opts?.signal,
- deadlineAtMs: Date.now() + timeout,
- onStderr: opts?.onStderr,
+ ...(stallTimeout === undefined
+ ? { deadlineAtMs: Date.now() + (opts?.timeout ?? DEFAULT_TIMEOUT) }
+ : { inactivityTimeoutMs: stallTimeout }),
+ onStderr:
+ stallTimeout === undefined
+ ? opts?.onStderr
+ : (chunk) => {
+ readStats(chunk);
+ opts?.onStderr?.(chunk);
+ },
});
return toResult(await managed.wait());
}