Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): long phone-video exports neither time out nor stall before the first frame (#5921)
Miguel Ángel472ee43c3a26
diff --git a/packages/engine/src/config.ts b/packages/engine/src/config.tsindex 9829642d83..053889ab67 100644--- a/packages/engine/src/config.ts+++ b/packages/engine/src/config.ts@@ -185,7 +185,7 @@ export interface EngineConfig {// ── FFmpeg timeouts ──────────────────────────────────────────────────/** Timeout for FFmpeg frame encoding (ms). Default: 600_000 */ffmpegEncodeTimeout: number;- /** Timeout for FFmpeg mux/faststart processes (ms). Default: 300_000 */+ /** FFmpeg mux/faststart timeout (ms); extraction waits this long for each new frame. Default: 300_000 */ffmpegProcessTimeout: number;/*** Inactivity timeout for FFmpeg streaming encode (ms). The timer resets ondiff --git a/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts b/packages/engine/src/services/videoFrameExtractor.concurrency.test.tsindex a7323debb7..90a78b87a0 100644--- a/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts+++ b/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts@@ -1,3 +1,4 @@+import { spawnSync } from "node:child_process";import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";@@ -104,6 +105,48 @@ describe.skipIf(process.platform === "win32")("extractAllVideoFrames ffmpeg concexpect(Math.max(...alive)).toBe(1);}, 60_000);+ const HAS_ZSCALE = /\szscale\s/.test(+ spawnSync(getFfmpegBinary(), ["-hide_banner", "-filters"]).stdout.toString(),+ );++ it.skipIf(!HAS_ZSCALE)(+ "finishes tone-mapped cuts of shared sources in the one slot, probing each source once",+ async () => {+ const hlg = [+ ...[+ "-vf",+ "zscale=pin=bt709:tin=bt709:min=bt709:p=bt2020:t=arib-std-b67:m=bt2020nc:r=tv,format=yuv420p",+ ],+ ...["-color_primaries", "bt2020", "-color_trc", "arib-std-b67", "-colorspace", "bt2020nc"],+ ];+ const sources = await Promise.all(+ [0, 1].map((i) => synth(`hlg-${i}`, `testsrc=s=32x32:d=4:r=10,hue=h=${i * 90}`, hlg)),+ );+ const run = join(dir, "tonemap");+ mkdirSync(join(run, "running"), { recursive: true });+ process.env.HF_TEST_SHIM_DIR = run;+ process.env[FFMPEG_PATH_ENV] = shim;+ try {+ const cuts = Array.from({ length: 8 }, (_, cut) =>+ extractVideoFramesRange(sources[cut % 2]!, `tonemap-${cut}`, cut * 0.4, 0.4, {+ fps: 10,+ outputDir: run,+ toneMapHdrToSdr: true,+ }),+ );+ expect((await Promise.all(cuts)).map((cut) => cut.totalFrames)).toEqual(Array(8).fill(4));+ } finally {+ if (previousFfmpeg === undefined) delete process.env[FFMPEG_PATH_ENV];+ else process.env[FFMPEG_PATH_ENV] = previousFfmpeg;+ delete process.env.HF_TEST_SHIM_DIR;+ }+ const alive = readFileSync(join(run, "alive"), "utf8").trim().split(/\s+/).map(Number);+ expect(alive).toHaveLength(8 + 2);+ expect(Math.max(...alive)).toBe(1);+ },+ 60_000,+ );+it("lets a cancelled extraction leave the queue while another holds the slot", async () => {const [held, queued] = await Promise.all([synth("held-0", "testsrc=s=32x32:d=1:r=10"),diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.tsindex b670a14c53..7e886d8a77 100644--- a/packages/engine/src/services/videoFrameExtractor.test.ts+++ b/packages/engine/src/services/videoFrameExtractor.test.ts@@ -2082,6 +2082,44 @@ describe.skipIf(!HAS_FFMPEG)("frame sampling at the output frame rate", () => {60_000,);+ it("keeps a run that outlasts the timeout while it still writes frames", async () => {+ const source = join(FIXTURE_DIR, "paced-5s.mp4");+ const synth = await runFfmpeg([+ ...["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi"],+ ...["-i", `testsrc=s=${WIDTH}x${HEIGHT}:r=30:d=5`, "-pix_fmt", "yuv420p", source],+ ]);+ if (!synth.success) throw new Error(`paced fixture synthesis failed: ${synth.stderr}`);+ const { runFfmpeg: real } =+ await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");+ // Real-time pacing makes 5 s of source take 5 s against a 2 s timeout.+ vi.mocked(runFfmpeg).mockImplementation((args, opts) =>+ real(["-stats_period", "0.1", "-re", ...args], opts),+ );+ try {+ const extracted = await extractVideoFramesRange(+ source,+ "paced-past-timeout",+ 0,+ 5,+ { fps: 10, outputDir: FIXTURE_DIR, format: "png" },+ undefined,+ { ffmpegProcessTimeout: 2000 },+ );+ expect(extracted.totalFrames).toBe(50);+ } finally {+ vi.mocked(runFfmpeg).mockImplementation(real);+ }+ }, 30_000);++ it("stops a run that writes no new frame for the stall timeout", async () => {+ // stdin stays open and empty, so ffmpeg waits for its first frame forever.+ const stalled = await runFfmpeg(+ ["-f", "rawvideo", "-pix_fmt", "gray", "-s", "2x2", "-i", "pipe:0", "-f", "null", "-"],+ { stallTimeout: 500 },+ );+ expect(stalled.terminationReason).toBe("inactivity");+ }, 30_000);+// A distributed chunk extracts only the frames it renders; they must be the full extraction's.describe("partial extraction", () => {const gopSource = join(FIXTURE_DIR, "gop-30fps.mp4");@@ -3628,6 +3666,60 @@ describe.skipIf(!HAS_ZSCALE)("forced-SDR HDR extraction", () => {60_000,);+ it("probes each source's first frame once, however many cuts of it extract together", async () => {+ const sources = ["a", "b"].map((name) => join(fixtureDir, `cuts-${name}.mp4`));+ for (const source of sources) await synthesizeHlgClip(source);+ const probes = () =>+ vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length;+ const before = probes();+ await Promise.all(+ Array.from({ length: 12 }, (_, cut) =>+ extractVideoFramesRange(sources[cut % 2]!, `cut-${cut}`, 0, 1, {+ fps: 1,+ outputDir: join(fixtureDir, "cuts"),+ format: "png",+ toneMapHdrToSdr: true,+ }),+ ),+ );+ expect(probes() - before).toBe(2);+ }, 60_000);++ it("probes a source again after its first-frame probe failed", async () => {+ const source = join(fixtureDir, "probe-retry.mp4");+ await synthesizeHlgClip(source);+ const probes = () =>+ vi.mocked(runFfmpeg).mock.calls.filter(([args]) => args.includes("showinfo")).length;+ const { runFfmpeg: real } =+ await vi.importActual<typeof import("../utils/runFfmpeg.js")>("../utils/runFfmpeg.js");+ vi.mocked(runFfmpeg).mockImplementation((args, opts) =>+ args.includes("showinfo")+ ? Promise.resolve({+ success: false,+ exitCode: 1,+ stderr: "",+ durationMs: 0,+ terminationReason: "exit",+ })+ : real(args, opts),+ );+ const extract = (id: string) =>+ extractVideoFramesRange(source, id, 0, 1, {+ fps: 1,+ outputDir: join(fixtureDir, "retry"),+ format: "png",+ toneMapHdrToSdr: true,+ });+ const before = probes();+ try {+ await extract("retry-failed");+ } finally {+ vi.mocked(runFfmpeg).mockImplementation(real);+ }+ await extract("retry-again");+ expect(probes() - before).toBe(2);+ }, 60_000);+// One stream with HLG frames, then PQ frames: each is tone-mapped with its own transfer.it("tone-maps each frame of a mixed HLG and PQ stream with its own transfer", async () => {const segment = async (name: string, transfer: number) => {diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.tsindex 282a6959d4..2705cbcc26 100644--- a/packages/engine/src/services/videoFrameExtractor.ts+++ b/packages/engine/src/services/videoFrameExtractor.ts@@ -42,6 +42,10 @@ import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";import {extractFinalVideoFrameTimestamp,extractMediaMetadata,+ mediaFileIdentity,+ readMetadataCache,+ writeMetadataCache,+ type MediaProbeCacheEntry,type VideoMetadata,} from "../utils/ffprobe.js";import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";@@ -815,12 +819,28 @@ export function parseImageElements(html: string): ImageElement[] {return images;}-/** The first shown frame's colour tags; see {@link firstFrameColourArgs}. */-async function readFirstFrameColour(videoPath: string, signal?: AbortSignal) {- const result = await runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)], {- signal,+const firstFrameColourCache = new Map<+ string,+ MediaProbeCacheEntry<ReturnType<typeof parseFirstFrameColour>>+>();++/** The first shown frame's colour tags (see {@link firstFrameColourArgs}), probed once per source file. */+function readFirstFrameColour(videoPath: string) {+ const identity = mediaFileIdentity(videoPath);+ const cached = readMetadataCache(firstFrameColourCache, videoPath, identity);+ if (cached) return cached;+ // In an extraction slot, so it never adds a decoder; unsignalled, so one cancel cannot cut it short for others.+ const probe = inExtractionSlot(() =>+ runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)]),+ ).then((result) => {+ // A failed probe answers this cut only; the next one probes again.+ if (!result.success && firstFrameColourCache.get(videoPath)?.promise === probe) {+ firstFrameColourCache.delete(videoPath);+ }+ return parseFirstFrameColour(result.stderr);});- return parseFirstFrameColour(result.stderr);+ writeMetadataCache(firstFrameColourCache, videoPath, identity, probe);+ return probe;}/** Chrome plays untagged VP9 and AV1 as BT.601, H.264 and VP8 as BT.709 from 720 coded lines (assumed for the rest). */@@ -834,10 +854,9 @@ function chromeGuessForUntaggedMatrix(metadata: VideoMetadata): string[] {}/**- * Longest stretch of output one ffmpeg process extracts. `ffmpegProcessTimeout` bounds a- * single process, so one process over a long clip timed out however much time the render- * had left. Longer ranges split into segments that run side by side. Segments sample the- * same absolute times as a single pass, so their frames are byte-identical to it.+ * Longest stretch of output one ffmpeg process extracts; longer ranges split into segments+ * that run side by side. Segments sample the same absolute times as a single pass, so their+ * frames are byte-identical to it.*/const EXTRACTION_SEGMENT_SECONDS = 120;/** Frame times since file start are `PTS*TB + seek`; snapping them to `lead + k/FR` gives every process one grid. */@@ -860,7 +879,7 @@ interface SegmentedExtraction {startTime: number;fps: number;segments: readonly FrameRange[];- runOptions: { signal?: AbortSignal; timeout: number };+ runOptions: { signal?: AbortSignal; stallTimeout: number };}/** Split runs so no single ffmpeg process extracts more than `segmentFrames` frames. */@@ -1067,7 +1086,7 @@ async function extractVideoFramesRangeUnredacted(vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER);}if (toneMappedToSdr) {- const frameColour = await readFirstFrameColour(videoPath, signal);+ const frameColour = await readFirstFrameColour(videoPath);vfFilters.push(hdrToSdrToneMapFilter(metadata.colorSpace ?? {}, frameColour));}if (toneMappedToSdr || (!isHdr && !options.sdrToHdrTransfer)) {@@ -1082,7 +1101,8 @@ async function extractVideoFramesRangeUnredacted(if (format === "png") encodeArgs.push("-compression_level", "1");encodeArgs.push("-y", outputPattern);- const runOptions = { signal, timeout: ffmpegProcessTimeout };+ // A long 4K run can take far longer than the timeout and still be healthy; only a stall is a hang.+ const runOptions = { signal, stallTimeout: ffmpegProcessTimeout };const filterArgsFor = (seek: number): string[] => {const filters = [...vfFilters];filters.splice(samplingAt, 0, ...samplingFilters(seek));@@ -1134,8 +1154,8 @@ async function extractVideoFramesRangeUnredacted(const hdrPrefix = options.sdrToHdrTransfer? `SDR→HDR conversion failed (colorspace filter in extract pass, target ${options.sdrToHdrTransfer}): `: "";- const timedOut = processResult.terminationReason === "deadline";- const timeoutSuffix = timedOut ? ` (timed out after ${ffmpegProcessTimeout} ms)` : "";+ const timedOut = processResult.terminationReason === "inactivity";+ const timeoutSuffix = timedOut ? ` (no new frame for ${ffmpegProcessTimeout} ms)` : "";const diagnostic =`${hdrPrefix}FFmpeg exited with code ${processResult.exitCode}${timeoutSuffix}: ` +processResult.stderr.slice(-500);@@ -2349,7 +2369,7 @@ export async function extractAllVideoFrames(try {rmSync(tempDir, { recursive: true, force: true });- // A long union can hit the fixed FFmpeg deadline even when each shorter+ // A long union can fail even when each shorter// member range succeeds. Do not retry the optimization itself; preserve// the established grouped→direct fallback and apply bounded retries only// to the individual source ranges below.diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.tsindex 7659580761..d45f64c15a 100644--- a/packages/engine/src/utils/ffprobe.ts+++ b/packages/engine/src/utils/ffprobe.ts@@ -158,7 +158,7 @@ const finalVideoFrameTimestampSignalCaches = new WeakMap<Map<string, MediaProbeCacheEntry<number>>>();const audioMetadataCache = new Map<string, MediaProbeCacheEntry<AudioMetadata>>();-interface MediaProbeCacheEntry<T = FFProbeOutput> {+export interface MediaProbeCacheEntry<T = FFProbeOutput> {identity: string | null;promise: Promise<T>;}@@ -341,7 +341,7 @@ function isPngImageProbe(output: FFProbeOutput): boolean {);}-function mediaFileIdentity(filePath: string): string | null {+export function mediaFileIdentity(filePath: string): string | null {try {const stat = statSync(filePath, { bigint: true });return [stat.dev, stat.ino, stat.size, stat.mtimeNs, stat.ctimeNs].join(":");@@ -350,7 +350,7 @@ function mediaFileIdentity(filePath: string): string | null {}}-function readMetadataCache<T>(+export function readMetadataCache<T>(cache: Map<string, MediaProbeCacheEntry<T>>,key: string,identity: string | null,@@ -359,7 +359,7 @@ function readMetadataCache<T>(return cached?.identity === identity ? cached.promise : undefined;}-function writeMetadataCache<T>(+export function writeMetadataCache<T>(cache: Map<string, MediaProbeCacheEntry<T>>,key: string,identity: string | null,diff --git a/packages/engine/src/utils/runFfmpeg.ts b/packages/engine/src/utils/runFfmpeg.tsindex 75a5836f1f..81ab66bb37 100644--- a/packages/engine/src/utils/runFfmpeg.ts+++ b/packages/engine/src/utils/runFfmpeg.ts@@ -18,6 +18,8 @@ import {export interface RunFfmpegOptions {signal?: AbortSignal;timeout?: number;+ /** Replaces `timeout`: stop only once this long passes without a new output frame. */+ stallTimeout?: number;onStderr?: (line: string) => void;}@@ -200,11 +202,25 @@ const ABORTED_BEFORE_START: RunFfmpegResult = {export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promise<RunFfmpegResult> {if (opts?.signal?.aborted) return { ...ABORTED_BEFORE_START };- const timeout = opts?.timeout ?? DEFAULT_TIMEOUT;+ const stallTimeout = opts?.stallTimeout;+ let framesWritten = 0;+ const readStats = ffmpegStatsReader(({ frames }) => {+ if (frames === undefined || frames <= framesWritten) return;+ framesWritten = frames;+ managed.markActivity();+ });const managed = new ManagedChildProcess(spawnFfmpeg(args), {signal: opts?.signal,- deadlineAtMs: Date.now() + timeout,- onStderr: opts?.onStderr,+ ...(stallTimeout === undefined+ ? { deadlineAtMs: Date.now() + (opts?.timeout ?? DEFAULT_TIMEOUT) }+ : { inactivityTimeoutMs: stallTimeout }),+ onStderr:+ stallTimeout === undefined+ ? opts?.onStderr+ : (chunk) => {+ readStats(chunk);+ opts?.onStderr?.(chunk);+ },});return toResult(await managed.wait());}