Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): long exports with many clips no longer stall at 90% or hide a timeout (#5375)
Miguel Ángel4b68f73fe76b
diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.tsindex 7215f15732..674f23c9c1 100644--- a/packages/engine/src/index.ts+++ b/packages/engine/src/index.ts@@ -344,6 +344,7 @@ export {export {runFfmpeg,formatFfmpegError,+ describeFfmpegFailure,isExternalFfmpegInterruption,type RunFfmpegOptions,type RunFfmpegResult,diff --git a/packages/engine/src/services/chunkEncoder.test.ts b/packages/engine/src/services/chunkEncoder.test.tsindex d664ff2758..3ad3dbb566 100644--- a/packages/engine/src/services/chunkEncoder.test.ts+++ b/packages/engine/src/services/chunkEncoder.test.ts@@ -430,6 +430,80 @@ describe("encodeFramesChunkedConcat ffmpegEncodeTimeout", () => {});});+describe("final mux and faststart ffmpegProcessTimeout", () => {+ afterEach(() => vi.useRealTimers());++ it("reports a mux killed at the configured timeout as a timeout", async () => {+ vi.useFakeTimers();+ const { spawn, calls } = createSpawnSpy();+ vi.resetModules();+ vi.doMock("child_process", () => ({ spawn }));++ const { muxVideoWithAudio } = await import("./chunkEncoder.js");+ const muxPromise = muxVideoWithAudio(+ "/tmp/video-only.mp4",+ "/tmp/audio.aac",+ "/tmp/output.mp4",+ undefined,+ { ffmpegProcessTimeout: 1000 },+ );++ await flushMuxCodecResolution();+ const proc = calls[0]!.proc;+ vi.advanceTimersByTime(999);+ expect(proc.kill).not.toHaveBeenCalled();+ vi.advanceTimersByTime(1);+ expect(proc.kill).toHaveBeenCalledWith("SIGTERM");+ proc.stderr.emit("data", Buffer.from("frame= 9000 time=00:05:00.00\n"));+ emitClose(proc, 255);++ const result = await muxPromise;+ expect(result.success).toBe(false);+ expect(result.error).toMatch(/^FFmpeg timed out after 1000 ms/);+ expect(result.error).toContain("FFMPEG_PROCESS_TIMEOUT_MS");+ expect(result.error).toContain("frame= 9000");+ });++ it("reports a faststart killed at the configured timeout as a timeout", async () => {+ vi.useFakeTimers();+ const { spawn, calls } = createSpawnSpy();+ vi.resetModules();+ vi.doMock("child_process", () => ({ spawn }));++ const { applyFaststart } = await import("./chunkEncoder.js");+ const faststartPromise = applyFaststart("/tmp/video-only.mp4", "/tmp/output.mp4", undefined, {+ ffmpegProcessTimeout: 2000,+ });++ const proc = calls[0]!.proc;+ vi.advanceTimersByTime(1999);+ expect(proc.kill).not.toHaveBeenCalled();+ vi.advanceTimersByTime(1);+ expect(proc.kill).toHaveBeenCalledWith("SIGTERM");+ emitClose(proc, 255);++ const result = await faststartPromise;+ expect(result.success).toBe(false);+ expect(result.error).toMatch(/^FFmpeg timed out after 2000 ms/);+ });++ it("leaves an ordinary mux failure's message as it was", async () => {+ const { spawn, calls } = createSpawnSpy();+ vi.resetModules();+ vi.doMock("child_process", () => ({ spawn }));++ const { muxVideoWithAudio } = await import("./chunkEncoder.js");+ const muxPromise = muxVideoWithAudio("/tmp/video-only.mp4", "/tmp/audio.aac", "/tmp/out.mp4");+ await flushMuxCodecResolution();+ calls[0]!.proc.stderr.emit("data", Buffer.from("Invalid data found when processing input\n"));+ emitClose(calls[0]!.proc, 1);++ const result = await muxPromise;+ expect(result.error).toMatch(/^FFmpeg exited with code 1/);+ expect(result.error).not.toContain("timed out");+ });+});+describe("muxVideoWithAudio audio codec handling", () => {it("preserves an external interruption from mux", async () => {const { spawn, calls } = createSpawnSpy();diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.tsindex 28243068a3..777f3b2dbc 100644--- a/packages/engine/src/services/chunkEncoder.ts+++ b/packages/engine/src/services/chunkEncoder.ts@@ -33,6 +33,7 @@ import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";import { withEvenDimensionPad } from "../utils/evenDimensions.js";import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";import {+ describeFfmpegFailure,ffmpegStatsReader,formatFfmpegError,isExternalFfmpegInterruption,@@ -845,7 +846,7 @@ export async function muxVideoWithAudio(success: result.success,outputPath,durationMs: result.durationMs,- error: !result.success ? formatFfmpegError(result.exitCode, result.stderr) : undefined,+ error: !result.success ? describeFfmpegFailure(result, processTimeout) : undefined,failureReason: result.failureReason,};}@@ -993,7 +994,7 @@ export async function packageHls(success: result.success,outputPath: outputDir,durationMs: result.durationMs,- error: !result.success ? formatFfmpegError(result.exitCode, result.stderr) : undefined,+ error: !result.success ? describeFfmpegFailure(result, processTimeout) : undefined,failureReason: result.failureReason,};}@@ -1041,7 +1042,7 @@ export async function applyFaststart(success: result.success,outputPath,durationMs: result.durationMs,- error: !result.success ? formatFfmpegError(result.exitCode, result.stderr) : undefined,+ error: !result.success ? describeFfmpegFailure(result, processTimeout) : undefined,failureReason: result.failureReason,};}diff --git a/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts b/packages/engine/src/services/videoFrameExtractor.concurrency.test.tsnew file mode 100644index 0000000000..502aa81645--- /dev/null+++ b/packages/engine/src/services/videoFrameExtractor.concurrency.test.ts@@ -0,0 +1,140 @@+import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";+import type { VideoElement } from "../types.js";+import { FFMPEG_PATH_ENV, getFfmpegBinary } from "../utils/ffmpegBinaries.js";+import { runFfmpeg } from "../utils/runFfmpeg.js";++// Two CPUs: the extractor runs at most one ffmpeg at a time.+vi.mock("os", async (importOriginal) => ({+ ...(await importOriginal<typeof import("os")>()),+ cpus: () => [{}, {}],+}));++const { extractAllVideoFrames, extractVideoFramesRange } = await import("./videoFrameExtractor.js");+const { extractMediaMetadata } = await import("../utils/ffprobe.js");++describe.skipIf(process.platform === "win32")("extractAllVideoFrames ffmpeg concurrency", () => {+ const dir = mkdtempSync(join(tmpdir(), "hf-extract-concurrency-"));+ const previousFfmpeg = process.env[FFMPEG_PATH_ENV];+ const shim = join(dir, "ffmpeg-shim.sh");++ async function synth(name: string, lavfi: string, extra: string[] = []): Promise<string> {+ const clip = join(dir, `${name}.mp4`);+ const made = await runFfmpeg([+ ...["-y", "-v", "error", "-f", "lavfi", "-i", lavfi, ...extra],+ ...["-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", clip],+ ]);+ expect(made.success, made.stderr).toBe(true);+ return clip;+ }++ // Extracts `clips` through a shim ffmpeg and returns how many ffmpegs were alive as each started.+ async function aliveCounts(run: string, clips: string[], end: number, fps: number) {+ mkdirSync(join(dir, run, "running"), { recursive: true });+ const videos: VideoElement[] = clips.map((src, i) => ({+ ...{ id: `${run}-${i}`, src, start: 0, end, mediaStart: 0 },+ ...{ loop: false, hasAudio: false },+ }));+ process.env.HF_TEST_SHIM_DIR = join(dir, run);+ process.env[FFMPEG_PATH_ENV] = shim;+ try {+ const result = await extractAllVideoFrames(videos, dir, {+ fps,+ outputDir: join(dir, run, "out"),+ });+ expect(result.errors).toEqual([]);+ } finally {+ if (previousFfmpeg === undefined) delete process.env[FFMPEG_PATH_ENV];+ else process.env[FFMPEG_PATH_ENV] = previousFfmpeg;+ delete process.env.HF_TEST_SHIM_DIR;+ }+ return readFileSync(join(dir, run, "alive"), "utf8")+ .trim()+ .split(/\s+/)+ .map(Number);+ }++ beforeAll(() => {+ writeFileSync(+ shim,+ [+ "#!/bin/sh",+ 'm="$HF_TEST_SHIM_DIR/running/$$"; : > "$m"',+ 'ls "$HF_TEST_SHIM_DIR/running" | wc -l >> "$HF_TEST_SHIM_DIR/alive"',+ 'case "$*" in *held-*) until [ -f "$HF_TEST_SHIM_DIR/release" ] || [ ! -d "$HF_TEST_SHIM_DIR" ]; do sleep 0.05; done ;; esac',+ `"${getFfmpegBinary()}" "$@"; rc=$?`,+ 'rm -f "$m"; exit $rc',+ ].join("\n"),+ );+ chmodSync(shim, 0o755);+ });++ afterAll(() => rmSync(dir, { recursive: true, force: true }));++ it("runs one clip's ffmpeg at a time when only one slot is free", async () => {+ const clips = await Promise.all(+ [0, 1, 2, 3].map((i) => synth(`short-${i}`, `testsrc=s=32x32:d=1:r=10,hue=h=${i * 60}`)),+ );+ const alive = await aliveCounts("short", clips, 1, 10);+ expect(alive).toHaveLength(4);+ expect(Math.max(...alive)).toBe(1);+ }, 60_000);++ it("shares the slot with the segments of long clips", async () => {+ const clips = await Promise.all(+ [0, 1].map((i) => synth(`long-${i}`, `testsrc=s=32x32:d=125:r=1,hue=h=${i * 90}`)),+ );+ const alive = await aliveCounts("long", clips, 125, 1);+ expect(alive.length).toBeGreaterThanOrEqual(4);+ expect(Math.max(...alive)).toBe(1);+ }, 120_000);++ it("gives a variable-frame-rate clip's two-process pipeline one slot", async () => {+ const clips = await Promise.all(+ [0, 1].map((i) =>+ synth(`vfr-${i}`, `color=c=0x${i ? "28C83C" : "C83C28"}:s=32x32:d=2:r=60`, [+ ...["-vf", "select='not(between(n\\,30\\,89))'", "-fps_mode", "vfr"],+ ]),+ ),+ );+ const alive = await aliveCounts("vfr", clips, 1, 30);+ expect(alive).toHaveLength(4);+ expect(Math.max(...alive)).toBeLessThanOrEqual(2);+ }, 60_000);++ it("lets a cancelled extraction leave the queue while another holds the slot", async () => {+ const [held, queued] = await Promise.all([+ synth("held-0", "testsrc=s=32x32:d=1:r=10"),+ synth("queued-0", "testsrc=s=32x32:d=1:r=10,hue=h=90"),+ ]);+ await extractMediaMetadata(queued);+ const run = join(dir, "cancel");+ mkdirSync(join(run, "running"), { recursive: true });+ const extract = (id: string, src: string, signal?: AbortSignal) =>+ extractVideoFramesRange(src, id, 0, 1, { fps: 10, outputDir: run }, signal);+ process.env.HF_TEST_SHIM_DIR = run;+ process.env[FFMPEG_PATH_ENV] = shim;+ try {+ const holder = extract("held", held);+ await vi.waitFor(() => expect(readFileSync(join(run, "alive"), "utf8").trim()).toBe("1"), {+ timeout: 10_000,+ });+ const cancel = new AbortController();+ const cancelled = extract("queued", queued, cancel.signal);+ await new Promise(setImmediate);+ cancel.abort();+ await expect(cancelled).rejects.toThrow(/cancelled/);+ expect(readFileSync(join(run, "alive"), "utf8").trim()).toBe("1");++ writeFileSync(join(run, "release"), "");+ expect((await holder).totalFrames).toBe(10);+ } finally {+ writeFileSync(join(run, "release"), "");+ if (previousFfmpeg === undefined) delete process.env[FFMPEG_PATH_ENV];+ else process.env[FFMPEG_PATH_ENV] = previousFfmpeg;+ delete process.env.HF_TEST_SHIM_DIR;+ }+ }, 30_000);+});diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.tsindex 1dfee36d99..bdf26ac4e0 100644--- a/packages/engine/src/services/videoFrameExtractor.ts+++ b/packages/engine/src/services/videoFrameExtractor.ts@@ -10,6 +10,7 @@ import { audioGroupsById, isMemberGroupHidden, isSelfOrAncestorHidden } from "./import { copyFileSync, existsSync, linkSync, mkdirSync, rmSync } from "fs";import { join } from "path";import { cpus } from "os";+import { createConcurrencyLimit } from "../utils/concurrencyLimit.js";import { parseHTML } from "linkedom";import { resolveProjectRelativeSrc } from "@hyperframes/parsers/asset-resolution";import {@@ -885,6 +886,10 @@ function restoreSourceColourFilter(metadata: VideoMetadata): string[] {*/const EXTRACTION_SEGMENT_SECONDS = 120;+// Caps extraction ffmpeg runs across all clips; uncapped, dozens of clips starved each other into timeouts.+const EXTRACTION_SLOTS = Math.max(1, Math.floor(cpus().length / 2));+const inExtractionSlot = createConcurrencyLimit(EXTRACTION_SLOTS);+interface SegmentedExtraction {decodeArgs: string[];filterAndEncodeArgs: string[];@@ -923,30 +928,34 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmpwhile (next < segmentCount && !signal.aborted) {const index = next++;const { firstFrame, frames } = segments[index]!;- const result = await runFfmpeg(- [- ...decodeArgs,- "-noaccurate_seek",- "-ss",- String(startTime + firstFrame / fps),- "-i",- videoPath,- // Read one frame past the segment and keep exactly its own frames.- "-t",- String((frames + 1) / fps),- "-frames:v",- String(frames),- "-start_number",- String(firstFrame + 1),- ...filterAndEncodeArgs,- ],- { ...job.runOptions, signal },+ const result = await inExtractionSlot(+ () =>+ runFfmpeg(+ [+ ...decodeArgs,+ "-noaccurate_seek",+ "-ss",+ String(startTime + firstFrame / fps),+ "-i",+ videoPath,+ // Read one frame past the segment and keep exactly its own frames.+ "-t",+ String((frames + 1) / fps),+ "-frames:v",+ String(frames),+ "-start_number",+ String(firstFrame + 1),+ ...filterAndEncodeArgs,+ ],+ { ...job.runOptions, signal },+ ),+ signal,);results[index] = result;if (!result.success) failed.abort();}};- const workers = Math.min(segmentCount, Math.max(1, Math.floor(cpus().length / 2)));+ const workers = Math.min(segmentCount, EXTRACTION_SLOTS);await Promise.all(Array.from({ length: workers }, worker));const ran = results.filter(Boolean);// Report the segment that failed, not the ones stopped because of it.@@ -1106,20 +1115,24 @@ export async function extractVideoFramesRange() {// ffmpeg <6.1 ignores frame durations in CFR resampling once any -vf is set,// cutting a trailing still short, so the SDR filters run in a second process.- processResult = await runFfmpegPipeline(- [...args, "-an", "-sn", "-dn", "-c:v", "rawvideo", "-f", "nut", "pipe:1"],- [- "-f",- "nut",- "-i",- "pipe:0",- "-vf",- [...restoreSourceColourFilter(metadata), ...vfFilters].join(","),- "-fps_mode",- "passthrough",- ...encodeArgs,- ],- runOptions,+ processResult = await inExtractionSlot(+ () =>+ runFfmpegPipeline(+ [...args, "-an", "-sn", "-dn", "-c:v", "rawvideo", "-f", "nut", "pipe:1"],+ [+ "-f",+ "nut",+ "-i",+ "pipe:0",+ "-vf",+ [...restoreSourceColourFilter(metadata), ...vfFilters].join(","),+ "-fps_mode",+ "passthrough",+ ...encodeArgs,+ ],+ runOptions,+ ),+ signal,);} else {const filterArgs = vfFilters.length > 0 ? ["-vf", vfFilters.join(",")] : [];@@ -1139,7 +1152,10 @@ export async function extractVideoFramesRange(),runOptions,})- : await runFfmpeg([...args, ...filterArgs, ...encodeArgs], runOptions);+ : await inExtractionSlot(+ () => runFfmpeg([...args, ...filterArgs, ...encodeArgs], runOptions),+ signal,+ );}if (processResult.failureReason === "external_interruption") {throw new VideoSourceExtractionError(diff --git a/packages/engine/src/utils/concurrencyLimit.test.ts b/packages/engine/src/utils/concurrencyLimit.test.tsnew file mode 100644index 0000000000..9efb48301f--- /dev/null+++ b/packages/engine/src/utils/concurrencyLimit.test.ts@@ -0,0 +1,81 @@+import { describe, expect, it } from "vitest";+import { createConcurrencyLimit } from "./concurrencyLimit.js";++function deferred() {+ let resolve!: () => void;+ const promise = new Promise<void>((res) => (resolve = res));+ return { promise, resolve };+}++const flush = () => new Promise<void>((resolve) => setImmediate(resolve));++describe("createConcurrencyLimit", () => {+ it("never runs more than the limit at once, and starts waiters in order", async () => {+ const limit = createConcurrencyLimit(2);+ const gates = Array.from({ length: 5 }, deferred);+ const started: number[] = [];+ let running = 0;+ let peak = 0;+ const all = gates.map((gate, i) =>+ limit(async () => {+ started.push(i);+ peak = Math.max(peak, ++running);+ await gate.promise;+ running--;+ }),+ );+ await flush();+ expect(started).toEqual([0, 1]);+ gates[1]!.resolve();+ await flush();+ expect(started).toEqual([0, 1, 2]);+ gates[0]!.resolve();+ gates[2]!.resolve();+ await flush();+ expect(started).toEqual([0, 1, 2, 3, 4]);+ gates[3]!.resolve();+ gates[4]!.resolve();+ await Promise.all(all);+ expect(peak).toBe(2);+ });++ it("frees the slot when a task fails", async () => {+ const limit = createConcurrencyLimit(1);+ const failing = limit(() => Promise.reject(new Error("ffmpeg failed")));+ const next = limit(async () => "ran");+ await expect(failing).rejects.toThrow("ffmpeg failed");+ await expect(next).resolves.toBe("ran");+ });++ it("lets a cancelled waiter leave the queue at once, without taking a slot", async () => {+ const limit = createConcurrencyLimit(1);+ const first = deferred();+ const started: string[] = [];+ const running = limit(async () => {+ started.push("first");+ await first.promise;+ });+ const cancel = new AbortController();+ const cancelled = limit(async () => void started.push("cancelled"), cancel.signal);+ const later = limit(async () => void started.push("later"));+ await flush();+ expect(started).toEqual(["first"]);++ cancel.abort();+ await cancelled;+ expect(started).toEqual(["first", "cancelled"]);++ first.resolve();+ await Promise.all([running, later]);+ expect(started).toEqual(["first", "cancelled", "later"]);+ });++ it("runs a task whose signal already aborted without waiting", async () => {+ const limit = createConcurrencyLimit(1);+ const first = deferred();+ const running = limit(() => first.promise);+ await expect(limit(async () => "ran", AbortSignal.abort())).resolves.toBe("ran");+ first.resolve();+ await running;+ });+});diff --git a/packages/engine/src/utils/concurrencyLimit.ts b/packages/engine/src/utils/concurrencyLimit.tsnew file mode 100644index 0000000000..3e57e2cff1--- /dev/null+++ b/packages/engine/src/utils/concurrencyLimit.ts@@ -0,0 +1,36 @@+/**+ * Runs at most `limit` tasks at once; later tasks wait, in order, for a free slot. A waiter whose+ * `signal` aborts stops waiting and runs at once without a slot, so a cancel never queues behind others.+ */+export function createConcurrencyLimit(+ limit: number,+): <T>(task: () => Promise<T>, signal?: AbortSignal) => Promise<T> {+ let active = 0;+ const waiting: Array<() => void> = [];+ const waitForSlot = (signal?: AbortSignal) =>+ new Promise<boolean>((resolve) => {+ const granted = () => {+ signal?.removeEventListener("abort", aborted);+ resolve(true);+ };+ const aborted = () => {+ waiting.splice(waiting.indexOf(granted), 1);+ resolve(false);+ };+ waiting.push(granted);+ signal?.addEventListener("abort", aborted, { once: true });+ });+ return async (task, signal) => {+ if (signal?.aborted) return task();+ if (active < limit) active++;+ else if (!(await waitForSlot(signal))) return task();+ try {+ return await task();+ } finally {+ // Hand the slot straight to the next waiter so a newcomer cannot take it first.+ const next = waiting.shift();+ if (next) next();+ else active--;+ }+ };+}diff --git a/packages/engine/src/utils/runFfmpeg.test.ts b/packages/engine/src/utils/runFfmpeg.test.tsindex 2bd153fad8..402685284e 100644--- a/packages/engine/src/utils/runFfmpeg.test.ts+++ b/packages/engine/src/utils/runFfmpeg.test.ts@@ -320,3 +320,26 @@ describe("ffmpegStatsReader", () => {expect(stats).toEqual([{ frames: undefined, seconds: 68.04 }]);});});++describe("a run whose signal already aborted", () => {+ afterEach(() => {+ vi.doUnmock("child_process");+ vi.resetModules();+ });++ it("starts no ffmpeg, alone or as a pipeline", async () => {+ const spawn = vi.fn();+ vi.resetModules();+ vi.doMock("child_process", () => ({ spawn }));+ const { runFfmpeg, runFfmpegPipeline } = await import("./runFfmpeg.js");+ const signal = AbortSignal.abort();++ const single = await runFfmpeg(["-version"], { signal });+ const piped = await runFfmpegPipeline(["-i", "in"], ["-i", "pipe:0"], { signal });++ expect(spawn).not.toHaveBeenCalled();+ for (const result of [single, piped]) {+ expect(result).toMatchObject({ success: false, terminationReason: "abort" });+ }+ });+});diff --git a/packages/engine/src/utils/runFfmpeg.ts b/packages/engine/src/utils/runFfmpeg.tsindex 92cceffefe..4d97042410 100644--- a/packages/engine/src/utils/runFfmpeg.ts+++ b/packages/engine/src/utils/runFfmpeg.ts@@ -138,6 +138,19 @@ export function formatFfmpegError(: `FFmpeg exited with code ${exitCode}`;}+/** formatFfmpegError, led by the timeout when this process was killed at its deadline. */+export function describeFfmpegFailure(+ result: Pick<RunFfmpegResult, "exitCode" | "stderr" | "terminationReason">,+ timeoutMs: number,+): string {+ const error = formatFfmpegError(result.exitCode, result.stderr);+ if (result.terminationReason !== "deadline") return error;+ return (+ `FFmpeg timed out after ${timeoutMs} ms (ffmpegProcessTimeout; long renders can raise ` ++ `FFMPEG_PROCESS_TIMEOUT_MS). ${error}`+ );+}+function spawnFfmpeg(args: string[]): ChildProcess {// windowsHide: ffmpeg/ffprobe are console-subsystem binaries, so without// this Node opens a visible console window per spawn on Windows. A render@@ -168,7 +181,18 @@ function toResult(outcome: ManagedChildProcessOutcome, stderr = outcome.stderr):return result;}+/** A run whose signal had already aborted starts no process. */+const ABORTED_BEFORE_START: RunFfmpegResult = {+ success: false,+ exitCode: null,+ signal: null,+ stderr: "",+ durationMs: 0,+ terminationReason: "abort",+};+export async function runFfmpeg(args: string[], opts?: RunFfmpegOptions): Promise<RunFfmpegResult> {+ if (opts?.signal?.aborted) return { ...ABORTED_BEFORE_START };const timeout = opts?.timeout ?? DEFAULT_TIMEOUT;const managed = new ManagedChildProcess(spawnFfmpeg(args), {signal: opts?.signal,@@ -187,6 +211,7 @@ export async function runFfmpegPipeline(consumerArgs: string[],opts?: RunFfmpegOptions,): Promise<RunFfmpegResult> {+ if (opts?.signal?.aborted) return { ...ABORTED_BEFORE_START };const deadlineAtMs = Date.now() + (opts?.timeout ?? DEFAULT_TIMEOUT);const producer = spawnFfmpeg(producerArgs);const consumer = spawnFfmpeg(consumerArgs);diff --git a/packages/producer/src/services/distributed/assemble.test.ts b/packages/producer/src/services/distributed/assemble.test.tsindex 12eb6e54e8..72af446b8e 100644--- a/packages/producer/src/services/distributed/assemble.test.ts+++ b/packages/producer/src/services/distributed/assemble.test.ts@@ -759,6 +759,32 @@ describe("assemble()", () => {TIMEOUT_MS,);+ it(+ "stops ffmpeg at FFMPEG_PROCESS_TIMEOUT_MS and says it timed out",+ async () => {+ if (!hasFfmpeg) return;+ const chunks: ChunkSliceJson[] = [+ { index: 0, startFrame: 0, endFrame: 5 },+ { index: 1, startFrame: 5, endFrame: 10 },+ ];+ const planDir = buildPlanDir("mp4", chunks, 10, false);+ const chunkPaths = [join(planDir, "chunk-0.mp4"), join(planDir, "chunk-1.mp4")];+ for (const chunkPath of chunkPaths) makeMp4Chunk(chunkPath, 5);++ const previous = process.env.FFMPEG_PROCESS_TIMEOUT_MS;+ process.env.FFMPEG_PROCESS_TIMEOUT_MS = "1";+ try {+ await expect(+ assemble(planDir, chunkPaths, null, join(planDir, "output.mp4")),+ ).rejects.toThrow(/FFmpeg timed out after 1 ms/);+ } finally {+ if (previous === undefined) delete process.env.FFMPEG_PROCESS_TIMEOUT_MS;+ else process.env.FFMPEG_PROCESS_TIMEOUT_MS = previous;+ }+ },+ TIMEOUT_MS,+ );+it("rejects when chunkPaths.length does not match chunks.json length", async () => {const chunks: ChunkSliceJson[] = [{ index: 0, startFrame: 0, endFrame: 5 },diff --git a/packages/producer/src/services/distributed/assemble.ts b/packages/producer/src/services/distributed/assemble.tsindex 2b2f36509e..81f2fb65a9 100644--- a/packages/producer/src/services/distributed/assemble.ts+++ b/packages/producer/src/services/distributed/assemble.ts@@ -38,8 +38,10 @@ import { dirname, join } from "node:path";import {appendRenderProvenanceArgs,applyFaststart,+ describeFfmpegFailure,MIXED_AUDIO_FILENAME,muxVideoWithAudio,+ resolveConfig,runFfmpeg,} from "@hyperframes/engine";import { fpsToFfmpegArg } from "@hyperframes/core";@@ -122,6 +124,7 @@ export async function assemble(const log = options?.logger ?? defaultLogger;const abortSignal = options?.abortSignal;const cfr = options?.cfr === true;+ const { ffmpegProcessTimeout } = resolveConfig();// ── 1. Validate planDir manifest matches chunkPaths shape ──────────────const planJsonPath = join(planDir, "plan.json");@@ -181,10 +184,13 @@ export async function assemble(const remuxArgs = ["-i", chunkPaths[0]!, "-c", "copy", "-r", fpsArg];appendRenderProvenanceArgs(remuxArgs, concatOutputPath);remuxArgs.push("-y", concatOutputPath);- const remuxResult = await runFfmpeg(remuxArgs, { signal: abortSignal });+ const remuxResult = await runFfmpeg(remuxArgs, {+ signal: abortSignal,+ timeout: ffmpegProcessTimeout,+ });if (!remuxResult.success) {throw encoderFailureError("[assemble] ffmpeg single-chunk remux failed", {- error: `exit ${remuxResult.exitCode}: ${remuxResult.stderr.slice(-400)}`,+ error: describeFfmpegFailure(remuxResult, ffmpegProcessTimeout),failureReason: remuxResult.failureReason,});}@@ -216,10 +222,13 @@ export async function assemble(];appendRenderProvenanceArgs(concatArgs, concatOutputPath);concatArgs.push("-y", concatOutputPath);- const concatResult = await runFfmpeg(concatArgs, { signal: abortSignal });+ const concatResult = await runFfmpeg(concatArgs, {+ signal: abortSignal,+ timeout: ffmpegProcessTimeout,+ });if (!concatResult.success) {throw encoderFailureError("[assemble] ffmpeg concat-copy failed", {- error: `exit ${concatResult.exitCode}: ${concatResult.stderr.slice(-400)}`,+ error: describeFfmpegFailure(concatResult, ffmpegProcessTimeout),failureReason: concatResult.failureReason,});}@@ -286,10 +295,13 @@ export async function assemble(];appendRenderProvenanceArgs(cfrArgs, cfrOutputPath);cfrArgs.push("-y", cfrOutputPath);- const cfrResult = await runFfmpeg(cfrArgs, { signal: abortSignal });+ const cfrResult = await runFfmpeg(cfrArgs, {+ signal: abortSignal,+ timeout: ffmpegProcessTimeout,+ });if (!cfrResult.success) {throw encoderFailureError("[assemble] ffmpeg cfr re-encode failed", {- error: `exit ${cfrResult.exitCode}: ${cfrResult.stderr.slice(-400)}`,+ error: describeFfmpegFailure(cfrResult, ffmpegProcessTimeout),failureReason: cfrResult.failureReason,});}@@ -310,6 +322,7 @@ export async function assemble(audioPath,outputPath: paddedAudioPath,signal: abortSignal,+ timeoutMs: ffmpegProcessTimeout,});if (!padTrimResult.success) {throw encoderFailureError("[assemble] audio pad/trim failed", padTrimResult);@@ -336,9 +349,7 @@ export async function assemble(normalizedAudioPath,muxOutputPath,abortSignal,- {- audioCodec: "aac",- },+ { audioCodec: "aac", ffmpegProcessTimeout },{ num: plan.dimensions.fpsNum, den: plan.dimensions.fpsDen },);if (!muxResult.success) {@@ -352,7 +363,7 @@ export async function assemble(muxOutputPath,outputPath,abortSignal,- undefined,+ { ffmpegProcessTimeout },{num: plan.dimensions.fpsNum,den: plan.dimensions.fpsDen,diff --git a/packages/producer/src/services/render/audioPadTrim.ts b/packages/producer/src/services/render/audioPadTrim.tsindex 50ae76f303..a302adfc82 100644--- a/packages/producer/src/services/render/audioPadTrim.ts+++ b/packages/producer/src/services/render/audioPadTrim.ts@@ -23,8 +23,9 @@ import { spawn } from "node:child_process";import { mkdtempSync, renameSync, rmSync } from "node:fs";import { dirname, join } from "node:path";import {+ DEFAULT_CONFIG,+ describeFfmpegFailure,extractAudioMetadata,- formatFfmpegError,getFfprobeBinary,ManagedChildProcess,runFfmpeg,@@ -73,6 +74,8 @@ export interface PadTrimAudioInput {/** Path the helper writes the duration-corrected audio to. */outputPath: string;signal?: AbortSignal;+ /** Deadline for each ffmpeg process; defaults to the engine's `ffmpegProcessTimeout`. */+ timeoutMs?: number;/*** Optional injectables for unit tests. Production callers omit them and* get the real `ffprobe`/`ffmpeg`-backed implementations.@@ -291,12 +294,18 @@ export async function padOrTrimAudioToVideoFrameCount(input.probeVideoFrameInfo ??((videoPath: string) => defaultProbeVideoFrameInfo(videoPath, input.signal));const probeAudio = input.probeAudioInfo ?? defaultProbeAudioInfo;- const runner = input.runFfmpeg ?? ((args: string[]) => defaultRunFfmpeg(args, input.signal));+ const timeoutMs = input.timeoutMs ?? DEFAULT_CONFIG.ffmpegProcessTimeout;+ const runner =+ input.runFfmpeg ?? ((args: string[]) => defaultRunFfmpeg(args, timeoutMs, input.signal));// Injected runners usually do not materialize media. Tests that exercise// peak correction opt in with a matching probe; production always uses the// real post-codec measurement.const probeTruePeak =- input.probeAudioTruePeakDbfs ?? (input.runFfmpeg ? undefined : defaultProbeAudioTruePeakDbfs);+ input.probeAudioTruePeakDbfs ??+ (input.runFfmpeg+ ? undefined+ : (audioPath: string, signal?: AbortSignal) =>+ defaultProbeAudioTruePeakDbfs(audioPath, timeoutMs, signal));// Probe video and audio in parallel — the two ffprobe invocations are// independent and account for most of this function's wall-clock time.@@ -607,14 +616,15 @@ async function defaultProbeAudioInfo(async function defaultProbeAudioTruePeakDbfs(audioPath: string,+ timeoutMs: number,signal?: AbortSignal,): Promise<number> {const result = await runFfmpeg(["-i", audioPath, "-map", "0:a:0", "-vn", "-af", "ebur128=peak=true", "-f", "null", "-"],- { signal },+ { signal, timeout: timeoutMs },);if (!result.success) {- throw new Error(`[audioPadTrim] ${formatFfmpegError(result.exitCode, result.stderr)}`);+ throw new Error(`[audioPadTrim] ${describeFfmpegFailure(result, timeoutMs)}`);}const matches = [...result.stderr.matchAll(/^\s*Peak:\s*([+-]?(?:[\d.]+|inf)) dBFS$/gim)];const token = matches.at(-1)?.[1]?.toLowerCase();@@ -627,17 +637,18 @@ async function defaultProbeAudioTruePeakDbfs(async function defaultRunFfmpeg(args: string[],+ timeoutMs: number,signal?: AbortSignal,): Promise<{success: boolean;error?: string;failureReason?: "external_interruption";}> {- const result = await runFfmpeg(args, { signal });+ const result = await runFfmpeg(args, { signal, timeout: timeoutMs });if (result.success) return { success: true };return {success: false,- error: `[audioPadTrim] ${formatFfmpegError(result.exitCode, result.stderr)}`,+ error: `[audioPadTrim] ${describeFfmpegFailure(result, timeoutMs)}`,failureReason: result.failureReason,};}diff --git a/packages/producer/src/services/render/stages/assembleStage.test.ts b/packages/producer/src/services/render/stages/assembleStage.test.tsindex da1ef78c77..faa18b14f7 100644--- a/packages/producer/src/services/render/stages/assembleStage.test.ts+++ b/packages/producer/src/services/render/stages/assembleStage.test.ts@@ -44,6 +44,7 @@ function makeInput(overrides: Partial<AssembleStageInput> = {}): AssembleStageInaudioOutputPath: "/tmp/audio.m4a",outputPath: "/tmp/output.mp4",hasAudio: true,+ ffmpegProcessTimeout: 1_234_000,abortSignal: undefined,assertNotAborted: () => {},...overrides,@@ -78,13 +79,27 @@ describe("runAssembleStage audio duration parity", () => {audioPath: "/tmp/audio.m4a",outputPath: "/tmp/audio.duration-normalized.m4a",signal: undefined,+ timeoutMs: 1_234_000,});expect(muxVideoWithAudioMock).toHaveBeenCalledWith("/tmp/video-only.mp4","/tmp/audio.duration-normalized.m4a","/tmp/output.mp4",undefined,- { audioCodec: "aac" },+ { audioCodec: "aac", ffmpegProcessTimeout: 1_234_000 },+ { num: 30, den: 1 },+ expect.any(Function),+ );+ });++ it("gives a silent mp4's faststart the render's configured ffmpeg timeout", async () => {+ await runAssembleStage(makeInput({ hasAudio: false }));++ expect(applyFaststartMock).toHaveBeenCalledWith(+ "/tmp/video-only.mp4",+ "/tmp/output.mp4",+ undefined,+ { ffmpegProcessTimeout: 1_234_000 },{ num: 30, den: 1 },expect.any(Function),);@@ -98,6 +113,7 @@ describe("runAssembleStage audio duration parity", () => {audioPath: "/tmp/audio.m4a",outputPath: "/tmp/audio.duration-normalized.m4a",signal: undefined,+ timeoutMs: 1_234_000,});});@@ -204,12 +220,13 @@ describe("runAssembleStage HLS packaging", () => {audioPath: "/tmp/audio.m4a",outputPath: "/tmp/audio.duration-normalized.m4a",signal: undefined,+ timeoutMs: 1_234_000,});expect(packageHlsMock).toHaveBeenCalledWith("/tmp/video-only.mp4","/tmp/audio.duration-normalized.m4a","/tmp/output-hls",- { segmentSeconds: 6, signal: undefined },+ { segmentSeconds: 6, signal: undefined, ffmpegProcessTimeout: 1_234_000 },);expect(muxVideoWithAudioMock).not.toHaveBeenCalled();expect(applyFaststartMock).not.toHaveBeenCalled();@@ -222,6 +239,7 @@ describe("runAssembleStage HLS packaging", () => {expect(packageHlsMock).toHaveBeenCalledWith("/tmp/video-only.mp4", null, "/tmp/output-hls", {segmentSeconds: 4,signal: undefined,+ ffmpegProcessTimeout: 1_234_000,});expect(applyFaststartMock).not.toHaveBeenCalled();});@@ -229,14 +247,22 @@ describe("runAssembleStage HLS packaging", () => {it("defaults to 4 s segments", async () => {await runAssembleStage(hlsInput());- expect(packageHlsMock.mock.calls[0]?.[3]).toEqual({ segmentSeconds: 4, signal: undefined });+ expect(packageHlsMock.mock.calls[0]?.[3]).toEqual({+ segmentSeconds: 4,+ signal: undefined,+ ffmpegProcessTimeout: 1_234_000,+ });});it("forwards the abort signal to the packager", async () => {const abortSignal = new AbortController().signal;await runAssembleStage(hlsInput({ abortSignal }));- expect(packageHlsMock.mock.calls[0]?.[3]).toEqual({ segmentSeconds: 4, signal: abortSignal });+ expect(packageHlsMock.mock.calls[0]?.[3]).toEqual({+ segmentSeconds: 4,+ signal: abortSignal,+ ffmpegProcessTimeout: 1_234_000,+ });});it("throws 'HLS packaging failed' with the ffmpeg error appended", async () => {diff --git a/packages/producer/src/services/render/stages/assembleStage.ts b/packages/producer/src/services/render/stages/assembleStage.tsindex 46b560f677..2e9ed28cd1 100644--- a/packages/producer/src/services/render/stages/assembleStage.ts+++ b/packages/producer/src/services/render/stages/assembleStage.ts@@ -46,6 +46,7 @@ export interface AssembleStageInput {format?: RenderOutputFormat;/** Segment length for `format: "hls"`. Defaults to {@link DEFAULT_HLS_SEGMENT_SECONDS}. */hlsSegmentSeconds?: number;+ ffmpegProcessTimeout: number;abortSignal: AbortSignal | undefined;assertNotAborted: () => void;onProgress?: ProgressCallback;@@ -98,6 +99,7 @@ async function assembleOutput(outputPath,hasAudio,format,+ ffmpegProcessTimeout,abortSignal,assertNotAborted,} = input;@@ -116,6 +118,7 @@ async function assembleOutput(audioPath: audioOutputPath,outputPath: normalizedAudioPath,signal: abortSignal,+ timeoutMs: ffmpegProcessTimeout,});assertNotAborted();if (!normalizeResult.success) {@@ -132,6 +135,7 @@ async function assembleOutput(abortSignal,{audioCodec: "aac",+ ffmpegProcessTimeout,},job.config.fps,onSecondsWritten,@@ -148,7 +152,7 @@ async function assembleOutput(videoOnlyPath,outputPath,abortSignal,- undefined,+ { ffmpegProcessTimeout },job.config.fps,onSecondsWritten,);@@ -177,6 +181,7 @@ async function runHlsPackaging({segmentSeconds: input.hlsSegmentSeconds ?? DEFAULT_HLS_SEGMENT_SECONDS,signal: input.abortSignal,+ ffmpegProcessTimeout: input.ffmpegProcessTimeout,},);input.assertNotAborted();diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.tsindex ac6a2cfb67..0e73cc855f 100644--- a/packages/producer/src/services/renderOrchestrator.ts+++ b/packages/producer/src/services/renderOrchestrator.ts@@ -5146,6 +5146,7 @@ async function executeRenderPipeline(input: {hasAudio,format: outputFormat,hlsSegmentSeconds,+ ffmpegProcessTimeout: cfg.ffmpegProcessTimeout,abortSignal: executionSignal,assertNotAborted,onProgress,