Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): kill an ffmpeg whose progress freezes [rebuild]

David Lea5cc325b1f7b
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.ts
index 3dd8f4b8..5c05757b 100644
--- a/site/src/cut/server/exportPipeline.test.ts
+++ b/site/src/cut/server/exportPipeline.test.ts
@@ -1,4 +1,6 @@
-import { describe, expect, test } from "bun:test";
+import { describe, expect, jest, test } from "bun:test";
+import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
import path from "node:path";
import { retimeOf } from "@donkeycut/effects-kit";
import {
@@ -7,6 +9,7 @@ import {
narrowSpecToRange,
passWindows,
runExport,
+ runFfmpeg,
videoCodecArgs,
type ExportPipelineIO,
type ExportSpec,
@@ -2038,3 +2041,56 @@ describe("clips shorter than an output frame", () => {
expect(g).toContain(`between(t,${(1.5 / 30).toFixed(6)},`);
});
});
+
+describe("the ffmpeg stall watchdog", () => {
+ // A wedged ffmpeg keeps printing its stats line twice a second with the
+ // cursor frozen ("frame= 4 ... time=00:00:00.16"). The watchdog counts
+ // that as silence. The fake ffmpeg prints two stats lines, the second a
+ // frozen repeat or a step on, and the fake clock runs 130s across them.
+ const LINE = (time: string) => `frame= 4 fps=0.0 q=-0.0 size= 256KiB time=${time} bitrate=12559.5kbits/s speed=0.0007x\r`;
+ const FAKE_FFMPEG = [
+ "#!/bin/bash",
+ `printf '${LINE("00:00:00.16")}' >&2`,
+ "read -r",
+ `[ "$1" = frozen ] && printf '${LINE("00:00:00.16")}' >&2 || printf '${LINE("00:00:00.20")}' >&2`,
+ "read -r",
+ ].join("\n");
+
+ const runAcross = async (mode: "frozen" | "moving") => {
+ const bin = await mkdtemp(path.join(os.tmpdir(), "cut-stall-"));
+ await writeFile(path.join(bin, "ffmpeg"), FAKE_FFMPEG);
+ await chmod(path.join(bin, "ffmpeg"), 0o755);
+ const pathBefore = process.env.PATH;
+ process.env.PATH = `${bin}:${pathBefore}`;
+ jest.useFakeTimers();
+ try {
+ const job: RenderHandle = { tmpDir: bin, outPath: path.join(bin, "out.mp4"), progress: 0, log: [] };
+ const run = runFfmpeg(job, [mode]);
+ const settled = run.then(() => "done", (err: Error) => err.message);
+ const stderr = job.proc!.stderr!;
+ const nextLine = () => new Promise((resolve) => stderr.once("data", resolve));
+
+ // Each step waits for ffmpeg's line, moves the clock, then lets the
+ // fake print its next line.
+ await nextLine();
+ jest.advanceTimersByTime(100_000);
+ job.proc!.stdin!.write("\n");
+ await nextLine();
+ jest.advanceTimersByTime(30_000);
+ job.proc!.stdin!.write("\n");
+ return await settled;
+ } finally {
+ jest.useRealTimers();
+ process.env.PATH = pathBefore;
+ await rm(bin, { recursive: true, force: true });
+ }
+ };
+
+ test("kills a run whose stats line stops moving", async () => {
+ expect(await runAcross("frozen")).toContain("stalled");
+ });
+
+ test("keeps a run whose stats line moves on", async () => {
+ expect(await runAcross("moving")).toBe("done");
+ });
+});
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index ad7765a5..e946762d 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -944,6 +944,9 @@ export function narrowSpecToRange(spec: ExportSpec): ExportSpec {
/** A clip's frame region in even pixels, or null when it fills the whole frame
* (the common case, which keeps the plain full-frame filter path). */
+/** How far an ffmpeg run has got, as its stats line prints it. */
+const STATS_CURSOR = /size=\s*\S+\s+time=\S+/g;
+
/**
* Spawn ffmpeg for one pass, tracking `job.proc` so a cancel kills the live
* process, bounding silence with the stall watchdog, and (when `onProgress` is
@@ -960,21 +963,31 @@ export function runFfmpeg(
const proc = spawn("ffmpeg", args);
job.proc = proc;
// Stall watchdog: legit exports can run long, so bound silence, not total
- // time — kill only if ffmpeg emits nothing for STALL_MS.
+ // time — kill only if ffmpeg makes no progress for STALL_MS. A wedged
+ // ffmpeg keeps printing its stats line with the cursor frozen
+ // ("size= 256KiB time=00:00:00.16"), so a repeated cursor is silence.
const STALL_MS = 120_000;
let watchdog: ReturnType<typeof setTimeout> | undefined;
+ let cursor: string | undefined;
const bump = () => {
clearTimeout(watchdog);
watchdog = setTimeout(() => {
proc.kill("SIGKILL");
- reject(new Error("Export stalled — no ffmpeg output for 120s."));
+ reject(new Error("Export stalled — no ffmpeg progress for 120s."));
}, STALL_MS);
watchdog.unref();
};
bump();
proc.stderr.on("data", (chunk: Buffer) => {
- bump();
const text = chunk.toString();
+
+ // Log lines are progress; a stats line is progress when its cursor moved.
+ const stats = text.match(STATS_CURSOR)?.at(-1);
+ if (stats === undefined || stats !== cursor) {
+ bump();
+ }
+ cursor = stats ?? cursor;
+
job.log.push(text);
if (job.log.length > 200) job.log.shift();
if (onProgress) {
diff --git a/site/src/cut/worker/proxyJob.test.ts b/site/src/cut/worker/proxyJob.test.ts
index 5bb83be7..517c64f2 100644
--- a/site/src/cut/worker/proxyJob.test.ts
+++ b/site/src/cut/worker/proxyJob.test.ts
@@ -119,7 +119,7 @@ describe("proxy worker", () => {
const killed = harness();
killed.io.make.mockImplementation(async (handle: unknown) => {
(handle as RenderHandle).proc = { signalCode: "SIGKILL", exitCode: null } as never;
- throw new Error("Export stalled — no ffmpeg output for 120s.");
+ throw new Error("Export stalled — no ffmpeg progress for 120s.");
});
expect((await killed.run().catch((e: unknown) => e)) instanceof InputFault).toBe(false);
});
diff --git a/site/src/types/bun-test.d.ts b/site/src/types/bun-test.d.ts
index 09152f0a..31ccae92 100644
--- a/site/src/types/bun-test.d.ts
+++ b/site/src/types/bun-test.d.ts
@@ -50,4 +50,10 @@ declare module "bun:test" {
<T extends (...args: never[]) => unknown>(fn: T): T & Spy<T>;
module(specifier: string, factory: () => unknown): void;
};
+ /** Fake timers, for the tests that move the clock themselves. */
+ export const jest: {
+ useFakeTimers(): void;
+ useRealTimers(): void;
+ advanceTimersByTime(ms: number): void;
+ };
}