Skip to content

Open source CapCut alternative.

TypeScript

fix(site): split each shared ffmpeg input once [rebuild]

David Le166e5eab6cda
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.ts
index 10eb1549..08669289 100644
--- a/site/src/cut/server/exportPipeline.test.ts
+++ b/site/src/cut/server/exportPipeline.test.ts
@@ -389,6 +389,21 @@ describe("export filtergraph timebases", () => {
});
});
+describe("cuts off one file", () => {
+ test("each stream of the file is fanned out by one split, so ffmpeg prepares each frame once", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { in: 1, out: 3 }), clip("a.mp4", { in: 5, out: 8 }), clip("a.mp4", { in: 10, out: 12 })],
+ });
+ expect(g.filter((f) => f.includes("[0:v]"))).toEqual(["[0:v]split=3[fan0_v_0][fan0_v_1][fan0_v_2]"]);
+ expect(g.filter((f) => f.includes("[0:a]"))).toEqual(["[0:a]asplit=3[fan0_a_0][fan0_a_1][fan0_a_2]"]);
+ expect(g.filter((f) => f.startsWith("[fan0_v_")).map((f) => f.slice(0, f.indexOf(",")))).toEqual([
+ "[fan0_v_0]trim=0.000:3.000",
+ "[fan0_v_1]trim=4.000:8.000",
+ "[fan0_v_2]trim=9.000:12.000",
+ ]);
+ });
+});
+
describe("a speed curve in the filtergraph", () => {
test("a curved clip lays its picture through the map and reads baked sound", async () => {
const curved = clip("a.mp4", { speedCurve: [[0, 1], [4, 4]] });
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index 92616e9e..08fbbb0b 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -8,7 +8,7 @@ import { readFile, stat, unlink, writeFile } from "node:fs/promises";
import path from "node:path";
import { atempoChain, audioChannels, hasStream, mediaDuration, num, videoDecodeCost, videoDimensions } from "./util";
import { xfadeTransition } from "./transitionExpr";
-import { assertGraphSafe, fexpr } from "./filterGraph";
+import { assertGraphSafe, fanOutInputs, fexpr } from "./filterGraph";
import { bakeRetimedAudio, setptsExpr, type BakedAudio } from "./retimeAudio";
import { bakeTurnedMedia } from "./turnMedia";
import { withSpecColors } from "./fileColor";
@@ -3253,7 +3253,7 @@ export async function runExport(
[
"-y",
...inputs,
- "-filter_complex", assertGraphSafe(filters.join(";")),
+ "-filter_complex", assertGraphSafe(fanOutInputs(filters).join(";")),
"-map", `[${vLabel}]`,
...(master ? [] : ["-map", `[${aLabel}]`]),
...videoCodecArgs(enc, spec),
diff --git a/site/src/cut/server/filterGraph.test.ts b/site/src/cut/server/filterGraph.test.ts
index f9b5ae8e..7aee9ca9 100644
--- a/site/src/cut/server/filterGraph.test.ts
+++ b/site/src/cut/server/filterGraph.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test";
-import { assertGraphSafe, fexpr } from "./filterGraph";
+import { assertGraphSafe, fanOutInputs, fexpr } from "./filterGraph";
const thrown = (fn: () => unknown): string => {
try {
@@ -34,3 +34,40 @@ describe("filter graph safety", () => {
expect(assertGraphSafe(g)).toBe(g);
});
});
+
+describe("input fan-out", () => {
+ test("a stream read more than once is split once, each read taking its own branch", () => {
+ const g = fanOutInputs([
+ "[0:v]trim=0:2,setpts=PTS-STARTPTS[v0]",
+ "[0:a]atrim=0:2,asetpts=PTS-STARTPTS[a0]",
+ "[0:v]trim=5:7,setpts=PTS-STARTPTS[v1]",
+ "[0:a]atrim=5:7,asetpts=PTS-STARTPTS[a1]",
+ ]);
+ expect(g).toEqual([
+ "[0:v]split=2[fan0_v_0][fan0_v_1]",
+ "[0:a]asplit=2[fan0_a_0][fan0_a_1]",
+ "[fan0_v_0]trim=0:2,setpts=PTS-STARTPTS[v0]",
+ "[fan0_a_0]atrim=0:2,asetpts=PTS-STARTPTS[a0]",
+ "[fan0_v_1]trim=5:7,setpts=PTS-STARTPTS[v1]",
+ "[fan0_a_1]atrim=5:7,asetpts=PTS-STARTPTS[a1]",
+ ]);
+ });
+
+ test("a stream named with and without its index is one stream", () => {
+ expect(fanOutInputs(["[3:a]atrim=0:2[a0]", "[3:a:0]atrim=4:6[a1]"])).toEqual([
+ "[3:a]asplit=2[fan3_a_0][fan3_a_1]",
+ "[fan3_a_0]atrim=0:2[a0]",
+ "[fan3_a_1]atrim=4:6[a1]",
+ ]);
+ });
+
+ test("a pad spelled inside a quoted option value is text, never a read", () => {
+ const g = ["[0:v]trim=0:2[v0]", "[v0]lut3d=file='/tmp/[0:v].cube'[v1]"];
+ expect(fanOutInputs(g)).toEqual(g);
+ });
+
+ test("a stream read once, and a graph's own labels, are left as written", () => {
+ const g = ["[0:v]trim=0:2[v0]", "[1:a:0]atrim=0:2[a0]", "[v0][2:v]overlay[o]"];
+ expect(fanOutInputs(g)).toEqual(g);
+ });
+});
diff --git a/site/src/cut/server/filterGraph.ts b/site/src/cut/server/filterGraph.ts
index 2c974560..aa8eee99 100644
--- a/site/src/cut/server/filterGraph.ts
+++ b/site/src/cut/server/filterGraph.ts
@@ -7,6 +7,64 @@
* assembled graph goes through `assertGraphSafe` on its way to ffmpeg.
*/
+/** An input file's stream named as a graph input: `[0:v]`, `[2:a]`, `[1:a:0]`. */
+const STREAM_PAD = /\[(\d+:[av](?::\d+)?)\]/g;
+
+/** The stream a pad names: `[0:a:0]` and `[0:a]` are both `0:a`. */
+const streamOf = (spec: string) => spec.replace(/:0$/, "");
+
+/** `f` with `fn` applied to each stream pad outside its quoted option values. */
+const mapPads = (f: string, fn: (pad: string, stream: string) => string) =>
+ f
+ .split("'")
+ .map((part, i) => (i % 2 === 0 ? part.replace(STREAM_PAD, (pad, spec: string) => fn(pad, streamOf(spec))) : part))
+ .join("'");
+
+/**
+ * The graph with every input stream it reads more than once fanned out by one
+ * split. ffmpeg feeds each mention of `[0:v]` as a graph input of its own and
+ * prepares every frame for each one: a phone clip's rotation runs once per
+ * mention on every frame, trimmed away or not, so 60 cuts off one recording
+ * rotated each frame 60 times. ffmpeg 8 added a cost per mention on the sound
+ * side too. One split per stream prepares each frame once.
+ */
+export function fanOutInputs(filters: string[]): string[] {
+ // How many times the graph reads each stream.
+ const reads = new Map<string, number>();
+ for (const f of filters) {
+ mapPads(f, (pad, stream) => {
+ reads.set(stream, (reads.get(stream) ?? 0) + 1);
+ return pad;
+ });
+ }
+ const shared = [...reads].filter(([, n]) => n > 1);
+ if (shared.length === 0) {
+ return filters;
+ }
+
+ // Each mention, in graph order, takes the next branch: `[0:v]` read three
+ // times becomes `[fan0_v_0]`, `[fan0_v_1]`, `[fan0_v_2]`.
+ const branch = (stream: string, k: number) => `fan${stream.replace(/:/g, "_")}_${k}`;
+ const taken = new Map<string, number>();
+ const body = filters.map((f) =>
+ mapPads(f, (pad, stream) => {
+ if (reads.get(stream)! < 2) {
+ return pad;
+ }
+ const k = taken.get(stream) ?? 0;
+ taken.set(stream, k + 1);
+ return `[${branch(stream, k)}]`;
+ })
+ );
+
+ // The splits lead the graph, one per shared stream.
+ const splits = shared.map(([stream, n]) => {
+ const split = stream.split(":")[1] === "a" ? "asplit" : "split";
+ return `[${stream}]${split}=${n}${Array.from({ length: n }, (_, k) => `[${branch(stream, k)}]`).join("")}`;
+ });
+ return [...splits, ...body];
+}
+
/** A filter option value single-quoted so the graph parser carries it whole. */
export function fexpr(expr: string): string {
if (expr.includes("'")) {
diff --git a/site/src/cut/server/transcribe.ts b/site/src/cut/server/transcribe.ts
index 8527fb3f..109ab615 100644
--- a/site/src/cut/server/transcribe.ts
+++ b/site/src/cut/server/transcribe.ts
@@ -3,7 +3,7 @@ import { mkdir, mkdtemp, readFile, rm, stat, unlink, writeFile } from "node:fs/p
import os from "node:os";
import path from "node:path";
import { alignCues } from "../lib/cueAlign";
-import { assertGraphSafe } from "./filterGraph";
+import { assertGraphSafe, fanOutInputs } from "./filterGraph";
import { assertLocalRuntime } from "./local-only";
import { createJobRegistry } from "./jobRegistry";
import { mediaPath, readProject } from "./projects";
@@ -467,7 +467,7 @@ async function runTranscribe(job: TranscribeJob, spec: TranscribeSpec) {
[
"-y",
...inputs,
- "-filter_complex", assertGraphSafe(filters.join(";")),
+ "-filter_complex", assertGraphSafe(fanOutInputs(filters).join(";")),
"-map", `[${aLabel}]`,
"-ac", "1",
"-ar", "16000",