Skip to content

Open source CapCut alternative.

TypeScript

fix(site): give every cut its own input so no branch waits on another

taerim.shin531773accf53
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.ts
index 08669289..b64918a7 100644
--- a/site/src/cut/server/exportPipeline.test.ts
+++ b/site/src/cut/server/exportPipeline.test.ts
@@ -389,18 +389,41 @@ describe("export filtergraph timebases", () => {
});
});
-describe("cuts off one file", () => {
- test("each stream of the file is fanned out by one split, so ffmpeg prepares each frame once", async () => {
- const g = await graphFor({
- clips: [clip("a.mp4", { in: 1, out: 3 }), clip("a.mp4", { in: 5, out: 8 }), clip("a.mp4", { in: 10, out: 12 })],
+describe("one input per cut", () => {
+ test("cuts off one file each open their own span, so no branch waits on another", async () => {
+ const runs = await runsFor({
+ clips: [
+ clip("a.mp4", { in: 1, out: 3 }),
+ clip("a.mp4", { in: 5, out: 8 }),
+ clip("a.mp4", { in: 10, out: 12 }),
+ ],
});
- expect(g.filter((f) => f.includes("[0:v]"))).toEqual(["[0:v]split=3[fan0_v_0][fan0_v_1][fan0_v_2]"]);
- expect(g.filter((f) => f.includes("[0:a]"))).toEqual(["[0:a]asplit=3[fan0_a_0][fan0_a_1][fan0_a_2]"]);
- expect(g.filter((f) => f.startsWith("[fan0_v_")).map((f) => f.slice(0, f.indexOf(",")))).toEqual([
- "[fan0_v_0]trim=0.000:3.000",
- "[fan0_v_1]trim=4.000:8.000",
- "[fan0_v_2]trim=9.000:12.000",
+ const enc = runs.find((a) => a.includes("-filter_complex"))!;
+ const inputs = enc.slice(0, enc.indexOf("-filter_complex"));
+ const chain = enc[enc.indexOf("-filter_complex") + 1];
+ // The file is opened once for sound and once more per cut.
+ const opened = inputs.filter((p, i) => inputs[i - 1] === "-i");
+ expect(opened.filter((p) => p.endsWith("a.mp4"))).toHaveLength(4);
+ // Each span is cut at the input, preroll included: in 1 -> from 0,
+ // in 5 -> from 4, in 10 -> from 9.
+ expect(seeks(enc)).toEqual([
+ ["0.000", "3.000"],
+ ["4.000", "4.000"],
+ ["9.000", "3.000"],
]);
+ // So the picture never trims a shared decoder.
+ expect(chain).not.toContain(":v]trim=");
+ // Sound still trims in the graph, off one opening fanned out by one
+ // asplit: audio frames are small enough that holding them costs little.
+ expect(chain).toContain("[0:a]asplit=3[fan0_a_0][fan0_a_1][fan0_a_2]");
+ expect(chain).toContain("[fan0_a_0]atrim=");
+ });
+
+ test("a still has no source span, so it keeps reading its looped input", async () => {
+ const runs = await runsFor({ clips: [{ ...clip("a.png"), image: true }] });
+ const enc = runs.find((a) => a.includes("-filter_complex"))!;
+ expect(seeks(enc)).toHaveLength(0);
+ expect(enc[enc.indexOf("-filter_complex") + 1]).toContain(":v]setpts=PTS-STARTPTS");
});
});
@@ -408,7 +431,7 @@ describe("a speed curve in the filtergraph", () => {
test("a curved clip lays its picture through the map and reads baked sound", async () => {
const curved = clip("a.mp4", { speedCurve: [[0, 1], [4, 4]] });
const g = await graphFor({ clips: [curved, clip("b.mp4")] });
- const video = g.find((f) => f.startsWith("[0:v]trim="))!;
+ const video = pictureAt(g, 0);
expect(video).toContain("setpts='(clip(T-0,0,");
expect(video).not.toContain("(PTS-STARTPTS)/");
// The baked WAV joins the inputs after the two media files, and the
@@ -425,11 +448,11 @@ describe("a speed curve in the filtergraph", () => {
describe("smooth slow motion in the filtergraph", () => {
test("a slowed clip runs motion interpolation over its whole span", async () => {
const g = await graphFor({ clips: [clip("a.mp4", { speed: 0.5, smoothSlow: true }), clip("b.mp4")] });
- const video = g.find((f) => f.startsWith("[0:v]trim="))!;
+ const video = pictureAt(g, 0);
expect(video).toContain("minterpolate=fps=30:mi_mode=mci");
expect(video).not.toContain("split=");
// The plain clip beside it is untouched.
- expect(g.find((f) => f.startsWith("[1:v]trim="))).not.toContain("minterpolate");
+ expect(pictureAt(g, 1)).not.toContain("minterpolate");
});
test("a curve that dips is interpolated in the dip alone", async () => {
@@ -443,7 +466,7 @@ describe("smooth slow motion in the filtergraph", () => {
],
});
const g = await graphFor({ clips: [curved, clip("b.mp4")] });
- const head = g.find((f) => f.startsWith("[0:v]trim="))!;
+ const head = pictureAt(g, 0);
expect(head).toContain("split=3[smic0_0][smic0_1][smic0_2]");
const pieces = g.filter((f) => f.startsWith("[smic0_"));
expect(pieces.length).toBe(3);
@@ -456,15 +479,17 @@ describe("smooth slow motion in the filtergraph", () => {
test("a curve stamps the preroll before 0, so interpolation never sees it", async () => {
// The 1 s preroll ahead of the in point must land before 0 for the
- // interpolation's trim=start=0 to drop it: 4.5 s is preroll, 5 s the in point.
+ // interpolation's trim=start=0 to drop it. The input opens at the preroll
+ // (source 4 s) and `-ss` restamps it to 0, so the in point sits at T=1 and
+ // the preroll runs T=0..1 — the same picture, read on the seeked clock.
const curved = clip("a.mp4", { in: 5, out: 9, smoothSlow: true, speedCurve: [[0, 0.5], [4, 0.5]] });
- const head = (await graphFor({ clips: [curved] })).find((f) => f.startsWith("[0:v]trim="))!;
+ const head = pictureAt(await graphFor({ clips: [curved] }), 0);
expect(head).toContain("trim=start=0,");
const expr = /setpts='([^']+)'/.exec(head)![1];
const at = (T: number) =>
new Function("T", "TB", "clip", "min", `return ${expr};`)(T, 1, (x: number, lo: number, hi: number) => Math.min(hi, Math.max(lo, x)), Math.min) as number;
- expect(at(4.5)).toBeLessThan(0);
- expect(at(5)).toBeCloseTo(0, 6);
+ expect(at(0.5)).toBeLessThan(0);
+ expect(at(1)).toBeCloseTo(0, 6);
expect(at(5.5)).toBeGreaterThan(0);
});
@@ -1009,8 +1034,11 @@ describe("a reversed clip in the filtergraph", () => {
expect(inputs.some((p) => p.endsWith("turned_clip_0.mov"))).toBe(true);
const idx = inputs.filter((p, i) => inputs[i - 1] === "-i").findIndex((p) => p.endsWith("turned_clip_0.mov"));
const g = graph[graph.indexOf("-filter_complex") + 1].split(";");
- const video = g.find((f) => f.startsWith(`[${idx}:v]trim=`))!;
- expect(video).toContain("trim=0.000:7.000,setpts=(PTS-1.000/TB)/2.000");
+ const video = pictureAt(g, 0);
+ // The mirrored span is cut at the input now; the re-stamp is unchanged.
+ expect(seeks(graph).some(([ss, t]) => ss === "0.000" && t === "7.000")).toBe(true);
+ expect(video).toContain("setpts=(PTS-1.000/TB)/2.000");
+ expect(video.startsWith("[2:v]setpts=")).toBe(true);
expect(video).not.toContain("reverse");
const audio = g.find((f) => f.startsWith(`[${idx}:a]atrim=`))!;
expect(audio).toContain("atrim=1.000:7.000");
@@ -1029,8 +1057,8 @@ describe("a reversed clip in the filtergraph", () => {
reverse: true,
});
const g = await graphFor({ clips: [curved, clip("b.mp4")] });
- const video = g.find((f) => f.startsWith("[1:v]trim="))!;
- expect(video).toContain("trim=0.000:4.000,setpts='(clip(T-0,0,");
+ const video = pictureAt(g, 0);
+ expect(video).toContain("setpts='(clip(T-0,0,");
const first = g.find((f) => f.includes("apad=whole_dur="))!;
expect(first).toContain(`apad=whole_dur=${retimeOf(curved).len.toFixed(3)}`);
});
@@ -1047,6 +1075,16 @@ const encodeRun = async (over: Partial<ExportSpec>) => {
};
const arg = (args: string[], flag: string) => args[args.indexOf(flag) + 1];
+/** A clip's picture chain, in clip order — footage now reads a seeked input
+ * (`setpts=`) where it used to trim a shared one (`trim=`), and a still has
+ * always read its looped input straight. */
+const pictureAt = (g: string[], nth: number) =>
+ g.filter((f) => /^\[\d+:v\](trim=|setpts=)/.test(f))[nth]!;
+
+/** The `-ss`/`-t` pairs ffmpeg is handed, in input order. */
+const seeks = (args: string[]) =>
+ args.flatMap((v, i) => (v === "-ss" ? [[args[i + 1], args[i + 3]] as const] : []));
+
describe("delivery", () => {
test("source bitrates reach the worker and Mac encoders", async () => {
const args = await encodeRun({ bitrate: 140_000, audioBitrate: 48_000 });
@@ -1179,7 +1217,8 @@ describe("delivery", () => {
test("no range delivers the whole cut", async () => {
const args = await encodeRun({ duration: 20 });
expect(arg(args, "-filter_complex")).not.toContain("[vrange]");
- expect(arg(args, "-t")).toBe("20.000");
+ // Inputs carry their own `-t` now, so read the one on the output side.
+ expect(args[args.lastIndexOf("-t") + 1]).toBe("20.000");
});
});
@@ -1220,7 +1259,7 @@ describe("clip color in the filtergraph", () => {
expect(cubes).toHaveLength(1);
expect(cubes[0].file).toBe(path.join("/tmp/graph-test", "clip_0.cube"));
expect(cubes[0].data).toContain("LUT_3D_SIZE 33");
- const line = g.find((l) => l.startsWith("[0:v]"))!;
+ const line = pictureAt(g, 0);
// The framing scale spells the file's matrix and range; the picture goes
// to 16-bit RGB, through the LUT, and comes back as BT.709 video before
// the letterbox pad.
@@ -1246,7 +1285,7 @@ describe("clip color in the filtergraph", () => {
lutSizeWide: 65,
clips: [clip("log.mov", { color: { profile: "apple-log", matrix: "bt2020nc", fullRange: false } })],
});
- const line = g.find((l) => l.startsWith("[0:v]"))!;
+ const line = pictureAt(g, 0);
expect(line).toMatch(/scale=[^,]*in_color_matrix=bt2020:in_range=tv/);
expect(line).toContain("format=gbrp16le,lut3d=file=");
expect(written.find((w) => w.file.endsWith(".cube"))!.data).toContain("LUT_3D_SIZE 65");
@@ -1256,7 +1295,7 @@ describe("clip color in the filtergraph", () => {
const g = await graphFor({
clips: [clip("sd.mp4", { color: { profile: "rec709", matrix: "bt601", fullRange: true } })],
});
- const line = g.find((l) => l.startsWith("[0:v]"))!;
+ const line = pictureAt(g, 0);
expect(line).not.toContain("lut3d");
expect(line).toContain("scale:in_color_matrix=bt601:in_range=pc:out_color_matrix=bt709:out_range=tv");
expect(line.indexOf("out_range=tv")).toBeLessThan(line.indexOf("pad=1080:1920"));
@@ -1267,7 +1306,7 @@ describe("clip color in the filtergraph", () => {
clips: [clip("a.mp4", { grade: { exposure: 5, sharpen: 25, clarity: 30 } })],
});
const joined = g.join(";");
- const head = g.find((l) => l.startsWith("[0:v]"))!;
+ const head = pictureAt(g, 0);
expect(head).toContain("lut3d=file=");
expect(head).toMatch(/scale=out_color_matrix=bt709:out_range=pc,format=yuv444p16le\[dtic0\]$/);
// The detail pass: bases from the input luma, gained detail merged back.
@@ -1345,7 +1384,7 @@ describe("clip color in the filtergraph", () => {
const g = await graphFor({
clips: [clip("a.mp4", { speed: 0.5, smoothSlow: true, grade: { exposure: 5 } })],
});
- const line = g.find((l) => l.startsWith("[0:v]"))!;
+ const line = pictureAt(g, 0);
expect(line).toContain("minterpolate=");
expect(line.indexOf("minterpolate=")).toBeLessThan(line.indexOf("scale:in_color_matrix=bt709:in_range=tv,format=gbrp16le,lut3d"));
});
@@ -1388,7 +1427,7 @@ describe("an HDR delivery", () => {
test("composites at ten bits in HLG: every clip maps into Rec.2020 and comes back as 10-bit video", async () => {
const g = await graphFor(hdrSpec());
- const line = g.find((l) => l.startsWith("[0:v]"))!;
+ const line = pictureAt(g, 0);
// An ungraded Rec.709 clip is a conversion here, so it takes the LUT.
expect(line).toMatch(/scale=[^,]*in_color_matrix=bt709:in_range=tv/);
expect(line).toContain("format=gbrp16le,lut3d=file=");
@@ -1405,9 +1444,9 @@ describe("an HDR delivery", () => {
test("an HLG file ungraded runs no LUT, and a graded one grades in place", async () => {
const hlg = { profile: "hlg" as const, matrix: "bt2020nc" as const, fullRange: false };
const plain = await graphFor(hdrSpec({ clips: [clip("h.mp4", { color: hlg })] }));
- expect(plain.find((l) => l.startsWith("[0:v]"))).not.toContain("lut3d");
+ expect(pictureAt(plain, 0)).not.toContain("lut3d");
const graded = await graphFor(hdrSpec({ clips: [clip("h.mp4", { color: hlg, grade: { exposure: 10 } })] }));
- const line = graded.find((l) => l.startsWith("[0:v]"))!;
+ const line = pictureAt(graded, 0);
expect(line).toMatch(/scale=[^,]*in_color_matrix=bt2020:in_range=tv/);
expect(line).toContain("lut3d=file=");
expect(line).toContain("scale=out_color_matrix=bt2020:out_range=tv");
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index 08fbbb0b..b14f1469 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -627,7 +627,22 @@ const SOURCE_PREROLL = 1;
* screen at the in point comes along (stamped before 0) and the re-stamp to
* the output rate shows it on the clip's first frame. */
const sourceSpan = (idx: number, c: { in: number; out: number }, rt: Retime) =>
- `[${idx}:v]trim=${num(Math.max(0, c.in - SOURCE_PREROLL))}:${num(c.out)},setpts=${retimedPts(rt, c.in)}`;
+ `[${idx}:v]trim=${num(spanStart(c))}:${num(c.out)},setpts=${retimedPts(rt, c.in)}`;
+
+/** Where a footage clip's span starts in its source, preroll included. */
+const spanStart = (c: { in: number }) => Math.max(0, c.in - SOURCE_PREROLL);
+
+/**
+ * The same span read off an input already seeked to it, so the picture never
+ * trims inside the graph.
+ *
+ * `-ss` before `-i` restamps the seeked input to start at 0, so the in point
+ * sits at `c.in - spanStart(c)` — the preroll it kept — and the re-stamp
+ * measures from there. Everything downstream sees the frames and timestamps
+ * the trimmed chain produced.
+ */
+const seekedSpan = (idx: number, c: { in: number }, rt: Retime) =>
+ `[${idx}:v]setpts=${retimedPts(rt, c.in - spanStart(c))}`;
/** A curve clamps everything before its first knot to 0; the preroll keeps
* its own seconds before 0, as one rate does, so a trim at 0 drops it. */
@@ -1181,8 +1196,11 @@ export async function runExport(
// Resolve paths in order first so ffmpeg input indices stay deterministic,
// then probe every file's streams concurrently.
const paths = await Promise.all(mediaFiles.map((f) => resolveMedia(io.stat, mediaPathFor, f)));
+ // file → the path ffmpeg opens, so a cut can open its own span below.
+ const pathOf = new Map<string, string>();
mediaFiles.forEach((f, i) => {
inputIndex.set(f, nInputs++);
+ pathOf.set(f, paths[i]);
inputs.push("-i", paths[i]);
});
await Promise.all(
@@ -1217,6 +1235,7 @@ export async function runExport(
// The turned copies: already probed at the bake.
for (const [file, streams] of turned) {
inputIndex.set(file, nInputs++);
+ pathOf.set(file, file);
inputs.push("-i", file);
audioPresence.set(file, streams.audio);
videoPresence.set(file, streams.video);
@@ -1743,6 +1762,40 @@ export async function runExport(
inputs.push("-i", file);
baked.set(c, { ...bake, idx });
}
+
+ /**
+ * One input per footage cut, seeked to that cut's span, instead of every cut
+ * trimming one shared decoder.
+ *
+ * `[0:v]trim=a:b` written N times makes ffmpeg split one decoder N ways, and
+ * `concat` reads those branches in order — so every branch past the one being
+ * read holds the frames already handed to it. What it holds grows with how
+ * long the encode runs, which is why a longer timeline dies where a short one
+ * survives, and why a slower preset dies earlier at the same length. Measured
+ * on one 30-cut 104s render, 1080x1920, veryfast, no subtitle lane:
+ *
+ * one input, 30 trim branches 8,320MB
+ * an input per cut 3,311MB
+ *
+ * Seeking at the input also skips decoding the head of each span, so a cut
+ * late in a long file no longer walks the file to reach its first frame.
+ *
+ * Left trimming in the graph: stills (no source span), removal clips (their
+ * picture is the keyed pair, not this file), and clips whose source is read
+ * twice over (a sound twin), where a second opening buys nothing.
+ */
+ const cutVideoInput = new Map<number, number>();
+ spec.clips.forEach((c, j) => {
+ if (c.image || clipRemovalInput.get(j) || c.hidden) return;
+ if (!c.file || !videoPresence.get(c.file)) return;
+ const src = pathOf.get(c.file);
+ if (!src) return;
+ const from = spanStart(c);
+ const span = Math.max(1 / fps, c.out - from);
+ cutVideoInput.set(j, nInputs++);
+ inputs.push("-ss", num(from), "-t", num(span), "-i", src);
+ });
+
/**
* The head of an audio chain reading a clip's sound over the timeline-local
* window `[fromT, toT]` (seconds from the clip's head; negative reaches
@@ -2286,9 +2339,12 @@ export async function runExport(
alphaSet(`rvc${j}`, `rva${j}`, `rvs${j}`, `rv${j}`, ",tpad=stop_mode=clone:stop_duration=1");
timebase = `[rvs${j}]null`;
} else {
+ const cutIdx = cutVideoInput.get(j);
timebase = c.image
? `[${idx}:v]setpts=PTS-STARTPTS`
- : sourceSpan(idx, c, rt);
+ : cutIdx === undefined
+ ? sourceSpan(idx, c, rt)
+ : seekedSpan(cutIdx, c, rt);
}
if ((c.image || rmIn || videoPresence.get(c.file)) && !c.hidden) {
const region = regionPx(c.frame, W, H);