Skip to content

Open source CapCut alternative.

TypeScript

fix(site): render long exports in bounded passes [rebuild]

David Leaf3f676ef214
diff --git a/site/src/cut/server/exportPipeline.e2e.test.ts b/site/src/cut/server/exportPipeline.e2e.test.ts
index 1ba56d64..a2a3ea56 100644
--- a/site/src/cut/server/exportPipeline.e2e.test.ts
+++ b/site/src/cut/server/exportPipeline.e2e.test.ts
@@ -1,9 +1,9 @@
import { describe, expect, test } from "bun:test";
import { spawnSync } from "node:child_process";
-import { mkdtemp, rm, stat } from "node:fs/promises";
+import { cp, mkdir, mkdtemp, rm, stat } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
-import { runExport, type ExportSpec, type RenderHandle } from "./exportPipeline";
+import { passWindows, runExport, type ExportSpec, type RenderHandle } from "./exportPipeline";
// The graph tests assert shapes; this one hands the real graph to a real
// ffmpeg. It synthesizes every asset kind from lavfi sources, builds one
@@ -93,6 +93,132 @@ const probe = (file: string) => {
const W = 320;
const H = 568;
+/** Synthesizes every asset kind from lavfi sources into `mediaDir` (project
+ * media) and `tmpDir` (client-painted stills), and returns one project that
+ * uses every ExportSpec feature over them. */
+async function everyFeature(mediaDir: string, tmpDir: string): Promise<ExportSpec> {
+ // ---- Footage, stills, and the audio bed (project media) ----------
+ const vid = (name: string, src: string, hz: number, secs: number) =>
+ ff([
+ "-f", "lavfi", "-i", `${src}=size=320x240:rate=24:duration=${secs}`,
+ "-f", "lavfi", "-i", `sine=frequency=${hz}:sample_rate=44100:duration=${secs}`,
+ "-shortest", "-pix_fmt", "yuv420p", "-c:a", "aac",
+ path.join(mediaDir, name),
+ ]);
+ vid("a.mp4", "testsrc2", 440, 3);
+ vid("b.mp4", "smptebars", 660, 3);
+ vid("o.mp4", "testsrc", 880, 3);
+ ff([
+ "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=1:duration=1",
+ "-frames:v", "1", path.join(mediaDir, "img.png"),
+ ]);
+ ff([
+ "-f", "lavfi", "-i", "sine=frequency=220:sample_rate=44100:duration=6",
+ "-c:a", "aac", path.join(mediaDir, "music.m4a"),
+ ]);
+
+ // ---- Client-painted stills and the person matte (job.tmpDir) -----
+ const png = (name: string, chain: string) =>
+ ff(["-f", "lavfi", "-i", chain, "-frames:v", "1", path.join(tmpDir, name)]);
+ png("mask.png", `color=c=white:s=${W}x${H},format=gray`);
+ png("omask.png", "color=c=0xC0C0C0:s=160x160,format=gray");
+ png("el.png", `[email protected]:s=${W}x${H},format=rgba`);
+ png("f0.png", "color=c=blue:s=120x80,format=rgba");
+ png("f1.png", "color=c=green:s=120x80,format=rgba");
+ png("blank.png", "[email protected]:s=120x80,format=rgba");
+ png("border.png", `[email protected]:s=${W}x${H},format=rgba`);
+ png("oborder.png", "[email protected]:s=128x256,format=rgba");
+ png("shadow.png", `[email protected]:s=${W}x${H},format=rgba`);
+ png("cap.png", `[email protected]:s=${W}x${H},format=rgba`);
+ png("sub_blank.png", `[email protected]:s=${W}x${H},format=rgba`);
+ ff([
+ "-f", "lavfi", "-i", `color=c=gray:s=${W}x${H}:d=6:r=24`,
+ "-pix_fmt", "yuv420p", path.join(tmpDir, "matte.mp4"),
+ ]);
+
+ // Timeline: 1.6s (sped-up cover crop) + 2s (graded, masked, keyed)
+ // + 1s gap + 1.5s regioned still = 6.1s.
+ return {
+ projectId: "e2e",
+ width: W,
+ height: H,
+ fps: 24,
+ crf: 30,
+ preset: "ultrafast",
+ duration: 6.1,
+ background: "#102030",
+ clips: [
+ {
+ file: "a.mp4", in: 0, out: 2, muted: false, volume: 0.8,
+ fit: "fill", zoom: 1.4, panX: 0.3, panY: -0.2,
+ speed: 1.25,
+ look: "vhs", lookAmount: 0.7,
+ animIn: { style: "pop", seconds: 0.4 },
+ transition: 0.5, transitionStyle: "crosszoom",
+ },
+ {
+ file: "b.mp4", in: 0, out: 2, muted: false,
+ fit: "fit", reverse: true,
+ grade: { brightness: 5, contrast: 8, saturation: -10, temperature: 12, hue: 10 },
+ mask: { file: "mask.png" },
+ kf: [
+ { t: 0, x: 0.5, y: 0.5, scale: 1, rotation: 0, opacity: 1 },
+ { t: 1.5, x: 0.6, y: 0.4, scale: 0.8, rotation: 15, opacity: 1 },
+ ],
+ animOut: { style: "fade", seconds: 0.4 },
+ transition: 0.4, transitionStyle: "wipeleft",
+ },
+ { file: "", in: 0, out: 1, muted: true, hidden: true },
+ {
+ file: "img.png", in: 0, out: 1.5, muted: true, image: true,
+ frame: { x: 0.5, y: 0.45, w: 0.45, h: 0.5 }, fit: "fill",
+ mask: { subject: { feather: 1 } },
+ border: "border.png",
+ shadow: { file: "shadow.png" },
+ },
+ ],
+ overlayVideos: [
+ {
+ file: "o.mp4", in: 0.2, out: 1.8, start: 0.8, track: 1,
+ frame: { x: 0.55, y: 0.5, w: 0.4, h: 0.45 },
+ fit: "fill", zoom: 1.3, panX: -0.4,
+ muted: false, volume: 0.5, speed: 2,
+ headFade: 0.3,
+ mask: { file: "omask.png" },
+ border: "oborder.png",
+ kf: [
+ { t: 0, x: 0.75, y: 0.7, scale: 1, rotation: 0, opacity: 1 },
+ { t: 0.6, x: 0.65, y: 0.75, scale: 1.1, rotation: -10, opacity: 1 },
+ ],
+ },
+ ],
+ audio: [
+ { file: "music.m4a", in: 0, out: 4, start: 0, volume: 0.6, fadeIn: 0.2, fadeOut: 0.4 },
+ { file: "a.mp4", in: 0, out: 1, start: 2, volume: 1, duck: 0.3, speed: 1.1 },
+ ],
+ overlays: [
+ { file: "el.png", start: 0.5, end: 2.5, lane: 1 },
+ {
+ frames: [
+ { file: "f0.png", duration: 0.5 },
+ { file: "f1.png", duration: 0.5 },
+ ],
+ blank: "blank.png", x: 40, y: 60, start: 1, end: 2,
+ subject: { invert: true, feather: 2 }, lane: 0,
+ },
+ ],
+ behindMask: { file: "matte.mp4", from: 0.5 },
+ effects: [
+ { effect: "vignette", amount: 0.6, start: 0.5, end: 2, lane: 0 },
+ { effect: "zoom", amount: 0.4, focus: { x: 0.5, y: 0.4 }, ramp: 0.4, start: 2, end: 3, lane: 1 },
+ ],
+ captions: [
+ { file: "cap.png", start: 0.3, end: 1.2 },
+ { file: "cap.png", start: 1.4, end: 2.0 },
+ ],
+ };
+}
+
for (const delivery of DELIVERIES) describe("export pipeline end to end", () => {
test.skipIf(!tools)(
`a project using every pipeline feature renders through real ffmpeg as ${delivery.name}`,
@@ -100,125 +226,8 @@ for (const delivery of DELIVERIES) describe("export pipeline end to end", () =>
const mediaDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-media-"));
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-job-"));
try {
- // ---- Footage, stills, and the audio bed (project media) ----------
- const vid = (name: string, src: string, hz: number, secs: number) =>
- ff([
- "-f", "lavfi", "-i", `${src}=size=320x240:rate=24:duration=${secs}`,
- "-f", "lavfi", "-i", `sine=frequency=${hz}:sample_rate=44100:duration=${secs}`,
- "-shortest", "-pix_fmt", "yuv420p", "-c:a", "aac",
- path.join(mediaDir, name),
- ]);
- vid("a.mp4", "testsrc2", 440, 3);
- vid("b.mp4", "smptebars", 660, 3);
- vid("o.mp4", "testsrc", 880, 3);
- ff([
- "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=1:duration=1",
- "-frames:v", "1", path.join(mediaDir, "img.png"),
- ]);
- ff([
- "-f", "lavfi", "-i", "sine=frequency=220:sample_rate=44100:duration=6",
- "-c:a", "aac", path.join(mediaDir, "music.m4a"),
- ]);
-
- // ---- Client-painted stills and the person matte (job.tmpDir) -----
- const png = (name: string, chain: string) =>
- ff(["-f", "lavfi", "-i", chain, "-frames:v", "1", path.join(tmpDir, name)]);
- png("mask.png", `color=c=white:s=${W}x${H},format=gray`);
- png("omask.png", "color=c=0xC0C0C0:s=160x160,format=gray");
- png("el.png", `[email protected]:s=${W}x${H},format=rgba`);
- png("f0.png", "color=c=blue:s=120x80,format=rgba");
- png("f1.png", "color=c=green:s=120x80,format=rgba");
- png("blank.png", "[email protected]:s=120x80,format=rgba");
- png("border.png", `[email protected]:s=${W}x${H},format=rgba`);
- png("oborder.png", "[email protected]:s=128x256,format=rgba");
- png("shadow.png", `[email protected]:s=${W}x${H},format=rgba`);
- png("cap.png", `[email protected]:s=${W}x${H},format=rgba`);
- png("sub_blank.png", `[email protected]:s=${W}x${H},format=rgba`);
- ff([
- "-f", "lavfi", "-i", `color=c=gray:s=${W}x${H}:d=6:r=24`,
- "-pix_fmt", "yuv420p", path.join(tmpDir, "matte.mp4"),
- ]);
-
- // Timeline: 1.6s (sped-up cover crop) + 2s (graded, masked, keyed)
- // + 1s gap + 1.5s regioned still = 6.1s.
const spec: ExportSpec = {
- projectId: "e2e",
- width: W,
- height: H,
- fps: 24,
- crf: 30,
- preset: "ultrafast",
- duration: 6.1,
- background: "#102030",
- clips: [
- {
- file: "a.mp4", in: 0, out: 2, muted: false, volume: 0.8,
- fit: "fill", zoom: 1.4, panX: 0.3, panY: -0.2,
- speed: 1.25,
- look: "vhs", lookAmount: 0.7,
- animIn: { style: "pop", seconds: 0.4 },
- transition: 0.5, transitionStyle: "crosszoom",
- },
- {
- file: "b.mp4", in: 0, out: 2, muted: false,
- fit: "fit", reverse: true,
- grade: { brightness: 5, contrast: 8, saturation: -10, temperature: 12, hue: 10 },
- mask: { file: "mask.png" },
- kf: [
- { t: 0, x: 0.5, y: 0.5, scale: 1, rotation: 0, opacity: 1 },
- { t: 1.5, x: 0.6, y: 0.4, scale: 0.8, rotation: 15, opacity: 1 },
- ],
- animOut: { style: "fade", seconds: 0.4 },
- transition: 0.4, transitionStyle: "wipeleft",
- },
- { file: "", in: 0, out: 1, muted: true, hidden: true },
- {
- file: "img.png", in: 0, out: 1.5, muted: true, image: true,
- frame: { x: 0.5, y: 0.45, w: 0.45, h: 0.5 }, fit: "fill",
- mask: { subject: { feather: 1 } },
- border: "border.png",
- shadow: { file: "shadow.png" },
- },
- ],
- overlayVideos: [
- {
- file: "o.mp4", in: 0.2, out: 1.8, start: 0.8, track: 1,
- frame: { x: 0.55, y: 0.5, w: 0.4, h: 0.45 },
- fit: "fill", zoom: 1.3, panX: -0.4,
- muted: false, volume: 0.5, speed: 2,
- headFade: 0.3,
- mask: { file: "omask.png" },
- border: "oborder.png",
- kf: [
- { t: 0, x: 0.75, y: 0.7, scale: 1, rotation: 0, opacity: 1 },
- { t: 0.6, x: 0.65, y: 0.75, scale: 1.1, rotation: -10, opacity: 1 },
- ],
- },
- ],
- audio: [
- { file: "music.m4a", in: 0, out: 4, start: 0, volume: 0.6, fadeIn: 0.2, fadeOut: 0.4 },
- { file: "a.mp4", in: 0, out: 1, start: 2, volume: 1, duck: 0.3, speed: 1.1 },
- ],
- overlays: [
- { file: "el.png", start: 0.5, end: 2.5, lane: 1 },
- {
- frames: [
- { file: "f0.png", duration: 0.5 },
- { file: "f1.png", duration: 0.5 },
- ],
- blank: "blank.png", x: 40, y: 60, start: 1, end: 2,
- subject: { invert: true, feather: 2 }, lane: 0,
- },
- ],
- behindMask: { file: "matte.mp4", from: 0.5 },
- effects: [
- { effect: "vignette", amount: 0.6, start: 0.5, end: 2, lane: 0 },
- { effect: "zoom", amount: 0.4, focus: { x: 0.5, y: 0.4 }, ramp: 0.4, start: 2, end: 3, lane: 1 },
- ],
- captions: [
- { file: "cap.png", start: 0.3, end: 1.2 },
- { file: "cap.png", start: 1.4, end: 2.0 },
- ],
+ ...(await everyFeature(mediaDir, tmpDir)),
...delivery.over,
};
@@ -254,3 +263,164 @@ for (const delivery of DELIVERIES) describe("export pipeline end to end", () =>
120_000
);
});
+
+/** Every frame and sample a file decodes to, one hash per frame. */
+// A render's content: each picture's hash, and every decoded sample as
+// 16-bit — a single pass hands the muxer sound in whatever packets the graph
+// made, a join in even ones, so the samples compare whole.
+const ffmpegOut = (args: string[]) => {
+ const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", ...args], { maxBuffer: 256 << 20 });
+ if (r.status !== 0) {
+ throw new Error(`ffmpeg ${args.join(" ")} failed:\n${r.stderr}`);
+ }
+ return r.stdout;
+};
+const framesOf = (file: string) => ({
+ pictures: ffmpegOut(["-i", file, "-map", "0:v?", "-f", "framemd5", "-"]).toString().split("\n").filter((l) => !l.startsWith("#")),
+ samples: new Int16Array(new Uint8Array(ffmpegOut(["-i", file, "-map", "0:a?", "-f", "s16le", "-c:a", "pcm_s16le", "-"])).buffer),
+});
+
+// The largest gap between two runs of samples, which must be as long.
+const sampleGap = (a: Int16Array, b: Int16Array) => {
+ expect(a.length).toBe(b.length);
+ return a.reduce((gap, v, i) => Math.max(gap, Math.abs(v - b[i])), 0);
+};
+
+describe("a delivery rendered in passes", () => {
+ // The same every-feature project rendered in one pass and in passes of two
+ // pieces. ProRes is intra-only and the sound is PCM, so the same frames and
+ // samples encode to the same bits whichever pass made them: the joined
+ // file, its mastered mix and its stems decode exactly as the one pass's do.
+ // Both sounds are resampled (the mix to 48kHz, the stems to theirs), the
+ // case where a pass cut off the sample grid would show.
+ test.skipIf(!tools)(
+ "joins into the frames, mix and stems one pass renders",
+ async () => {
+ const mediaDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-media-"));
+ const tmpDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-job-"));
+ const jobsDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-jobs-"));
+ try {
+ const spec: ExportSpec = {
+ ...(await everyFeature(mediaDir, tmpDir)),
+ codec: "prores4444",
+ container: "mov",
+ audioCodec: "pcm",
+ audioSampleRate: 48000,
+ loudness: -16,
+ truePeakCeiling: -1,
+ stemPlan: [
+ { name: "Dialogue", lane: null, file: "1 Dialogue.wav" },
+ { name: "Music", lane: 0, file: "2 Music.wav" },
+ ],
+ };
+ expect(passWindows(spec, () => ({ pixels: 0, fps: 30 }), 2).length).toBeGreaterThan(2);
+
+ // Each render runs in its own copy of the job dir, so the stems it
+ // packs stay on disk beside its file.
+ const render = async (name: string, passPieces: number) => {
+ const dir = path.join(jobsDir, name);
+ await cp(tmpDir, dir, { recursive: true });
+ const job: RenderHandle = {
+ tmpDir: dir,
+ outPath: path.join(dir, "out.mov"),
+ stemsPath: path.join(dir, "stems.zip"),
+ progress: 0,
+ log: [],
+ };
+ await runExport(job, spec, (file) => path.join(mediaDir, file), undefined, passPieces);
+ return { file: job.outPath, stems: spec.stemPlan!.map((_, i) => path.join(dir, `stem_${i}.wav`)) };
+ };
+ const whole = await render("whole", Infinity);
+ const passes = await render("passes", 2);
+
+ expect(framesOf(passes.file)).toEqual(framesOf(whole.file));
+ for (const [i, stem] of passes.stems.entries()) {
+ expect(framesOf(stem)).toEqual(framesOf(whole.stems[i]));
+ }
+ } finally {
+ await rm(mediaDir, { recursive: true, force: true });
+ await rm(tmpDir, { recursive: true, force: true });
+ await rm(jobsDir, { recursive: true, force: true });
+ }
+ },
+ 240_000
+ );
+});
+
+describe("a long recording cut out of order", () => {
+ // Cuts from all over one 40s recording, played out of source order, with
+ // sound handles, a split edit, a transition, a cross dissolve, a layered
+ // clip and a sound entry reading the same file late. Rendered in passes of
+ // three, each pass opens the file at its own earliest read; the file
+ // decodes exactly as the one pass's does.
+ test.skipIf(!tools)(
+ "renders in passes as in one",
+ async () => {
+ const mediaDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-media-"));
+ const jobsDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-jobs-"));
+ try {
+ ff([
+ "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=24:duration=40",
+ "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100:duration=40",
+ "-shortest", "-pix_fmt", "yuv420p", "-c:a", "aac", "-g", "48",
+ path.join(mediaDir, "talk.mp4"),
+ ]);
+ const at = [31, 4, 22, 12, 36, 8, 27, 17, 2, 33, 14, 25];
+ const clips: ExportSpec["clips"] = at.map((from, k) => ({
+ file: "talk.mp4",
+ in: from,
+ out: from + 1.5,
+ muted: false,
+ soundBack: 0.4,
+ soundAhead: 0.4,
+ ...(k === 3 ? { speed: 1.5 } : {}),
+ ...(k === 5 ? { transition: 0.4 } : {}),
+ ...(k === 7 ? { soundCross: 0.2 } : {}),
+ ...(k === 9 ? { soundLead: 0.3, splitFade: 0.05 } : {}),
+ }));
+ const spec: ExportSpec = {
+ projectId: "e2e",
+ width: W,
+ height: H,
+ fps: 24,
+ crf: 30,
+ preset: "ultrafast",
+ duration: clips.reduce((sum, c) => sum + (c.out - c.in) / (c.speed ?? 1), 0),
+ clips,
+ overlayVideos: [
+ {
+ file: "talk.mp4", in: 29, out: 31, start: 12, track: 1,
+ frame: { x: 0.55, y: 0.5, w: 0.4, h: 0.45 }, muted: false, volume: 0.5,
+ },
+ ],
+ audio: [{ file: "talk.mp4", in: 35, out: 38, start: 9, volume: 0.4 }],
+ overlays: [],
+ codec: "prores4444",
+ container: "mov",
+ audioCodec: "pcm",
+ };
+ expect(passWindows(spec, () => ({ pixels: 0, fps: 30 }), 3).length).toBeGreaterThan(3);
+
+ const render = async (name: string, passPieces: number) => {
+ const dir = path.join(jobsDir, name);
+ await mkdir(dir);
+ const job: RenderHandle = { tmpDir: dir, outPath: path.join(dir, "out.mov"), progress: 0, log: [] };
+ await runExport(job, spec, (file) => path.join(mediaDir, file), undefined, passPieces);
+ return job.outPath;
+ };
+ const whole = await render("whole", Infinity);
+ const passes = await render("passes", 3);
+ // An AAC decoder fills noise-coded bands from a generator a seek
+ // restarts, so a pass that reads from its window's first source
+ // second carries the same sound to within one 16-bit step.
+ const [got, want] = [framesOf(passes), framesOf(whole)];
+ expect(got.pictures).toEqual(want.pictures);
+ expect(sampleGap(got.samples, want.samples)).toBeLessThanOrEqual(1);
+ } finally {
+ await rm(mediaDir, { recursive: true, force: true });
+ await rm(jobsDir, { recursive: true, force: true });
+ }
+ },
+ 240_000
+ );
+});
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.ts
index fe9817bf..09bda98e 100644
--- a/site/src/cut/server/exportPipeline.test.ts
+++ b/site/src/cut/server/exportPipeline.test.ts
@@ -5,6 +5,7 @@ import {
colorParamsFilter,
colorTagArgs,
narrowSpecToRange,
+ passWindows,
runExport,
videoCodecArgs,
type ExportPipelineIO,
@@ -30,7 +31,10 @@ let packed: { files: { path: string; name: string }[]; output: string }[] = [];
/** A pass to cancel the job during, the way the dock's cancel lands mid-render. */
let cancelDuring: "master" | "stems" | null = null;
-const runsFor = async (over: Partial<ExportSpec>): Promise<string[][]> => {
+const runsFor = (over: Partial<ExportSpec>): Promise<string[][]> => runsWithCap(over, undefined);
+
+/** runsFor with the pieces one pass opens capped at `cap`. */
+const runsWithCap = async (over: Partial<ExportSpec>, cap: number | undefined): Promise<string[][]> => {
const ffmpegCalls: string[][] = [];
written = [];
mastered = [];
@@ -92,7 +96,7 @@ const runsFor = async (over: Partial<ExportSpec>): Promise<string[][]> => {
progress: 0,
log: [],
};
- await runExport(job, spec, (f) => `/media/${f}`, io);
+ await runExport(job, spec, (f) => `/media/${f}`, io, cap);
return ffmpegCalls;
};
@@ -389,41 +393,34 @@ describe("export filtergraph timebases", () => {
});
});
-describe("one input per cut", () => {
- test("cuts off one file each open their own span, so no branch waits on another", async () => {
- const runs = await runsFor({
- clips: [
- clip("a.mp4", { in: 1, out: 3 }),
- clip("a.mp4", { in: 5, out: 8 }),
- clip("a.mp4", { in: 10, out: 12 }),
- ],
+describe("cuts off one file", () => {
+ test("each stream of the file is fanned out by one split, so ffmpeg prepares each frame once", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { in: 1, out: 3 }), clip("a.mp4", { in: 5, out: 8 }), clip("a.mp4", { in: 10, out: 12 })],
});
- const enc = runs.find((a) => a.includes("-filter_complex"))!;
- const inputs = enc.slice(0, enc.indexOf("-filter_complex"));
- const chain = enc[enc.indexOf("-filter_complex") + 1];
- // The file is opened once for sound and once more per cut.
- const opened = inputs.filter((p, i) => inputs[i - 1] === "-i");
- expect(opened.filter((p) => p.endsWith("a.mp4"))).toHaveLength(4);
- // Each span is cut at the input, preroll included: in 1 -> from 0,
- // in 5 -> from 4, in 10 -> from 9.
- expect(seeks(enc)).toEqual([
- ["0.000", "3.000"],
- ["4.000", "4.000"],
- ["9.000", "3.000"],
+ expect(g.filter((f) => f.includes("[0:v]"))).toEqual(["[0:v]split=3[fan0_v_0][fan0_v_1][fan0_v_2]"]);
+ expect(g.filter((f) => f.includes("[0:a]"))).toEqual(["[0:a]asplit=3[fan0_a_0][fan0_a_1][fan0_a_2]"]);
+ expect(g.filter((f) => f.startsWith("[fan0_v_")).map((f) => f.slice(0, f.indexOf(",")))).toEqual([
+ "[fan0_v_0]trim=0.000:3.000",
+ "[fan0_v_1]trim=4.000:8.000",
+ "[fan0_v_2]trim=9.000:12.000",
]);
- // So the picture never trims a shared decoder.
- expect(chain).not.toContain(":v]trim=");
- // Sound still trims in the graph, off one opening fanned out by one
- // asplit: audio frames are small enough that holding them costs little.
- expect(chain).toContain("[0:a]asplit=3[fan0_a_0][fan0_a_1][fan0_a_2]");
- expect(chain).toContain("[fan0_a_0]atrim=");
});
- test("a still has no source span, so it keeps reading its looped input", async () => {
- const runs = await runsFor({ clips: [{ ...clip("a.png"), image: true }] });
- const enc = runs.find((a) => a.includes("-filter_complex"))!;
- expect(seeks(enc)).toHaveLength(0);
- expect(enc[enc.indexOf("-filter_complex") + 1]).toContain(":v]setpts=PTS-STARTPTS");
+ test("a file read late opens at a whole second ahead of its first read, on the clock it always had", async () => {
+ const enc = (await runsFor({
+ clips: [clip("a.mp4", { in: 40, out: 42 }), clip("a.mp4", { in: 31.5, out: 33 }), clip("b.mp4", { in: 4, out: 6 })],
+ })).find((a) => a.includes("-filter_complex"))!;
+ // a.mp4's first read is 31.5 less the 1 s preroll and the 1 s margin,
+ // 29.5, so it opens at 29 and `-itsoffset` puts the clock back; b.mp4
+ // is read near its start and opens as it is.
+ const seekOf = (file: string) => {
+ const i = enc.indexOf(file);
+ return enc[i - 5] === "-ss" ? enc.slice(i - 5, i - 1) : [];
+ };
+ expect(seekOf("/media/a.mp4")).toEqual(["-ss", "29.000", "-itsoffset", "29.000"]);
+ expect(seekOf("/media/b.mp4")).toEqual([]);
+ expect(arg(enc, "-filter_complex")).toContain("[fan0_v_0]trim=39.000:42.000");
});
});
@@ -445,11 +442,84 @@ describe("joins", () => {
});
});
+describe("passes", () => {
+ const long = (n: number, over: Partial<ExportSpec> = {}): ExportSpec => ({
+ projectId: "p", width: 1080, height: 1920, fps: 30, crf: 24, preset: "veryfast",
+ duration: n * 2,
+ clips: Array.from({ length: n }, (_, i) => clip("a.mp4", { in: i, out: i + 2 })),
+ audio: [],
+ overlays: [],
+ ...over,
+ });
+
+ test("a long delivery is cut into windows that lay end to end on frames", () => {
+ const windows = passWindows(long(120), () => ({ pixels: 1080 * 1920, fps: 30 }));
+ expect(windows.length).toBeGreaterThan(1);
+ expect(windows[0].start).toBe(0);
+ expect(windows.at(-1)!.end).toBe(240);
+ for (let k = 1; k < windows.length; k++) {
+ expect(windows[k].start).toBe(windows[k - 1].end);
+ expect(Math.abs(windows[k].start * 30 - Math.round(windows[k].start * 30))).toBeLessThan(1e-6);
+ }
+ });
+
+ test("a heavier source fills a pass with fewer cuts", () => {
+ const hd = passWindows(long(120), () => ({ pixels: 1080 * 1920, fps: 30 }));
+ const uhd = passWindows(long(120), () => ({ pixels: 3840 * 2160, fps: 30 }));
+ expect(uhd.length).toBeGreaterThan(hd.length);
+ // A short cut stays one pass.
+ expect(passWindows(long(8), () => ({ pixels: 1080 * 1920, fps: 30 }))).toEqual([{ start: 0, end: 16 }]);
+ });
+
+ test("cuts out of file order fill a pass sooner, by the frames the decoder holds for them", () => {
+ // 60 two-second cuts over a minute of one file, in order and played back
+ // to front: back to front, every cut waits while the decoder runs past it.
+ const cuts = (n: number) => Array.from({ length: n }, (_, i) => clip("a.mp4", { in: i * 2, out: i * 2 + 2 }));
+ const inOrder = passWindows(long(60, { clips: cuts(60) }), () => ({ pixels: 1280 * 720, fps: 30 }));
+ const backward = passWindows(long(60, { clips: cuts(60).reverse() }), () => ({ pixels: 1280 * 720, fps: 30 }));
+ expect(backward.length).toBeGreaterThan(inOrder.length);
+ // A faster source holds more frames for the same seconds.
+ const fast = passWindows(long(60, { clips: cuts(60).reverse() }), () => ({ pixels: 1280 * 720, fps: 60 }));
+ expect(fast.length).toBeGreaterThan(backward.length);
+ });
+
+ test("a range is split inside itself", () => {
+ const windows = passWindows(long(120, { range: { start: 31, end: 200 } }), () => ({ pixels: 1080 * 1920, fps: 30 }));
+ expect(windows[0].start).toBe(31);
+ expect(windows.at(-1)!.end).toBe(200);
+ });
+
+ test("each pass writes a piece and the join stream-copies the pictures under the whole mix", async () => {
+ const runs = await runsFor({ clips: Array.from({ length: 6 }, (_, i) => clip("a.mp4", { in: i, out: i + 2 })) });
+ expect(runs.filter((a) => a.includes("-filter_complex"))).toHaveLength(1);
+
+ // Forced into passes of two pieces.
+ const passes = await runsWithCap({ clips: Array.from({ length: 6 }, (_, i) => clip("a.mp4", { in: i, out: i + 2 })) }, 2);
+ const pieces = passes.filter((a) => a.includes("-filter_complex"));
+ expect(pieces.length).toBeGreaterThan(1);
+ for (const [k, a] of pieces.entries()) {
+ // The picture goes to its own file with no sound, the mix out raw
+ // beside it.
+ const video = a.indexOf(`/tmp/graph-test/piece_${k}.mp4`);
+ const mix = a.indexOf(`/tmp/graph-test/piece_${k}.f32`);
+ expect(a.slice(a.indexOf("-filter_complex") + 2, video)).not.toContain("-c:a");
+ expect(a.slice(video, mix)).toContain("pcm_f32le");
+ }
+ const join = passes.at(-1)!;
+ expect(arg(join, "-display_rotation")).toBe("0");
+ expect(arg(join, "-f")).toBe("concat");
+ expect(arg(join, "-c:v")).toBe("copy");
+ expect(join).toContain(`concat:${pieces.map((_, k) => `/tmp/graph-test/piece_${k}.f32`).join("|")}`);
+ expect(join.at(-1)).toBe("/tmp/graph-test/out.mp4");
+ expect(written.find((w) => w.file.endsWith("pieces.ffconcat"))!.data).toContain("file '/tmp/graph-test/piece_0.mp4'");
+ });
+});
+
describe("a speed curve in the filtergraph", () => {
test("a curved clip lays its picture through the map and reads baked sound", async () => {
const curved = clip("a.mp4", { speedCurve: [[0, 1], [4, 4]] });
const g = await graphFor({ clips: [curved, clip("b.mp4")] });
- const video = pictureAt(g, 0);
+ const video = g.find((f) => f.startsWith("[0:v]trim="))!;
expect(video).toContain("setpts='(clip(T-0,0,");
expect(video).not.toContain("(PTS-STARTPTS)/");
// The baked WAV joins the inputs after the two media files, and the
@@ -466,11 +536,11 @@ describe("a speed curve in the filtergraph", () => {
describe("smooth slow motion in the filtergraph", () => {
test("a slowed clip runs motion interpolation over its whole span", async () => {
const g = await graphFor({ clips: [clip("a.mp4", { speed: 0.5, smoothSlow: true }), clip("b.mp4")] });
- const video = pictureAt(g, 0);
+ const video = g.find((f) => f.startsWith("[0:v]trim="))!;
expect(video).toContain("minterpolate=fps=30:mi_mode=mci");
expect(video).not.toContain("split=");
// The plain clip beside it is untouched.
- expect(pictureAt(g, 1)).not.toContain("minterpolate");
+ expect(g.find((f) => f.startsWith("[1:v]trim="))).not.toContain("minterpolate");
});
test("a curve that dips is interpolated in the dip alone", async () => {
@@ -484,7 +554,7 @@ describe("smooth slow motion in the filtergraph", () => {
],
});
const g = await graphFor({ clips: [curved, clip("b.mp4")] });
- const head = pictureAt(g, 0);
+ const head = g.find((f) => f.startsWith("[0:v]trim="))!;
expect(head).toContain("split=3[smic0_0][smic0_1][smic0_2]");
const pieces = g.filter((f) => f.startsWith("[smic0_"));
expect(pieces.length).toBe(3);
@@ -497,17 +567,15 @@ describe("smooth slow motion in the filtergraph", () => {
test("a curve stamps the preroll before 0, so interpolation never sees it", async () => {
// The 1 s preroll ahead of the in point must land before 0 for the
- // interpolation's trim=start=0 to drop it. The input opens at the preroll
- // (source 4 s) and `-ss` restamps it to 0, so the in point sits at T=1 and
- // the preroll runs T=0..1 — the same picture, read on the seeked clock.
+ // interpolation's trim=start=0 to drop it: 4.5 s is preroll, 5 s the in point.
const curved = clip("a.mp4", { in: 5, out: 9, smoothSlow: true, speedCurve: [[0, 0.5], [4, 0.5]] });
- const head = pictureAt(await graphFor({ clips: [curved] }), 0);
+ const head = (await graphFor({ clips: [curved] })).find((f) => f.startsWith("[0:v]trim="))!;
expect(head).toContain("trim=start=0,");
const expr = /setpts='([^']+)'/.exec(head)![1];
const at = (T: number) =>
new Function("T", "TB", "clip", "min", `return ${expr};`)(T, 1, (x: number, lo: number, hi: number) => Math.min(hi, Math.max(lo, x)), Math.min) as number;
- expect(at(0.5)).toBeLessThan(0);
- expect(at(1)).toBeCloseTo(0, 6);
+ expect(at(4.5)).toBeLessThan(0);
+ expect(at(5)).toBeCloseTo(0, 6);
expect(at(5.5)).toBeGreaterThan(0);
});
@@ -1052,11 +1120,8 @@ describe("a reversed clip in the filtergraph", () => {
expect(inputs.some((p) => p.endsWith("turned_clip_0.mov"))).toBe(true);
const idx = inputs.filter((p, i) => inputs[i - 1] === "-i").findIndex((p) => p.endsWith("turned_clip_0.mov"));
const g = graph[graph.indexOf("-filter_complex") + 1].split(";");
- const video = pictureAt(g, 0);
- // The mirrored span is cut at the input now; the re-stamp is unchanged.
- expect(seeks(graph).some(([ss, t]) => ss === "0.000" && t === "7.000")).toBe(true);
- expect(video).toContain("setpts=(PTS-1.000/TB)/2.000");
- expect(video.startsWith("[2:v]setpts=")).toBe(true);
+ const video = g.find((f) => f.startsWith(`[${idx}:v]trim=`))!;
+ expect(video).toContain("trim=0.000:7.000,setpts=(PTS-1.000/TB)/2.000");
expect(video).not.toContain("reverse");
const audio = g.find((f) => f.startsWith(`[${idx}:a]atrim=`))!;
expect(audio).toContain("atrim=1.000:7.000");
@@ -1075,8 +1140,8 @@ describe("a reversed clip in the filtergraph", () => {
reverse: true,
});
const g = await graphFor({ clips: [curved, clip("b.mp4")] });
- const video = pictureAt(g, 0);
- expect(video).toContain("setpts='(clip(T-0,0,");
+ const video = g.find((f) => f.startsWith("[1:v]trim="))!;
+ expect(video).toContain("trim=0.000:4.000,setpts='(clip(T-0,0,");
const first = g.find((f) => f.includes("apad=whole_dur="))!;
expect(first).toContain(`apad=whole_dur=${retimeOf(curved).len.toFixed(3)}`);
});
@@ -1093,16 +1158,6 @@ const encodeRun = async (over: Partial<ExportSpec>) => {
};
const arg = (args: string[], flag: string) => args[args.indexOf(flag) + 1];
-/** A clip's picture chain, in clip order — footage now reads a seeked input
- * (`setpts=`) where it used to trim a shared one (`trim=`), and a still has
- * always read its looped input straight. */
-const pictureAt = (g: string[], nth: number) =>
- g.filter((f) => /^\[\d+:v\](trim=|setpts=)/.test(f))[nth]!;
-
-/** The `-ss`/`-t` pairs ffmpeg is handed, in input order. */
-const seeks = (args: string[]) =>
- args.flatMap((v, i) => (v === "-ss" ? [[args[i + 1], args[i + 3]] as const] : []));
-
describe("delivery", () => {
test("source bitrates reach the worker and Mac encoders", async () => {
const args = await encodeRun({ bitrate: 140_000, audioBitrate: 48_000 });
@@ -1232,11 +1287,27 @@ describe("delivery", () => {
expect(narrowSpecToRange(spec).clips.map((c) => !!c.hidden)).toEqual([false, false, true]);
});
+ test("a slot the range hides opens none of its inputs", async () => {
+ const runs = await runsFor({
+ duration: 15,
+ range: { start: 11, end: 14 },
+ clips: [
+ clip("a.mp4", { out: 5, removal: { rgb: "rm_rgb.mov", alpha: "rm_a.mov" } }),
+ clip("still.png", { out: 5, image: true }),
+ clip("c.mp4", { out: 5 }),
+ ],
+ });
+ const enc = runs.find((a) => a.includes("-filter_complex"))!;
+ const opened = enc.filter((p, i) => enc[i - 1] === "-i");
+ // The hidden removal clip, still and footage leave the graph whole: no
+ // keyed pair whose chain nothing reads, no looped still, no file.
+ expect(opened.map((p) => path.basename(p))).toEqual(["c.mp4"]);
+ });
+
test("no range delivers the whole cut", async () => {
const args = await encodeRun({ duration: 20 });
expect(arg(args, "-filter_complex")).not.toContain("[vrange]");
- // Inputs carry their own `-t` now, so read the one on the output side.
- expect(args[args.lastIndexOf("-t") + 1]).toBe("20.000");
+ expect(arg(args, "-t")).toBe("20.000");
});
});
@@ -1277,7 +1348,7 @@ describe("clip color in the filtergraph", () => {
expect(cubes).toHaveLength(1);
expect(cubes[0].file).toBe(path.join("/tmp/graph-test", "clip_0.cube"));
expect(cubes[0].data).toContain("LUT_3D_SIZE 33");
- const line = pictureAt(g, 0);
+ const line = g.find((l) => l.startsWith("[0:v]"))!;
// The framing scale spells the file's matrix and range; the picture goes
// to 16-bit RGB, through the LUT, and comes back as BT.709 video before
// the letterbox pad.
@@ -1303,7 +1374,7 @@ describe("clip color in the filtergraph", () => {
lutSizeWide: 65,
clips: [clip("log.mov", { color: { profile: "apple-log", matrix: "bt2020nc", fullRange: false } })],
});
- const line = pictureAt(g, 0);
+ const line = g.find((l) => l.startsWith("[0:v]"))!;
expect(line).toMatch(/scale=[^,]*in_color_matrix=bt2020:in_range=tv/);
expect(line).toContain("format=gbrp16le,lut3d=file=");
expect(written.find((w) => w.file.endsWith(".cube"))!.data).toContain("LUT_3D_SIZE 65");
@@ -1313,7 +1384,7 @@ describe("clip color in the filtergraph", () => {
const g = await graphFor({
clips: [clip("sd.mp4", { color: { profile: "rec709", matrix: "bt601", fullRange: true } })],
});
- const line = pictureAt(g, 0);
+ const line = g.find((l) => l.startsWith("[0:v]"))!;
expect(line).not.toContain("lut3d");
expect(line).toContain("scale:in_color_matrix=bt601:in_range=pc:out_color_matrix=bt709:out_range=tv");
expect(line.indexOf("out_range=tv")).toBeLessThan(line.indexOf("pad=1080:1920"));
@@ -1324,7 +1395,7 @@ describe("clip color in the filtergraph", () => {
clips: [clip("a.mp4", { grade: { exposure: 5, sharpen: 25, clarity: 30 } })],
});
const joined = g.join(";");
- const head = pictureAt(g, 0);
+ const head = g.find((l) => l.startsWith("[0:v]"))!;
expect(head).toContain("lut3d=file=");
expect(head).toMatch(/scale=out_color_matrix=bt709:out_range=pc,format=yuv444p16le\[dtic0\]$/);
// The detail pass: bases from the input luma, gained detail merged back.
@@ -1402,7 +1473,7 @@ describe("clip color in the filtergraph", () => {
const g = await graphFor({
clips: [clip("a.mp4", { speed: 0.5, smoothSlow: true, grade: { exposure: 5 } })],
});
- const line = pictureAt(g, 0);
+ const line = g.find((l) => l.startsWith("[0:v]"))!;
expect(line).toContain("minterpolate=");
expect(line.indexOf("minterpolate=")).toBeLessThan(line.indexOf("scale:in_color_matrix=bt709:in_range=tv,format=gbrp16le,lut3d"));
});
@@ -1445,7 +1516,7 @@ describe("an HDR delivery", () => {
test("composites at ten bits in HLG: every clip maps into Rec.2020 and comes back as 10-bit video", async () => {
const g = await graphFor(hdrSpec());
- const line = pictureAt(g, 0);
+ const line = g.find((l) => l.startsWith("[0:v]"))!;
// An ungraded Rec.709 clip is a conversion here, so it takes the LUT.
expect(line).toMatch(/scale=[^,]*in_color_matrix=bt709:in_range=tv/);
expect(line).toContain("format=gbrp16le,lut3d=file=");
@@ -1462,9 +1533,9 @@ describe("an HDR delivery", () => {
test("an HLG file ungraded runs no LUT, and a graded one grades in place", async () => {
const hlg = { profile: "hlg" as const, matrix: "bt2020nc" as const, fullRange: false };
const plain = await graphFor(hdrSpec({ clips: [clip("h.mp4", { color: hlg })] }));
- expect(pictureAt(plain, 0)).not.toContain("lut3d");
+ expect(plain.find((l) => l.startsWith("[0:v]"))).not.toContain("lut3d");
const graded = await graphFor(hdrSpec({ clips: [clip("h.mp4", { color: hlg, grade: { exposure: 10 } })] }));
- const line = pictureAt(graded, 0);
+ const line = graded.find((l) => l.startsWith("[0:v]"))!;
expect(line).toMatch(/scale=[^,]*in_color_matrix=bt2020:in_range=tv/);
expect(line).toContain("lut3d=file=");
expect(line).toContain("scale=out_color_matrix=bt2020:out_range=tv");
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index 1972f1f5..7c71f052 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -627,22 +627,7 @@ const SOURCE_PREROLL = 1;
* screen at the in point comes along (stamped before 0) and the re-stamp to
* the output rate shows it on the clip's first frame. */
const sourceSpan = (idx: number, c: { in: number; out: number }, rt: Retime) =>
- `[${idx}:v]trim=${num(spanStart(c))}:${num(c.out)},setpts=${retimedPts(rt, c.in)}`;
-
-/** Where a footage clip's span starts in its source, preroll included. */
-const spanStart = (c: { in: number }) => Math.max(0, c.in - SOURCE_PREROLL);
-
-/**
- * The same span read off an input already seeked to it, so the picture never
- * trims inside the graph.
- *
- * `-ss` before `-i` restamps the seeked input to start at 0, so the in point
- * sits at `c.in - spanStart(c)` — the preroll it kept — and the re-stamp
- * measures from there. Everything downstream sees the frames and timestamps
- * the trimmed chain produced.
- */
-const seekedSpan = (idx: number, c: { in: number }, rt: Retime) =>
- `[${idx}:v]setpts=${retimedPts(rt, c.in - spanStart(c))}`;
+ `[${idx}:v]trim=${num(Math.max(0, c.in - SOURCE_PREROLL))}:${num(c.out)},setpts=${retimedPts(rt, c.in)}`;
/** A curve clamps everything before its first knot to 0; the preroll keeps
* its own seconds before 0, as one rate does, so a trim at 0 drops it. */
@@ -762,6 +747,149 @@ const spanLen = (c: { in: number; out: number; speed?: number; speedCurve?: Spee
* still shapes. */
const RANGE_SLACK_S = 0.5;
+/** Source seconds a file's input opens ahead of the earliest read the graph
+ * makes of it, and the earliest it seeks to at all: a file read from near
+ * its start opens as it is. The seek lands on a whole second, a point on
+ * every sample and frame grid. */
+const READ_MARGIN_S = 1;
+const READ_SEEK_MIN_S = 10;
+
+/** How far past a range's edges a clip still shapes the frames inside it. */
+const rangeReach = (spec: ExportSpec) =>
+ Math.max(
+ 0,
+ ...spec.clips.map((c) => Math.max(c.transition ?? 0, c.soundCross ?? 0, c.soundAhead ?? 0, c.soundBack ?? 0))
+ ) + RANGE_SLACK_S;
+
+/**
+ * What one encode pass may hold for its pieces — track-0 clips, layered clips
+ * and elements. Every piece's input is primed when the graph opens and its
+ * chain keeps frames through the encode, about PIECE_FRAMES of its source's
+ * size or the delivery's, whichever is larger; cuts played out of file order
+ * add the frames their decoder holds until their turn. One pass over a long
+ * timeline grows with it, and passes held to this budget stay flat; a larger
+ * budget means fewer passes, each decoding the stretch of file its cuts span.
+ * PASS_PIECES caps the count whatever the size: each piece brings a decoder
+ * and its threads.
+ */
+const PASS_HOLD_BYTES = 2 * 1024 * 1024 * 1024;
+const PIECE_FRAMES = 16;
+const PASS_PIECES = 48;
+
+/** A source's decoded picture: its size in pixels and its frame rate. */
+export type SourcePicture = { pixels: number; fps: number };
+
+/** A piece of one window: its timeline span and, for footage, the file and
+ * the source seconds its picture reads. */
+type PassSpan = { start: number; end: number; file: string; lo?: number; hi?: number };
+
+/**
+ * Seconds of source a file's one decoder holds at most for the window's cuts
+ * still to come. The decoder reads each file forward once and the join takes
+ * cuts in timeline order, so a cut that sits later in the timeline than in the
+ * file waits as frames until its turn. Cuts read in file order hold nothing:
+ * 0–2, 5–7, 9–11 hold 0 s; 9–11, 0–2, 5–7 hold the 4 s of 0–2 and 5–7 the
+ * decoder passes on its way to 11.
+ */
+function heldSeconds(cuts: PassSpan[]): number {
+ const order = [...cuts].sort((x, y) => x.start - y.start);
+ let reached = -Infinity;
+ let most = 0;
+ for (const [i, cut] of order.entries()) {
+ reached = Math.max(reached, cut.hi!);
+ const held = order.slice(i + 1).reduce((sum, c) => sum + Math.max(0, Math.min(c.hi!, reached) - c.lo!), 0);
+ most = Math.max(most, held);
+ }
+ return most;
+}
+
+/**
+ * The delivery cut into the windows it renders in, one pass each: every
+ * window touches at most `cap` pieces, and their primed frames plus the frames
+ * its decoders hold for cuts out of file order stay within PASS_HOLD_BYTES
+ * (with the reach a range keeps); each window ends on a frame at a piece's
+ * start. `sourceOf` is a file's decoded picture. A delivery under the budget
+ * is one window. Each pass is a range export, so its frames and samples are
+ * the ones the whole composite has there, and the windows laid end to end
+ * are the delivery.
+ */
+export function passWindows(
+ spec: ExportSpec,
+ sourceOf: (file: string) => SourcePicture,
+ cap: number = PASS_PIECES
+): ExportRange[] {
+ const from = Math.max(0, spec.range?.start ?? 0);
+ const to = Math.min(spec.duration, spec.range?.end ?? spec.duration);
+ const reach = rangeReach(spec);
+ const frameBytes = (file: string) => 1.5 * Math.max(sourceOf(file).pixels, spec.width * spec.height);
+
+ // Every piece the graph builds a chain for, as a timeline span, with the
+ // source a footage piece's picture reads.
+ const spans: PassSpan[] = [];
+ const footage = (c: Parameters<typeof retimeOf>[0] & { image?: boolean }): Pick<PassSpan, "lo" | "hi"> => {
+ if (c.image) {
+ return {};
+ }
+ const rt = retimeOf(c);
+ const { lo, hi } = srcSpan(rt, 0, rt.len);
+ return { lo: Math.max(0, lo - SOURCE_PREROLL), hi };
+ };
+ let cursor = 0;
+ for (const c of spec.clips) {
+ spans.push({ start: cursor, end: cursor + spanLen(c), file: c.file, ...footage(c) });
+ cursor += spanLen(c);
+ }
+ for (const o of spec.overlayVideos ?? []) {
+ spans.push({ start: o.start, end: o.start + spanLen(o), file: o.file, ...footage(o) });
+ }
+ for (const o of spec.overlays) {
+ spans.push({ start: o.start, end: o.end, file: "" });
+ }
+
+ // A window fits when its pieces' primed frames and its decoders' waiting
+ // ones come in under the budget.
+ const fits = (a: number, b: number) => {
+ const held = spans.filter((s) => s.end >= a - reach && s.start <= b + reach);
+ if (held.length > cap) {
+ return false;
+ }
+ let bytes = held.reduce((sum, s) => sum + PIECE_FRAMES * frameBytes(s.file), 0);
+ for (const file of new Set(held.filter((s) => s.lo !== undefined).map((s) => s.file))) {
+ const cuts = held.filter((s) => s.file === file && s.lo !== undefined);
+ bytes += heldSeconds(cuts) * sourceOf(file).fps * frameBytes(file);
+ }
+ return bytes <= PASS_HOLD_BYTES;
+ };
+
+ // Window edges sit on whole frames at piece starts, so each pass's trim
+ // takes exactly the frames between two edges.
+ const edges = [...new Set(spans.map((s) => gridFrame(s.start, spec.fps) / spec.fps))]
+ .filter((t) => t > from && t < to)
+ .sort((a, b) => a - b);
+
+ // Each window runs to the furthest edge that keeps it within the budget; a
+ // window whose first edge is already over still ends there.
+ const windows: ExportRange[] = [];
+ let start = from;
+ while (!fits(start, to)) {
+ const next = edges.filter((t) => t > start);
+ if (next.length === 0) {
+ break;
+ }
+ let end = next[0];
+ for (const t of next) {
+ if (!fits(start, t)) {
+ break;
+ }
+ end = t;
+ }
+ windows.push({ start, end });
+ start = end;
+ }
+ windows.push({ start, end: to });
+ return windows;
+}
+
/**
* The spec with everything that cannot reach the range taken out of the
* graph's way: a track-0 clip whose slot ends before the range or starts
@@ -774,13 +902,7 @@ const RANGE_SLACK_S = 0.5;
*/
export function narrowSpecToRange(spec: ExportSpec): ExportSpec {
if (!spec.range) return spec;
- const reach =
- Math.max(
- 0,
- ...spec.clips.map((c) =>
- Math.max(c.transition ?? 0, c.soundCross ?? 0, c.soundAhead ?? 0, c.soundBack ?? 0)
- )
- ) + RANGE_SLACK_S;
+ const reach = rangeReach(spec);
const from = Math.max(0, spec.range.start) - reach;
const to = Math.min(spec.duration, spec.range.end) + reach;
const inside = (start: number, end: number) => end >= from && start <= to;
@@ -912,12 +1034,14 @@ const realIO: ExportPipelineIO = {
/** Render `spec` into `job.outPath`. `mediaPathFor` maps a spec media file
* name to its staged path on disk (the engine reads the project folder; the
* worker reads its download dir); overlay/caption PNGs are read from
- * `job.tmpDir` by base name in both. */
+ * `job.tmpDir` by base name in both. `passPieces` caps the pieces one encode
+ * pass opens (see passWindows). */
export async function runExport(
job: RenderHandle,
given: ExportSpec,
mediaPathFor: (file: string) => string,
- io: ExportPipelineIO = realIO
+ io: ExportPipelineIO = realIO,
+ passPieces: number = PASS_PIECES
) {
if (given.clips.length === 0) throw new Error("Nothing to export.");
const codec = given.codec ?? "h264";
@@ -935,6 +1059,47 @@ export async function runExport(
}
if (await io.exportSourceFiles(job, mediaPathFor, given.sourceSegments, codec, given.audioBitrate)) return;
}
+
+ // A long delivery renders in passes held to a memory budget, joined into
+ // the one file at the end. A piece's cost follows its source's size and
+ // frame rate.
+ const pictures = new Map<string, SourcePicture>();
+ const footage = [...given.clips, ...(given.overlayVideos ?? [])].filter((c) => c.file && !c.image && !c.removal);
+ await Promise.all(
+ [...new Set(footage.map((c) => c.file))].map(async (file) => {
+ const at = await resolveMedia(io.stat, mediaPathFor, file);
+ const [dims, perSecond] = await Promise.all([io.videoDimensions(at), io.videoDecodeCost(at)]);
+ const pixels = dims ? dims.width * dims.height : 0;
+ pictures.set(file, { pixels, fps: pixels > 0 && perSecond ? perSecond / (1.5 * pixels) : given.fps });
+ })
+ );
+ const windows = passWindows(given, (file) => pictures.get(file) ?? { pixels: 0, fps: given.fps }, passPieces);
+ if (windows.length === 1) {
+ await renderPass(job, given, mediaPathFor, io, { kind: "delivery" });
+ return;
+ }
+ await renderInPasses(job, given, windows, mediaPathFor, io);
+}
+
+/**
+ * Where one pass's render lands: the delivery itself, or a piece of a
+ * delivery rendered in passes — its picture encoded for the delivery, its mix
+ * and stems as sound to lay end to end — with the share of the job's
+ * progress the pass covers.
+ */
+type PassOutput =
+ | { kind: "delivery" }
+ | { kind: "piece"; video: string; mix: string; stems: string[]; progress: [number, number] };
+
+/** One encode pass over `given`, composited whole and cut to its range. */
+async function renderPass(
+ job: RenderHandle,
+ given: ExportSpec,
+ mediaPathFor: (file: string) => string,
+ io: ExportPipelineIO,
+ out: PassOutput
+) {
+ const codec = given.codec ?? "h264";
let spec = await withSpecColors(narrowSpecToRange(given), mediaPathFor);
const { width: W, height: H, fps } = spec;
@@ -1166,11 +1331,12 @@ export async function runExport(
// plus one per uploaded overlay PNG.
// Still images are excluded here: a plain `-i file` decodes one frame, so
// each image clip/overlay gets its own looped input below instead. Gap
- // spacers (empty file) reference no media at all — they render as black.
+ // spacers (empty file) reference no media at all — they render as black,
+ // and so do hidden clips outside a range, whose files stay closed.
const mediaFiles = [
...new Set(
[
- ...spec.clips.filter((c) => !c.image),
+ ...spec.clips.filter((c) => !c.image && !c.hidden),
...spec.audio,
...overlayVideos.filter((o) => !o.image),
]
@@ -1196,12 +1362,21 @@ export async function runExport(
// Resolve paths in order first so ffmpeg input indices stay deterministic,
// then probe every file's streams concurrently.
const paths = await Promise.all(mediaFiles.map((f) => resolveMedia(io.stat, mediaPathFor, f)));
- // file → the path ffmpeg opens, so a cut can open its own span below.
- const pathOf = new Map<string, string>();
+ // Each file opens at the earliest source second the graph reads of it, so
+ // a pass deep into a long recording never decodes the file from its start.
+ // Re-stamped by the same amount, the input carries the timestamps an
+ // unseeked one does, and every trim below reads it unchanged.
+ const readFrom = new Map<string, number>();
+ const readers = [...spec.clips.filter((c) => !c.hidden && !c.image), ...overlayVideos.filter((o) => !o.image), ...spec.audio];
+ for (const c of readers) {
+ const back = "soundBack" in c ? (c.soundBack ?? 0) : 0;
+ const lo = srcSpan(retimeOf(c), -back, 0).lo - SOURCE_PREROLL - READ_MARGIN_S;
+ readFrom.set(c.file, Math.min(readFrom.get(c.file) ?? Infinity, lo));
+ }
mediaFiles.forEach((f, i) => {
inputIndex.set(f, nInputs++);
- pathOf.set(f, paths[i]);
- inputs.push("-i", paths[i]);
+ const at = Math.floor(readFrom.get(f) ?? 0);
+ inputs.push(...(at >= READ_SEEK_MIN_S ? ["-ss", num(at), "-itsoffset", num(at)] : []), "-i", paths[i]);
});
await Promise.all(
mediaFiles.map(async (f, i) => {
@@ -1235,7 +1410,6 @@ export async function runExport(
// The turned copies: already probed at the bake.
for (const [file, streams] of turned) {
inputIndex.set(file, nInputs++);
- pathOf.set(file, file);
inputs.push("-i", file);
audioPresence.set(file, streams.audio);
videoPresence.set(file, streams.video);
@@ -1294,7 +1468,9 @@ export async function runExport(
const imageClipInput = new Map<number, number>();
for (let j = 0; j < spec.clips.length; j++) {
const c = spec.clips[j];
- if (!c.image || !c.file) continue;
+ if (!c.image || !c.file || c.hidden) {
+ continue;
+ }
const dur = spanLen(c);
imageClipInput.set(j, nInputs++);
inputs.push(
@@ -1411,7 +1587,9 @@ export async function runExport(
const clipRemovalInput = new Map<number, { rgb: number; a: number }>();
for (let j = 0; j < spec.clips.length; j++) {
const c = spec.clips[j];
- if (c.removal) clipRemovalInput.set(j, removalInputs(c.removal));
+ if (c.removal && !c.hidden) {
+ clipRemovalInput.set(j, removalInputs(c.removal));
+ }
}
const overlayRemovalInput = new Map<(typeof overlayVideos)[number], { rgb: number; a: number }>();
for (const oc of overlayVideos) {
@@ -1762,40 +1940,6 @@ export async function runExport(
inputs.push("-i", file);
baked.set(c, { ...bake, idx });
}
-
- /**
- * One input per footage cut, seeked to that cut's span, instead of every cut
- * trimming one shared decoder.
- *
- * `[0:v]trim=a:b` written N times makes ffmpeg split one decoder N ways, and
- * `concat` reads those branches in order — so every branch past the one being
- * read holds the frames already handed to it. What it holds grows with how
- * long the encode runs, which is why a longer timeline dies where a short one
- * survives, and why a slower preset dies earlier at the same length. Measured
- * on one 30-cut 104s render, 1080x1920, veryfast, no subtitle lane:
- *
- * one input, 30 trim branches 8,320MB
- * an input per cut 3,311MB
- *
- * Seeking at the input also skips decoding the head of each span, so a cut
- * late in a long file no longer walks the file to reach its first frame.
- *
- * Left trimming in the graph: stills (no source span), removal clips (their
- * picture is the keyed pair, not this file), and clips whose source is read
- * twice over (a sound twin), where a second opening buys nothing.
- */
- const cutVideoInput = new Map<number, number>();
- spec.clips.forEach((c, j) => {
- if (c.image || clipRemovalInput.get(j) || c.hidden) return;
- if (!c.file || !videoPresence.get(c.file)) return;
- const src = pathOf.get(c.file);
- if (!src) return;
- const from = spanStart(c);
- const span = Math.max(1 / fps, c.out - from);
- cutVideoInput.set(j, nInputs++);
- inputs.push("-ss", num(from), "-t", num(span), "-i", src);
- });
-
/**
* The head of an audio chain reading a clip's sound over the timeline-local
* window `[fromT, toT]` (seconds from the clip's head; negative reaches
@@ -2339,12 +2483,9 @@ export async function runExport(
alphaSet(`rvc${j}`, `rva${j}`, `rvs${j}`, `rv${j}`, ",tpad=stop_mode=clone:stop_duration=1");
timebase = `[rvs${j}]null`;
} else {
- const cutIdx = cutVideoInput.get(j);
timebase = c.image
? `[${idx}:v]setpts=PTS-STARTPTS`
- : cutIdx === undefined
- ? sourceSpan(idx, c, rt)
- : seekedSpan(cutIdx, c, rt);
+ : sourceSpan(idx, c, rt);
}
if ((c.image || rmIn || videoPresence.get(c.file)) && !c.hidden) {
const region = regionPx(c.frame, W, H);
@@ -3262,7 +3403,12 @@ export async function runExport(
const to = num(Math.min(spec.duration, spec.range.end));
filters.push(`[${vLabel}]trim=start=${from}:end=${to},setpts=PTS-STARTPTS[vrange]`);
vLabel = "vrange";
- filters.push(`[${aLabel}]atrim=start=${from}:end=${to},asetpts=PTS-STARTPTS[arange]`);
+ // The sound takes the delivery's rate before the cut, so a range starts
+ // on the whole delivery's sample grid: ranges laid end to end are the
+ // samples one pass would write.
+ filters.push(
+ `[${aLabel}]aresample=${spec.audioSampleRate ?? MIX_RATE},atrim=start=${from}:end=${to},asetpts=PTS-STARTPTS[arange]`
+ );
aLabel = "arange";
}
@@ -3280,13 +3426,16 @@ export async function runExport(
}
label = treatAudio(label, `s${i}_`);
const window = spec.range
- ? `,atrim=start=${num(Math.max(0, spec.range.start))}:end=${num(Math.min(spec.duration, spec.range.end))},asetpts=PTS-STARTPTS`
+ ? `,aresample=${STEM_RATE},atrim=start=${num(Math.max(0, spec.range.start))}:end=${num(Math.min(spec.duration, spec.range.end))},asetpts=PTS-STARTPTS`
: "";
filters.push(`[${label}]apad=whole_dur=${num(spec.duration)},atrim=0:${num(spec.duration)}${window}[stem${i}]`);
return `stem${i}`;
});
if (stemPlan.length > 0 && !job.stemsPath) throw new Error("This export has nowhere to put its stems.");
- const stemFiles = stemPlan.map((s, i) => ({ path: path.join(job.tmpDir, `stem_${i}.wav`), name: s.file }));
+ const stemFiles = stemPlan.map((s, i) => ({
+ path: out.kind === "piece" ? out.stems[i] : path.join(job.tmpDir, `stem_${i}.wav`),
+ name: s.file,
+ }));
const stemOutputs = stemLabels.flatMap((label, i) => [
"-map", `[${label}]`,
"-c:a", "pcm_s24le",
@@ -3301,9 +3450,12 @@ export async function runExport(
// The raw mix comes out in the delivery's own rate and layout, so the
// master weighs the same channels the tab's master does: a mono delivery
// is measured as mono, a 5.1 one with its surround weights.
+ // A piece's mix always comes out raw: the pieces' samples laid end to end
+ // are the delivery's mix, mastered or encoded once at the join.
const master = spec.loudness !== undefined;
if (master && spec.truePeakCeiling === undefined) throw new Error("A mastered export needs its true-peak ceiling.");
- const mixPath = path.join(job.tmpDir, "mix.f32");
+ const rawMix = master || out.kind === "piece";
+ const mixPath = out.kind === "piece" ? out.mix : path.join(job.tmpDir, "mix.f32");
const mixRate = spec.audioSampleRate ?? MIX_RATE;
const mixChannels = spec.audioChannels ?? 2;
@@ -3312,7 +3464,8 @@ export async function runExport(
// Encode into the tmp dir, then re-emit the container to strip a stray output
// rotation flag (see the strip pass below). Keeping the encode intermediate
// lets the second pass own faststart.
- const encodePath = path.join(job.tmpDir, `encode${containerExtension(spec)}`);
+ const encodePath = out.kind === "piece" ? out.video : path.join(job.tmpDir, `encode${containerExtension(spec)}`);
+ const [shareFrom, shareTo] = out.kind === "piece" ? out.progress : [0, 1];
await io.runFfmpeg(
job,
[
@@ -3320,20 +3473,26 @@ export async function runExport(
...inputs,
"-filter_complex", assertGraphSafe(fanOutInputs(filters).join(";")),
"-map", `[${vLabel}]`,
- ...(master ? [] : ["-map", `[${aLabel}]`]),
+ ...(rawMix ? [] : ["-map", `[${aLabel}]`]),
...videoCodecArgs(enc, spec),
...colorTagArgs(spec),
- ...(master ? [] : audioCodecArgs(spec)),
+ ...(rawMix ? [] : audioCodecArgs(spec)),
"-t", num(span),
encodePath,
- ...(master
+ ...(rawMix
? ["-map", `[${aLabel}]`, "-c:a", "pcm_f32le", "-ar", String(mixRate), "-ac", String(mixChannels), "-f", "f32le", "-t", num(span), mixPath]
: []),
...stemOutputs,
],
- (t) => (job.progress = Math.min(0.99, t / Math.max(0.1, span)))
+ (t) => (job.progress = Math.min(0.99, shareFrom + (shareTo - shareFrom) * (t / Math.max(0.1, span))))
);
+ // A piece is done once its files are written; the join masters, packs and
+ // muxes the whole.
+ if (out.kind === "piece") {
+ return;
+ }
+
// The master and the stems pack run in this process, out of a cancel's
// reach, so the job is checked after each: a canceled render never goes on
// to write its file.
@@ -3377,3 +3536,106 @@ export async function runExport(
job.progress = 1;
}
+
+/** An ffconcat list of pieces laid end to end, each held for its window's
+ * length so the next starts on the frame the last one stopped short of. */
+function pieceList(pieces: { file: string; seconds: number }[]): string {
+ const lines = ["ffconcat version 1.0"];
+ for (const p of pieces) {
+ lines.push(`file '${p.file}'`, `duration ${edge(p.seconds)}`);
+ }
+ return lines.join("\n") + "\n";
+}
+
+/**
+ * Render `given` window by window and join the pieces into the delivery.
+ *
+ * Each pass composites one window — a range export — so it holds what that
+ * window's pieces need and lets go of it when it ends. The pictures were
+ * encoded for the delivery and join without a re-encode; the raw mixes and
+ * stems lay end to end sample for sample, then are mastered, packed and
+ * encoded once over the whole, as a single pass would.
+ */
+async function renderInPasses(
+ job: RenderHandle,
+ given: ExportSpec,
+ windows: ExportRange[],
+ mediaPathFor: (file: string) => string,
+ io: ExportPipelineIO
+) {
+ const span = deliverySpan(given.range, given.duration);
+ const ext = containerExtension(given);
+ const stemPlan = given.stemPlan ?? [];
+ const pieces = windows.map((w, k) => ({
+ window: w,
+ video: path.join(job.tmpDir, `piece_${k}${ext}`),
+ mix: path.join(job.tmpDir, `piece_${k}.f32`),
+ stems: stemPlan.map((_, i) => path.join(job.tmpDir, `piece_${k}_stem_${i}.wav`)),
+ }));
+
+ // The passes, in order; each takes its window's share of the progress.
+ let done = 0;
+ for (const p of pieces) {
+ const seconds = p.window.end - p.window.start;
+ const progress: [number, number] = [done / span, (done + seconds) / span];
+ await renderPass(job, { ...given, range: p.window }, mediaPathFor, io, { kind: "piece", ...p, progress });
+ stopIfCanceled(job);
+ done += seconds;
+ }
+
+ // The mix: the pieces' raw samples read as one stream, mastered when the
+ // delivery asks for it.
+ const mixRate = given.audioSampleRate ?? MIX_RATE;
+ const mixChannels = given.audioChannels ?? 2;
+ const raw = ["-f", "f32le", "-ar", String(mixRate), "-ac", String(mixChannels)];
+ let mix = `concat:${pieces.map((p) => p.mix).join("|")}`;
+ if (given.loudness !== undefined) {
+ if (given.truePeakCeiling === undefined) {
+ throw new Error("A mastered export needs its true-peak ceiling.");
+ }
+ const mixPath = path.join(job.tmpDir, "mix.f32");
+ const masteredPath = path.join(job.tmpDir, "master.f32");
+ await io.runFfmpeg(job, ["-y", ...raw, "-i", mix, "-c", "copy", "-f", "f32le", mixPath]);
+ await io.masterRawMix(mixPath, masteredPath, {
+ sampleRate: mixRate,
+ channels: mixChannels,
+ targetLufs: given.loudness,
+ ceilingDbtp: given.truePeakCeiling,
+ });
+ stopIfCanceled(job);
+ mix = masteredPath;
+ }
+
+ // Each stem: its pieces joined into one file, then the pack.
+ if (stemPlan.length > 0) {
+ if (!job.stemsPath) {
+ throw new Error("This export has nowhere to put its stems.");
+ }
+ const stemFiles = stemPlan.map((s, i) => ({ path: path.join(job.tmpDir, `stem_${i}.wav`), name: s.file }));
+ for (const [i, stem] of stemFiles.entries()) {
+ const list = path.join(job.tmpDir, `stem_${i}.ffconcat`);
+ await io.writeFile(list, pieceList(pieces.map((p) => ({ file: p.stems[i], seconds: p.window.end - p.window.start }))));
+ await io.runFfmpeg(job, ["-y", "-f", "concat", "-safe", "0", "-i", list, "-c", "copy", stem.path]);
+ }
+ await io.packStems(stemFiles, job.stemsPath);
+ stopIfCanceled(job);
+ }
+
+ // The delivery: the pictures joined as encoded, the stray rotation flag
+ // stripped as a single pass strips it, and the mix encoded under them.
+ const list = path.join(job.tmpDir, "pieces.ffconcat");
+ await io.writeFile(list, pieceList(pieces.map((p) => ({ file: p.video, seconds: p.window.end - p.window.start }))));
+ await io.runFfmpeg(job, [
+ "-y",
+ "-display_rotation", "0",
+ "-f", "concat", "-safe", "0", "-i", list,
+ ...raw, "-i", mix,
+ "-map", "0:v", "-map", "1:a",
+ "-c:v", "copy", ...audioCodecArgs(given),
+ "-t", num(span),
+ "-movflags", "+faststart",
+ job.outPath,
+ ]);
+
+ job.progress = 1;
+}