Open source CapCut alternative.
TypeScript
fix(site): render long exports in bounded passes [rebuild]
David Leaf3f676ef214
diff --git a/site/src/cut/server/exportPipeline.e2e.test.ts b/site/src/cut/server/exportPipeline.e2e.test.tsindex 1ba56d64..a2a3ea56 100644--- a/site/src/cut/server/exportPipeline.e2e.test.ts+++ b/site/src/cut/server/exportPipeline.e2e.test.ts@@ -1,9 +1,9 @@import { describe, expect, test } from "bun:test";import { spawnSync } from "node:child_process";-import { mkdtemp, rm, stat } from "node:fs/promises";+import { cp, mkdir, mkdtemp, rm, stat } from "node:fs/promises";import os from "node:os";import path from "node:path";-import { runExport, type ExportSpec, type RenderHandle } from "./exportPipeline";+import { passWindows, runExport, type ExportSpec, type RenderHandle } from "./exportPipeline";// The graph tests assert shapes; this one hands the real graph to a real// ffmpeg. It synthesizes every asset kind from lavfi sources, builds one@@ -93,6 +93,132 @@ const probe = (file: string) => {const W = 320;const H = 568;+/** Synthesizes every asset kind from lavfi sources into `mediaDir` (project+ * media) and `tmpDir` (client-painted stills), and returns one project that+ * uses every ExportSpec feature over them. */+async function everyFeature(mediaDir: string, tmpDir: string): Promise<ExportSpec> {+ // ---- Footage, stills, and the audio bed (project media) ----------+ const vid = (name: string, src: string, hz: number, secs: number) =>+ ff([+ "-f", "lavfi", "-i", `${src}=size=320x240:rate=24:duration=${secs}`,+ "-f", "lavfi", "-i", `sine=frequency=${hz}:sample_rate=44100:duration=${secs}`,+ "-shortest", "-pix_fmt", "yuv420p", "-c:a", "aac",+ path.join(mediaDir, name),+ ]);+ vid("a.mp4", "testsrc2", 440, 3);+ vid("b.mp4", "smptebars", 660, 3);+ vid("o.mp4", "testsrc", 880, 3);+ ff([+ "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=1:duration=1",+ "-frames:v", "1", path.join(mediaDir, "img.png"),+ ]);+ ff([+ "-f", "lavfi", "-i", "sine=frequency=220:sample_rate=44100:duration=6",+ "-c:a", "aac", path.join(mediaDir, "music.m4a"),+ ]);++ // ---- Client-painted stills and the person matte (job.tmpDir) -----+ const png = (name: string, chain: string) =>+ ff(["-f", "lavfi", "-i", chain, "-frames:v", "1", path.join(tmpDir, name)]);+ png("mask.png", `color=c=white:s=${W}x${H},format=gray`);+ png("omask.png", "color=c=0xC0C0C0:s=160x160,format=gray");+ png("el.png", `[email protected]:s=${W}x${H},format=rgba`);+ png("f0.png", "color=c=blue:s=120x80,format=rgba");+ png("f1.png", "color=c=green:s=120x80,format=rgba");+ png("blank.png", "[email protected]:s=120x80,format=rgba");+ png("border.png", `[email protected]:s=${W}x${H},format=rgba`);+ png("oborder.png", "[email protected]:s=128x256,format=rgba");+ png("shadow.png", `[email protected]:s=${W}x${H},format=rgba`);+ png("cap.png", `[email protected]:s=${W}x${H},format=rgba`);+ png("sub_blank.png", `[email protected]:s=${W}x${H},format=rgba`);+ ff([+ "-f", "lavfi", "-i", `color=c=gray:s=${W}x${H}:d=6:r=24`,+ "-pix_fmt", "yuv420p", path.join(tmpDir, "matte.mp4"),+ ]);++ // Timeline: 1.6s (sped-up cover crop) + 2s (graded, masked, keyed)+ // + 1s gap + 1.5s regioned still = 6.1s.+ return {+ projectId: "e2e",+ width: W,+ height: H,+ fps: 24,+ crf: 30,+ preset: "ultrafast",+ duration: 6.1,+ background: "#102030",+ clips: [+ {+ file: "a.mp4", in: 0, out: 2, muted: false, volume: 0.8,+ fit: "fill", zoom: 1.4, panX: 0.3, panY: -0.2,+ speed: 1.25,+ look: "vhs", lookAmount: 0.7,+ animIn: { style: "pop", seconds: 0.4 },+ transition: 0.5, transitionStyle: "crosszoom",+ },+ {+ file: "b.mp4", in: 0, out: 2, muted: false,+ fit: "fit", reverse: true,+ grade: { brightness: 5, contrast: 8, saturation: -10, temperature: 12, hue: 10 },+ mask: { file: "mask.png" },+ kf: [+ { t: 0, x: 0.5, y: 0.5, scale: 1, rotation: 0, opacity: 1 },+ { t: 1.5, x: 0.6, y: 0.4, scale: 0.8, rotation: 15, opacity: 1 },+ ],+ animOut: { style: "fade", seconds: 0.4 },+ transition: 0.4, transitionStyle: "wipeleft",+ },+ { file: "", in: 0, out: 1, muted: true, hidden: true },+ {+ file: "img.png", in: 0, out: 1.5, muted: true, image: true,+ frame: { x: 0.5, y: 0.45, w: 0.45, h: 0.5 }, fit: "fill",+ mask: { subject: { feather: 1 } },+ border: "border.png",+ shadow: { file: "shadow.png" },+ },+ ],+ overlayVideos: [+ {+ file: "o.mp4", in: 0.2, out: 1.8, start: 0.8, track: 1,+ frame: { x: 0.55, y: 0.5, w: 0.4, h: 0.45 },+ fit: "fill", zoom: 1.3, panX: -0.4,+ muted: false, volume: 0.5, speed: 2,+ headFade: 0.3,+ mask: { file: "omask.png" },+ border: "oborder.png",+ kf: [+ { t: 0, x: 0.75, y: 0.7, scale: 1, rotation: 0, opacity: 1 },+ { t: 0.6, x: 0.65, y: 0.75, scale: 1.1, rotation: -10, opacity: 1 },+ ],+ },+ ],+ audio: [+ { file: "music.m4a", in: 0, out: 4, start: 0, volume: 0.6, fadeIn: 0.2, fadeOut: 0.4 },+ { file: "a.mp4", in: 0, out: 1, start: 2, volume: 1, duck: 0.3, speed: 1.1 },+ ],+ overlays: [+ { file: "el.png", start: 0.5, end: 2.5, lane: 1 },+ {+ frames: [+ { file: "f0.png", duration: 0.5 },+ { file: "f1.png", duration: 0.5 },+ ],+ blank: "blank.png", x: 40, y: 60, start: 1, end: 2,+ subject: { invert: true, feather: 2 }, lane: 0,+ },+ ],+ behindMask: { file: "matte.mp4", from: 0.5 },+ effects: [+ { effect: "vignette", amount: 0.6, start: 0.5, end: 2, lane: 0 },+ { effect: "zoom", amount: 0.4, focus: { x: 0.5, y: 0.4 }, ramp: 0.4, start: 2, end: 3, lane: 1 },+ ],+ captions: [+ { file: "cap.png", start: 0.3, end: 1.2 },+ { file: "cap.png", start: 1.4, end: 2.0 },+ ],+ };+}+for (const delivery of DELIVERIES) describe("export pipeline end to end", () => {test.skipIf(!tools)(`a project using every pipeline feature renders through real ffmpeg as ${delivery.name}`,@@ -100,125 +226,8 @@ for (const delivery of DELIVERIES) describe("export pipeline end to end", () =>const mediaDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-media-"));const tmpDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-job-"));try {- // ---- Footage, stills, and the audio bed (project media) ----------- const vid = (name: string, src: string, hz: number, secs: number) =>- ff([- "-f", "lavfi", "-i", `${src}=size=320x240:rate=24:duration=${secs}`,- "-f", "lavfi", "-i", `sine=frequency=${hz}:sample_rate=44100:duration=${secs}`,- "-shortest", "-pix_fmt", "yuv420p", "-c:a", "aac",- path.join(mediaDir, name),- ]);- vid("a.mp4", "testsrc2", 440, 3);- vid("b.mp4", "smptebars", 660, 3);- vid("o.mp4", "testsrc", 880, 3);- ff([- "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=1:duration=1",- "-frames:v", "1", path.join(mediaDir, "img.png"),- ]);- ff([- "-f", "lavfi", "-i", "sine=frequency=220:sample_rate=44100:duration=6",- "-c:a", "aac", path.join(mediaDir, "music.m4a"),- ]);-- // ---- Client-painted stills and the person matte (job.tmpDir) ------ const png = (name: string, chain: string) =>- ff(["-f", "lavfi", "-i", chain, "-frames:v", "1", path.join(tmpDir, name)]);- png("mask.png", `color=c=white:s=${W}x${H},format=gray`);- png("omask.png", "color=c=0xC0C0C0:s=160x160,format=gray");- png("el.png", `[email protected]:s=${W}x${H},format=rgba`);- png("f0.png", "color=c=blue:s=120x80,format=rgba");- png("f1.png", "color=c=green:s=120x80,format=rgba");- png("blank.png", "[email protected]:s=120x80,format=rgba");- png("border.png", `[email protected]:s=${W}x${H},format=rgba`);- png("oborder.png", "[email protected]:s=128x256,format=rgba");- png("shadow.png", `[email protected]:s=${W}x${H},format=rgba`);- png("cap.png", `[email protected]:s=${W}x${H},format=rgba`);- png("sub_blank.png", `[email protected]:s=${W}x${H},format=rgba`);- ff([- "-f", "lavfi", "-i", `color=c=gray:s=${W}x${H}:d=6:r=24`,- "-pix_fmt", "yuv420p", path.join(tmpDir, "matte.mp4"),- ]);-- // Timeline: 1.6s (sped-up cover crop) + 2s (graded, masked, keyed)- // + 1s gap + 1.5s regioned still = 6.1s.const spec: ExportSpec = {- projectId: "e2e",- width: W,- height: H,- fps: 24,- crf: 30,- preset: "ultrafast",- duration: 6.1,- background: "#102030",- clips: [- {- file: "a.mp4", in: 0, out: 2, muted: false, volume: 0.8,- fit: "fill", zoom: 1.4, panX: 0.3, panY: -0.2,- speed: 1.25,- look: "vhs", lookAmount: 0.7,- animIn: { style: "pop", seconds: 0.4 },- transition: 0.5, transitionStyle: "crosszoom",- },- {- file: "b.mp4", in: 0, out: 2, muted: false,- fit: "fit", reverse: true,- grade: { brightness: 5, contrast: 8, saturation: -10, temperature: 12, hue: 10 },- mask: { file: "mask.png" },- kf: [- { t: 0, x: 0.5, y: 0.5, scale: 1, rotation: 0, opacity: 1 },- { t: 1.5, x: 0.6, y: 0.4, scale: 0.8, rotation: 15, opacity: 1 },- ],- animOut: { style: "fade", seconds: 0.4 },- transition: 0.4, transitionStyle: "wipeleft",- },- { file: "", in: 0, out: 1, muted: true, hidden: true },- {- file: "img.png", in: 0, out: 1.5, muted: true, image: true,- frame: { x: 0.5, y: 0.45, w: 0.45, h: 0.5 }, fit: "fill",- mask: { subject: { feather: 1 } },- border: "border.png",- shadow: { file: "shadow.png" },- },- ],- overlayVideos: [- {- file: "o.mp4", in: 0.2, out: 1.8, start: 0.8, track: 1,- frame: { x: 0.55, y: 0.5, w: 0.4, h: 0.45 },- fit: "fill", zoom: 1.3, panX: -0.4,- muted: false, volume: 0.5, speed: 2,- headFade: 0.3,- mask: { file: "omask.png" },- border: "oborder.png",- kf: [- { t: 0, x: 0.75, y: 0.7, scale: 1, rotation: 0, opacity: 1 },- { t: 0.6, x: 0.65, y: 0.75, scale: 1.1, rotation: -10, opacity: 1 },- ],- },- ],- audio: [- { file: "music.m4a", in: 0, out: 4, start: 0, volume: 0.6, fadeIn: 0.2, fadeOut: 0.4 },- { file: "a.mp4", in: 0, out: 1, start: 2, volume: 1, duck: 0.3, speed: 1.1 },- ],- overlays: [- { file: "el.png", start: 0.5, end: 2.5, lane: 1 },- {- frames: [- { file: "f0.png", duration: 0.5 },- { file: "f1.png", duration: 0.5 },- ],- blank: "blank.png", x: 40, y: 60, start: 1, end: 2,- subject: { invert: true, feather: 2 }, lane: 0,- },- ],- behindMask: { file: "matte.mp4", from: 0.5 },- effects: [- { effect: "vignette", amount: 0.6, start: 0.5, end: 2, lane: 0 },- { effect: "zoom", amount: 0.4, focus: { x: 0.5, y: 0.4 }, ramp: 0.4, start: 2, end: 3, lane: 1 },- ],- captions: [- { file: "cap.png", start: 0.3, end: 1.2 },- { file: "cap.png", start: 1.4, end: 2.0 },- ],+ ...(await everyFeature(mediaDir, tmpDir)),...delivery.over,};@@ -254,3 +263,164 @@ for (const delivery of DELIVERIES) describe("export pipeline end to end", () =>120_000);});++/** Every frame and sample a file decodes to, one hash per frame. */+// A render's content: each picture's hash, and every decoded sample as+// 16-bit — a single pass hands the muxer sound in whatever packets the graph+// made, a join in even ones, so the samples compare whole.+const ffmpegOut = (args: string[]) => {+ const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", ...args], { maxBuffer: 256 << 20 });+ if (r.status !== 0) {+ throw new Error(`ffmpeg ${args.join(" ")} failed:\n${r.stderr}`);+ }+ return r.stdout;+};+const framesOf = (file: string) => ({+ pictures: ffmpegOut(["-i", file, "-map", "0:v?", "-f", "framemd5", "-"]).toString().split("\n").filter((l) => !l.startsWith("#")),+ samples: new Int16Array(new Uint8Array(ffmpegOut(["-i", file, "-map", "0:a?", "-f", "s16le", "-c:a", "pcm_s16le", "-"])).buffer),+});++// The largest gap between two runs of samples, which must be as long.+const sampleGap = (a: Int16Array, b: Int16Array) => {+ expect(a.length).toBe(b.length);+ return a.reduce((gap, v, i) => Math.max(gap, Math.abs(v - b[i])), 0);+};++describe("a delivery rendered in passes", () => {+ // The same every-feature project rendered in one pass and in passes of two+ // pieces. ProRes is intra-only and the sound is PCM, so the same frames and+ // samples encode to the same bits whichever pass made them: the joined+ // file, its mastered mix and its stems decode exactly as the one pass's do.+ // Both sounds are resampled (the mix to 48kHz, the stems to theirs), the+ // case where a pass cut off the sample grid would show.+ test.skipIf(!tools)(+ "joins into the frames, mix and stems one pass renders",+ async () => {+ const mediaDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-media-"));+ const tmpDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-job-"));+ const jobsDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-jobs-"));+ try {+ const spec: ExportSpec = {+ ...(await everyFeature(mediaDir, tmpDir)),+ codec: "prores4444",+ container: "mov",+ audioCodec: "pcm",+ audioSampleRate: 48000,+ loudness: -16,+ truePeakCeiling: -1,+ stemPlan: [+ { name: "Dialogue", lane: null, file: "1 Dialogue.wav" },+ { name: "Music", lane: 0, file: "2 Music.wav" },+ ],+ };+ expect(passWindows(spec, () => ({ pixels: 0, fps: 30 }), 2).length).toBeGreaterThan(2);++ // Each render runs in its own copy of the job dir, so the stems it+ // packs stay on disk beside its file.+ const render = async (name: string, passPieces: number) => {+ const dir = path.join(jobsDir, name);+ await cp(tmpDir, dir, { recursive: true });+ const job: RenderHandle = {+ tmpDir: dir,+ outPath: path.join(dir, "out.mov"),+ stemsPath: path.join(dir, "stems.zip"),+ progress: 0,+ log: [],+ };+ await runExport(job, spec, (file) => path.join(mediaDir, file), undefined, passPieces);+ return { file: job.outPath, stems: spec.stemPlan!.map((_, i) => path.join(dir, `stem_${i}.wav`)) };+ };+ const whole = await render("whole", Infinity);+ const passes = await render("passes", 2);++ expect(framesOf(passes.file)).toEqual(framesOf(whole.file));+ for (const [i, stem] of passes.stems.entries()) {+ expect(framesOf(stem)).toEqual(framesOf(whole.stems[i]));+ }+ } finally {+ await rm(mediaDir, { recursive: true, force: true });+ await rm(tmpDir, { recursive: true, force: true });+ await rm(jobsDir, { recursive: true, force: true });+ }+ },+ 240_000+ );+});++describe("a long recording cut out of order", () => {+ // Cuts from all over one 40s recording, played out of source order, with+ // sound handles, a split edit, a transition, a cross dissolve, a layered+ // clip and a sound entry reading the same file late. Rendered in passes of+ // three, each pass opens the file at its own earliest read; the file+ // decodes exactly as the one pass's does.+ test.skipIf(!tools)(+ "renders in passes as in one",+ async () => {+ const mediaDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-media-"));+ const jobsDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-jobs-"));+ try {+ ff([+ "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=24:duration=40",+ "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100:duration=40",+ "-shortest", "-pix_fmt", "yuv420p", "-c:a", "aac", "-g", "48",+ path.join(mediaDir, "talk.mp4"),+ ]);+ const at = [31, 4, 22, 12, 36, 8, 27, 17, 2, 33, 14, 25];+ const clips: ExportSpec["clips"] = at.map((from, k) => ({+ file: "talk.mp4",+ in: from,+ out: from + 1.5,+ muted: false,+ soundBack: 0.4,+ soundAhead: 0.4,+ ...(k === 3 ? { speed: 1.5 } : {}),+ ...(k === 5 ? { transition: 0.4 } : {}),+ ...(k === 7 ? { soundCross: 0.2 } : {}),+ ...(k === 9 ? { soundLead: 0.3, splitFade: 0.05 } : {}),+ }));+ const spec: ExportSpec = {+ projectId: "e2e",+ width: W,+ height: H,+ fps: 24,+ crf: 30,+ preset: "ultrafast",+ duration: clips.reduce((sum, c) => sum + (c.out - c.in) / (c.speed ?? 1), 0),+ clips,+ overlayVideos: [+ {+ file: "talk.mp4", in: 29, out: 31, start: 12, track: 1,+ frame: { x: 0.55, y: 0.5, w: 0.4, h: 0.45 }, muted: false, volume: 0.5,+ },+ ],+ audio: [{ file: "talk.mp4", in: 35, out: 38, start: 9, volume: 0.4 }],+ overlays: [],+ codec: "prores4444",+ container: "mov",+ audioCodec: "pcm",+ };+ expect(passWindows(spec, () => ({ pixels: 0, fps: 30 }), 3).length).toBeGreaterThan(3);++ const render = async (name: string, passPieces: number) => {+ const dir = path.join(jobsDir, name);+ await mkdir(dir);+ const job: RenderHandle = { tmpDir: dir, outPath: path.join(dir, "out.mov"), progress: 0, log: [] };+ await runExport(job, spec, (file) => path.join(mediaDir, file), undefined, passPieces);+ return job.outPath;+ };+ const whole = await render("whole", Infinity);+ const passes = await render("passes", 3);+ // An AAC decoder fills noise-coded bands from a generator a seek+ // restarts, so a pass that reads from its window's first source+ // second carries the same sound to within one 16-bit step.+ const [got, want] = [framesOf(passes), framesOf(whole)];+ expect(got.pictures).toEqual(want.pictures);+ expect(sampleGap(got.samples, want.samples)).toBeLessThanOrEqual(1);+ } finally {+ await rm(mediaDir, { recursive: true, force: true });+ await rm(jobsDir, { recursive: true, force: true });+ }+ },+ 240_000+ );+});diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.tsindex fe9817bf..09bda98e 100644--- a/site/src/cut/server/exportPipeline.test.ts+++ b/site/src/cut/server/exportPipeline.test.ts@@ -5,6 +5,7 @@ import {colorParamsFilter,colorTagArgs,narrowSpecToRange,+ passWindows,runExport,videoCodecArgs,type ExportPipelineIO,@@ -30,7 +31,10 @@ let packed: { files: { path: string; name: string }[]; output: string }[] = [];/** A pass to cancel the job during, the way the dock's cancel lands mid-render. */let cancelDuring: "master" | "stems" | null = null;-const runsFor = async (over: Partial<ExportSpec>): Promise<string[][]> => {+const runsFor = (over: Partial<ExportSpec>): Promise<string[][]> => runsWithCap(over, undefined);++/** runsFor with the pieces one pass opens capped at `cap`. */+const runsWithCap = async (over: Partial<ExportSpec>, cap: number | undefined): Promise<string[][]> => {const ffmpegCalls: string[][] = [];written = [];mastered = [];@@ -92,7 +96,7 @@ const runsFor = async (over: Partial<ExportSpec>): Promise<string[][]> => {progress: 0,log: [],};- await runExport(job, spec, (f) => `/media/${f}`, io);+ await runExport(job, spec, (f) => `/media/${f}`, io, cap);return ffmpegCalls;};@@ -389,41 +393,34 @@ describe("export filtergraph timebases", () => {});});-describe("one input per cut", () => {- test("cuts off one file each open their own span, so no branch waits on another", async () => {- const runs = await runsFor({- clips: [- clip("a.mp4", { in: 1, out: 3 }),- clip("a.mp4", { in: 5, out: 8 }),- clip("a.mp4", { in: 10, out: 12 }),- ],+describe("cuts off one file", () => {+ test("each stream of the file is fanned out by one split, so ffmpeg prepares each frame once", async () => {+ const g = await graphFor({+ clips: [clip("a.mp4", { in: 1, out: 3 }), clip("a.mp4", { in: 5, out: 8 }), clip("a.mp4", { in: 10, out: 12 })],});- const enc = runs.find((a) => a.includes("-filter_complex"))!;- const inputs = enc.slice(0, enc.indexOf("-filter_complex"));- const chain = enc[enc.indexOf("-filter_complex") + 1];- // The file is opened once for sound and once more per cut.- const opened = inputs.filter((p, i) => inputs[i - 1] === "-i");- expect(opened.filter((p) => p.endsWith("a.mp4"))).toHaveLength(4);- // Each span is cut at the input, preroll included: in 1 -> from 0,- // in 5 -> from 4, in 10 -> from 9.- expect(seeks(enc)).toEqual([- ["0.000", "3.000"],- ["4.000", "4.000"],- ["9.000", "3.000"],+ expect(g.filter((f) => f.includes("[0:v]"))).toEqual(["[0:v]split=3[fan0_v_0][fan0_v_1][fan0_v_2]"]);+ expect(g.filter((f) => f.includes("[0:a]"))).toEqual(["[0:a]asplit=3[fan0_a_0][fan0_a_1][fan0_a_2]"]);+ expect(g.filter((f) => f.startsWith("[fan0_v_")).map((f) => f.slice(0, f.indexOf(",")))).toEqual([+ "[fan0_v_0]trim=0.000:3.000",+ "[fan0_v_1]trim=4.000:8.000",+ "[fan0_v_2]trim=9.000:12.000",]);- // So the picture never trims a shared decoder.- expect(chain).not.toContain(":v]trim=");- // Sound still trims in the graph, off one opening fanned out by one- // asplit: audio frames are small enough that holding them costs little.- expect(chain).toContain("[0:a]asplit=3[fan0_a_0][fan0_a_1][fan0_a_2]");- expect(chain).toContain("[fan0_a_0]atrim=");});- test("a still has no source span, so it keeps reading its looped input", async () => {- const runs = await runsFor({ clips: [{ ...clip("a.png"), image: true }] });- const enc = runs.find((a) => a.includes("-filter_complex"))!;- expect(seeks(enc)).toHaveLength(0);- expect(enc[enc.indexOf("-filter_complex") + 1]).toContain(":v]setpts=PTS-STARTPTS");+ test("a file read late opens at a whole second ahead of its first read, on the clock it always had", async () => {+ const enc = (await runsFor({+ clips: [clip("a.mp4", { in: 40, out: 42 }), clip("a.mp4", { in: 31.5, out: 33 }), clip("b.mp4", { in: 4, out: 6 })],+ })).find((a) => a.includes("-filter_complex"))!;+ // a.mp4's first read is 31.5 less the 1 s preroll and the 1 s margin,+ // 29.5, so it opens at 29 and `-itsoffset` puts the clock back; b.mp4+ // is read near its start and opens as it is.+ const seekOf = (file: string) => {+ const i = enc.indexOf(file);+ return enc[i - 5] === "-ss" ? enc.slice(i - 5, i - 1) : [];+ };+ expect(seekOf("/media/a.mp4")).toEqual(["-ss", "29.000", "-itsoffset", "29.000"]);+ expect(seekOf("/media/b.mp4")).toEqual([]);+ expect(arg(enc, "-filter_complex")).toContain("[fan0_v_0]trim=39.000:42.000");});});@@ -445,11 +442,84 @@ describe("joins", () => {});});+describe("passes", () => {+ const long = (n: number, over: Partial<ExportSpec> = {}): ExportSpec => ({+ projectId: "p", width: 1080, height: 1920, fps: 30, crf: 24, preset: "veryfast",+ duration: n * 2,+ clips: Array.from({ length: n }, (_, i) => clip("a.mp4", { in: i, out: i + 2 })),+ audio: [],+ overlays: [],+ ...over,+ });++ test("a long delivery is cut into windows that lay end to end on frames", () => {+ const windows = passWindows(long(120), () => ({ pixels: 1080 * 1920, fps: 30 }));+ expect(windows.length).toBeGreaterThan(1);+ expect(windows[0].start).toBe(0);+ expect(windows.at(-1)!.end).toBe(240);+ for (let k = 1; k < windows.length; k++) {+ expect(windows[k].start).toBe(windows[k - 1].end);+ expect(Math.abs(windows[k].start * 30 - Math.round(windows[k].start * 30))).toBeLessThan(1e-6);+ }+ });++ test("a heavier source fills a pass with fewer cuts", () => {+ const hd = passWindows(long(120), () => ({ pixels: 1080 * 1920, fps: 30 }));+ const uhd = passWindows(long(120), () => ({ pixels: 3840 * 2160, fps: 30 }));+ expect(uhd.length).toBeGreaterThan(hd.length);+ // A short cut stays one pass.+ expect(passWindows(long(8), () => ({ pixels: 1080 * 1920, fps: 30 }))).toEqual([{ start: 0, end: 16 }]);+ });++ test("cuts out of file order fill a pass sooner, by the frames the decoder holds for them", () => {+ // 60 two-second cuts over a minute of one file, in order and played back+ // to front: back to front, every cut waits while the decoder runs past it.+ const cuts = (n: number) => Array.from({ length: n }, (_, i) => clip("a.mp4", { in: i * 2, out: i * 2 + 2 }));+ const inOrder = passWindows(long(60, { clips: cuts(60) }), () => ({ pixels: 1280 * 720, fps: 30 }));+ const backward = passWindows(long(60, { clips: cuts(60).reverse() }), () => ({ pixels: 1280 * 720, fps: 30 }));+ expect(backward.length).toBeGreaterThan(inOrder.length);+ // A faster source holds more frames for the same seconds.+ const fast = passWindows(long(60, { clips: cuts(60).reverse() }), () => ({ pixels: 1280 * 720, fps: 60 }));+ expect(fast.length).toBeGreaterThan(backward.length);+ });++ test("a range is split inside itself", () => {+ const windows = passWindows(long(120, { range: { start: 31, end: 200 } }), () => ({ pixels: 1080 * 1920, fps: 30 }));+ expect(windows[0].start).toBe(31);+ expect(windows.at(-1)!.end).toBe(200);+ });++ test("each pass writes a piece and the join stream-copies the pictures under the whole mix", async () => {+ const runs = await runsFor({ clips: Array.from({ length: 6 }, (_, i) => clip("a.mp4", { in: i, out: i + 2 })) });+ expect(runs.filter((a) => a.includes("-filter_complex"))).toHaveLength(1);++ // Forced into passes of two pieces.+ const passes = await runsWithCap({ clips: Array.from({ length: 6 }, (_, i) => clip("a.mp4", { in: i, out: i + 2 })) }, 2);+ const pieces = passes.filter((a) => a.includes("-filter_complex"));+ expect(pieces.length).toBeGreaterThan(1);+ for (const [k, a] of pieces.entries()) {+ // The picture goes to its own file with no sound, the mix out raw+ // beside it.+ const video = a.indexOf(`/tmp/graph-test/piece_${k}.mp4`);+ const mix = a.indexOf(`/tmp/graph-test/piece_${k}.f32`);+ expect(a.slice(a.indexOf("-filter_complex") + 2, video)).not.toContain("-c:a");+ expect(a.slice(video, mix)).toContain("pcm_f32le");+ }+ const join = passes.at(-1)!;+ expect(arg(join, "-display_rotation")).toBe("0");+ expect(arg(join, "-f")).toBe("concat");+ expect(arg(join, "-c:v")).toBe("copy");+ expect(join).toContain(`concat:${pieces.map((_, k) => `/tmp/graph-test/piece_${k}.f32`).join("|")}`);+ expect(join.at(-1)).toBe("/tmp/graph-test/out.mp4");+ expect(written.find((w) => w.file.endsWith("pieces.ffconcat"))!.data).toContain("file '/tmp/graph-test/piece_0.mp4'");+ });+});+describe("a speed curve in the filtergraph", () => {test("a curved clip lays its picture through the map and reads baked sound", async () => {const curved = clip("a.mp4", { speedCurve: [[0, 1], [4, 4]] });const g = await graphFor({ clips: [curved, clip("b.mp4")] });- const video = pictureAt(g, 0);+ const video = g.find((f) => f.startsWith("[0:v]trim="))!;expect(video).toContain("setpts='(clip(T-0,0,");expect(video).not.toContain("(PTS-STARTPTS)/");// The baked WAV joins the inputs after the two media files, and the@@ -466,11 +536,11 @@ describe("a speed curve in the filtergraph", () => {describe("smooth slow motion in the filtergraph", () => {test("a slowed clip runs motion interpolation over its whole span", async () => {const g = await graphFor({ clips: [clip("a.mp4", { speed: 0.5, smoothSlow: true }), clip("b.mp4")] });- const video = pictureAt(g, 0);+ const video = g.find((f) => f.startsWith("[0:v]trim="))!;expect(video).toContain("minterpolate=fps=30:mi_mode=mci");expect(video).not.toContain("split=");// The plain clip beside it is untouched.- expect(pictureAt(g, 1)).not.toContain("minterpolate");+ expect(g.find((f) => f.startsWith("[1:v]trim="))).not.toContain("minterpolate");});test("a curve that dips is interpolated in the dip alone", async () => {@@ -484,7 +554,7 @@ describe("smooth slow motion in the filtergraph", () => {],});const g = await graphFor({ clips: [curved, clip("b.mp4")] });- const head = pictureAt(g, 0);+ const head = g.find((f) => f.startsWith("[0:v]trim="))!;expect(head).toContain("split=3[smic0_0][smic0_1][smic0_2]");const pieces = g.filter((f) => f.startsWith("[smic0_"));expect(pieces.length).toBe(3);@@ -497,17 +567,15 @@ describe("smooth slow motion in the filtergraph", () => {test("a curve stamps the preroll before 0, so interpolation never sees it", async () => {// The 1 s preroll ahead of the in point must land before 0 for the- // interpolation's trim=start=0 to drop it. The input opens at the preroll- // (source 4 s) and `-ss` restamps it to 0, so the in point sits at T=1 and- // the preroll runs T=0..1 — the same picture, read on the seeked clock.+ // interpolation's trim=start=0 to drop it: 4.5 s is preroll, 5 s the in point.const curved = clip("a.mp4", { in: 5, out: 9, smoothSlow: true, speedCurve: [[0, 0.5], [4, 0.5]] });- const head = pictureAt(await graphFor({ clips: [curved] }), 0);+ const head = (await graphFor({ clips: [curved] })).find((f) => f.startsWith("[0:v]trim="))!;expect(head).toContain("trim=start=0,");const expr = /setpts='([^']+)'/.exec(head)![1];const at = (T: number) =>new Function("T", "TB", "clip", "min", `return ${expr};`)(T, 1, (x: number, lo: number, hi: number) => Math.min(hi, Math.max(lo, x)), Math.min) as number;- expect(at(0.5)).toBeLessThan(0);- expect(at(1)).toBeCloseTo(0, 6);+ expect(at(4.5)).toBeLessThan(0);+ expect(at(5)).toBeCloseTo(0, 6);expect(at(5.5)).toBeGreaterThan(0);});@@ -1052,11 +1120,8 @@ describe("a reversed clip in the filtergraph", () => {expect(inputs.some((p) => p.endsWith("turned_clip_0.mov"))).toBe(true);const idx = inputs.filter((p, i) => inputs[i - 1] === "-i").findIndex((p) => p.endsWith("turned_clip_0.mov"));const g = graph[graph.indexOf("-filter_complex") + 1].split(";");- const video = pictureAt(g, 0);- // The mirrored span is cut at the input now; the re-stamp is unchanged.- expect(seeks(graph).some(([ss, t]) => ss === "0.000" && t === "7.000")).toBe(true);- expect(video).toContain("setpts=(PTS-1.000/TB)/2.000");- expect(video.startsWith("[2:v]setpts=")).toBe(true);+ const video = g.find((f) => f.startsWith(`[${idx}:v]trim=`))!;+ expect(video).toContain("trim=0.000:7.000,setpts=(PTS-1.000/TB)/2.000");expect(video).not.toContain("reverse");const audio = g.find((f) => f.startsWith(`[${idx}:a]atrim=`))!;expect(audio).toContain("atrim=1.000:7.000");@@ -1075,8 +1140,8 @@ describe("a reversed clip in the filtergraph", () => {reverse: true,});const g = await graphFor({ clips: [curved, clip("b.mp4")] });- const video = pictureAt(g, 0);- expect(video).toContain("setpts='(clip(T-0,0,");+ const video = g.find((f) => f.startsWith("[1:v]trim="))!;+ expect(video).toContain("trim=0.000:4.000,setpts='(clip(T-0,0,");const first = g.find((f) => f.includes("apad=whole_dur="))!;expect(first).toContain(`apad=whole_dur=${retimeOf(curved).len.toFixed(3)}`);});@@ -1093,16 +1158,6 @@ const encodeRun = async (over: Partial<ExportSpec>) => {};const arg = (args: string[], flag: string) => args[args.indexOf(flag) + 1];-/** A clip's picture chain, in clip order — footage now reads a seeked input- * (`setpts=`) where it used to trim a shared one (`trim=`), and a still has- * always read its looped input straight. */-const pictureAt = (g: string[], nth: number) =>- g.filter((f) => /^\[\d+:v\](trim=|setpts=)/.test(f))[nth]!;--/** The `-ss`/`-t` pairs ffmpeg is handed, in input order. */-const seeks = (args: string[]) =>- args.flatMap((v, i) => (v === "-ss" ? [[args[i + 1], args[i + 3]] as const] : []));-describe("delivery", () => {test("source bitrates reach the worker and Mac encoders", async () => {const args = await encodeRun({ bitrate: 140_000, audioBitrate: 48_000 });@@ -1232,11 +1287,27 @@ describe("delivery", () => {expect(narrowSpecToRange(spec).clips.map((c) => !!c.hidden)).toEqual([false, false, true]);});+ test("a slot the range hides opens none of its inputs", async () => {+ const runs = await runsFor({+ duration: 15,+ range: { start: 11, end: 14 },+ clips: [+ clip("a.mp4", { out: 5, removal: { rgb: "rm_rgb.mov", alpha: "rm_a.mov" } }),+ clip("still.png", { out: 5, image: true }),+ clip("c.mp4", { out: 5 }),+ ],+ });+ const enc = runs.find((a) => a.includes("-filter_complex"))!;+ const opened = enc.filter((p, i) => enc[i - 1] === "-i");+ // The hidden removal clip, still and footage leave the graph whole: no+ // keyed pair whose chain nothing reads, no looped still, no file.+ expect(opened.map((p) => path.basename(p))).toEqual(["c.mp4"]);+ });+test("no range delivers the whole cut", async () => {const args = await encodeRun({ duration: 20 });expect(arg(args, "-filter_complex")).not.toContain("[vrange]");- // Inputs carry their own `-t` now, so read the one on the output side.- expect(args[args.lastIndexOf("-t") + 1]).toBe("20.000");+ expect(arg(args, "-t")).toBe("20.000");});});@@ -1277,7 +1348,7 @@ describe("clip color in the filtergraph", () => {expect(cubes).toHaveLength(1);expect(cubes[0].file).toBe(path.join("/tmp/graph-test", "clip_0.cube"));expect(cubes[0].data).toContain("LUT_3D_SIZE 33");- const line = pictureAt(g, 0);+ const line = g.find((l) => l.startsWith("[0:v]"))!;// The framing scale spells the file's matrix and range; the picture goes// to 16-bit RGB, through the LUT, and comes back as BT.709 video before// the letterbox pad.@@ -1303,7 +1374,7 @@ describe("clip color in the filtergraph", () => {lutSizeWide: 65,clips: [clip("log.mov", { color: { profile: "apple-log", matrix: "bt2020nc", fullRange: false } })],});- const line = pictureAt(g, 0);+ const line = g.find((l) => l.startsWith("[0:v]"))!;expect(line).toMatch(/scale=[^,]*in_color_matrix=bt2020:in_range=tv/);expect(line).toContain("format=gbrp16le,lut3d=file=");expect(written.find((w) => w.file.endsWith(".cube"))!.data).toContain("LUT_3D_SIZE 65");@@ -1313,7 +1384,7 @@ describe("clip color in the filtergraph", () => {const g = await graphFor({clips: [clip("sd.mp4", { color: { profile: "rec709", matrix: "bt601", fullRange: true } })],});- const line = pictureAt(g, 0);+ const line = g.find((l) => l.startsWith("[0:v]"))!;expect(line).not.toContain("lut3d");expect(line).toContain("scale:in_color_matrix=bt601:in_range=pc:out_color_matrix=bt709:out_range=tv");expect(line.indexOf("out_range=tv")).toBeLessThan(line.indexOf("pad=1080:1920"));@@ -1324,7 +1395,7 @@ describe("clip color in the filtergraph", () => {clips: [clip("a.mp4", { grade: { exposure: 5, sharpen: 25, clarity: 30 } })],});const joined = g.join(";");- const head = pictureAt(g, 0);+ const head = g.find((l) => l.startsWith("[0:v]"))!;expect(head).toContain("lut3d=file=");expect(head).toMatch(/scale=out_color_matrix=bt709:out_range=pc,format=yuv444p16le\[dtic0\]$/);// The detail pass: bases from the input luma, gained detail merged back.@@ -1402,7 +1473,7 @@ describe("clip color in the filtergraph", () => {const g = await graphFor({clips: [clip("a.mp4", { speed: 0.5, smoothSlow: true, grade: { exposure: 5 } })],});- const line = pictureAt(g, 0);+ const line = g.find((l) => l.startsWith("[0:v]"))!;expect(line).toContain("minterpolate=");expect(line.indexOf("minterpolate=")).toBeLessThan(line.indexOf("scale:in_color_matrix=bt709:in_range=tv,format=gbrp16le,lut3d"));});@@ -1445,7 +1516,7 @@ describe("an HDR delivery", () => {test("composites at ten bits in HLG: every clip maps into Rec.2020 and comes back as 10-bit video", async () => {const g = await graphFor(hdrSpec());- const line = pictureAt(g, 0);+ const line = g.find((l) => l.startsWith("[0:v]"))!;// An ungraded Rec.709 clip is a conversion here, so it takes the LUT.expect(line).toMatch(/scale=[^,]*in_color_matrix=bt709:in_range=tv/);expect(line).toContain("format=gbrp16le,lut3d=file=");@@ -1462,9 +1533,9 @@ describe("an HDR delivery", () => {test("an HLG file ungraded runs no LUT, and a graded one grades in place", async () => {const hlg = { profile: "hlg" as const, matrix: "bt2020nc" as const, fullRange: false };const plain = await graphFor(hdrSpec({ clips: [clip("h.mp4", { color: hlg })] }));- expect(pictureAt(plain, 0)).not.toContain("lut3d");+ expect(plain.find((l) => l.startsWith("[0:v]"))).not.toContain("lut3d");const graded = await graphFor(hdrSpec({ clips: [clip("h.mp4", { color: hlg, grade: { exposure: 10 } })] }));- const line = pictureAt(graded, 0);+ const line = graded.find((l) => l.startsWith("[0:v]"))!;expect(line).toMatch(/scale=[^,]*in_color_matrix=bt2020:in_range=tv/);expect(line).toContain("lut3d=file=");expect(line).toContain("scale=out_color_matrix=bt2020:out_range=tv");diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.tsindex 1972f1f5..7c71f052 100644--- a/site/src/cut/server/exportPipeline.ts+++ b/site/src/cut/server/exportPipeline.ts@@ -627,22 +627,7 @@ const SOURCE_PREROLL = 1;* screen at the in point comes along (stamped before 0) and the re-stamp to* the output rate shows it on the clip's first frame. */const sourceSpan = (idx: number, c: { in: number; out: number }, rt: Retime) =>- `[${idx}:v]trim=${num(spanStart(c))}:${num(c.out)},setpts=${retimedPts(rt, c.in)}`;--/** Where a footage clip's span starts in its source, preroll included. */-const spanStart = (c: { in: number }) => Math.max(0, c.in - SOURCE_PREROLL);--/**- * The same span read off an input already seeked to it, so the picture never- * trims inside the graph.- *- * `-ss` before `-i` restamps the seeked input to start at 0, so the in point- * sits at `c.in - spanStart(c)` — the preroll it kept — and the re-stamp- * measures from there. Everything downstream sees the frames and timestamps- * the trimmed chain produced.- */-const seekedSpan = (idx: number, c: { in: number }, rt: Retime) =>- `[${idx}:v]setpts=${retimedPts(rt, c.in - spanStart(c))}`;+ `[${idx}:v]trim=${num(Math.max(0, c.in - SOURCE_PREROLL))}:${num(c.out)},setpts=${retimedPts(rt, c.in)}`;/** A curve clamps everything before its first knot to 0; the preroll keeps* its own seconds before 0, as one rate does, so a trim at 0 drops it. */@@ -762,6 +747,149 @@ const spanLen = (c: { in: number; out: number; speed?: number; speedCurve?: Spee* still shapes. */const RANGE_SLACK_S = 0.5;+/** Source seconds a file's input opens ahead of the earliest read the graph+ * makes of it, and the earliest it seeks to at all: a file read from near+ * its start opens as it is. The seek lands on a whole second, a point on+ * every sample and frame grid. */+const READ_MARGIN_S = 1;+const READ_SEEK_MIN_S = 10;++/** How far past a range's edges a clip still shapes the frames inside it. */+const rangeReach = (spec: ExportSpec) =>+ Math.max(+ 0,+ ...spec.clips.map((c) => Math.max(c.transition ?? 0, c.soundCross ?? 0, c.soundAhead ?? 0, c.soundBack ?? 0))+ ) + RANGE_SLACK_S;++/**+ * What one encode pass may hold for its pieces — track-0 clips, layered clips+ * and elements. Every piece's input is primed when the graph opens and its+ * chain keeps frames through the encode, about PIECE_FRAMES of its source's+ * size or the delivery's, whichever is larger; cuts played out of file order+ * add the frames their decoder holds until their turn. One pass over a long+ * timeline grows with it, and passes held to this budget stay flat; a larger+ * budget means fewer passes, each decoding the stretch of file its cuts span.+ * PASS_PIECES caps the count whatever the size: each piece brings a decoder+ * and its threads.+ */+const PASS_HOLD_BYTES = 2 * 1024 * 1024 * 1024;+const PIECE_FRAMES = 16;+const PASS_PIECES = 48;++/** A source's decoded picture: its size in pixels and its frame rate. */+export type SourcePicture = { pixels: number; fps: number };++/** A piece of one window: its timeline span and, for footage, the file and+ * the source seconds its picture reads. */+type PassSpan = { start: number; end: number; file: string; lo?: number; hi?: number };++/**+ * Seconds of source a file's one decoder holds at most for the window's cuts+ * still to come. The decoder reads each file forward once and the join takes+ * cuts in timeline order, so a cut that sits later in the timeline than in the+ * file waits as frames until its turn. Cuts read in file order hold nothing:+ * 0–2, 5–7, 9–11 hold 0 s; 9–11, 0–2, 5–7 hold the 4 s of 0–2 and 5–7 the+ * decoder passes on its way to 11.+ */+function heldSeconds(cuts: PassSpan[]): number {+ const order = [...cuts].sort((x, y) => x.start - y.start);+ let reached = -Infinity;+ let most = 0;+ for (const [i, cut] of order.entries()) {+ reached = Math.max(reached, cut.hi!);+ const held = order.slice(i + 1).reduce((sum, c) => sum + Math.max(0, Math.min(c.hi!, reached) - c.lo!), 0);+ most = Math.max(most, held);+ }+ return most;+}++/**+ * The delivery cut into the windows it renders in, one pass each: every+ * window touches at most `cap` pieces, and their primed frames plus the frames+ * its decoders hold for cuts out of file order stay within PASS_HOLD_BYTES+ * (with the reach a range keeps); each window ends on a frame at a piece's+ * start. `sourceOf` is a file's decoded picture. A delivery under the budget+ * is one window. Each pass is a range export, so its frames and samples are+ * the ones the whole composite has there, and the windows laid end to end+ * are the delivery.+ */+export function passWindows(+ spec: ExportSpec,+ sourceOf: (file: string) => SourcePicture,+ cap: number = PASS_PIECES+): ExportRange[] {+ const from = Math.max(0, spec.range?.start ?? 0);+ const to = Math.min(spec.duration, spec.range?.end ?? spec.duration);+ const reach = rangeReach(spec);+ const frameBytes = (file: string) => 1.5 * Math.max(sourceOf(file).pixels, spec.width * spec.height);++ // Every piece the graph builds a chain for, as a timeline span, with the+ // source a footage piece's picture reads.+ const spans: PassSpan[] = [];+ const footage = (c: Parameters<typeof retimeOf>[0] & { image?: boolean }): Pick<PassSpan, "lo" | "hi"> => {+ if (c.image) {+ return {};+ }+ const rt = retimeOf(c);+ const { lo, hi } = srcSpan(rt, 0, rt.len);+ return { lo: Math.max(0, lo - SOURCE_PREROLL), hi };+ };+ let cursor = 0;+ for (const c of spec.clips) {+ spans.push({ start: cursor, end: cursor + spanLen(c), file: c.file, ...footage(c) });+ cursor += spanLen(c);+ }+ for (const o of spec.overlayVideos ?? []) {+ spans.push({ start: o.start, end: o.start + spanLen(o), file: o.file, ...footage(o) });+ }+ for (const o of spec.overlays) {+ spans.push({ start: o.start, end: o.end, file: "" });+ }++ // A window fits when its pieces' primed frames and its decoders' waiting+ // ones come in under the budget.+ const fits = (a: number, b: number) => {+ const held = spans.filter((s) => s.end >= a - reach && s.start <= b + reach);+ if (held.length > cap) {+ return false;+ }+ let bytes = held.reduce((sum, s) => sum + PIECE_FRAMES * frameBytes(s.file), 0);+ for (const file of new Set(held.filter((s) => s.lo !== undefined).map((s) => s.file))) {+ const cuts = held.filter((s) => s.file === file && s.lo !== undefined);+ bytes += heldSeconds(cuts) * sourceOf(file).fps * frameBytes(file);+ }+ return bytes <= PASS_HOLD_BYTES;+ };++ // Window edges sit on whole frames at piece starts, so each pass's trim+ // takes exactly the frames between two edges.+ const edges = [...new Set(spans.map((s) => gridFrame(s.start, spec.fps) / spec.fps))]+ .filter((t) => t > from && t < to)+ .sort((a, b) => a - b);++ // Each window runs to the furthest edge that keeps it within the budget; a+ // window whose first edge is already over still ends there.+ const windows: ExportRange[] = [];+ let start = from;+ while (!fits(start, to)) {+ const next = edges.filter((t) => t > start);+ if (next.length === 0) {+ break;+ }+ let end = next[0];+ for (const t of next) {+ if (!fits(start, t)) {+ break;+ }+ end = t;+ }+ windows.push({ start, end });+ start = end;+ }+ windows.push({ start, end: to });+ return windows;+}+/*** The spec with everything that cannot reach the range taken out of the* graph's way: a track-0 clip whose slot ends before the range or starts@@ -774,13 +902,7 @@ const RANGE_SLACK_S = 0.5;*/export function narrowSpecToRange(spec: ExportSpec): ExportSpec {if (!spec.range) return spec;- const reach =- Math.max(- 0,- ...spec.clips.map((c) =>- Math.max(c.transition ?? 0, c.soundCross ?? 0, c.soundAhead ?? 0, c.soundBack ?? 0)- )- ) + RANGE_SLACK_S;+ const reach = rangeReach(spec);const from = Math.max(0, spec.range.start) - reach;const to = Math.min(spec.duration, spec.range.end) + reach;const inside = (start: number, end: number) => end >= from && start <= to;@@ -912,12 +1034,14 @@ const realIO: ExportPipelineIO = {/** Render `spec` into `job.outPath`. `mediaPathFor` maps a spec media file* name to its staged path on disk (the engine reads the project folder; the* worker reads its download dir); overlay/caption PNGs are read from- * `job.tmpDir` by base name in both. */+ * `job.tmpDir` by base name in both. `passPieces` caps the pieces one encode+ * pass opens (see passWindows). */export async function runExport(job: RenderHandle,given: ExportSpec,mediaPathFor: (file: string) => string,- io: ExportPipelineIO = realIO+ io: ExportPipelineIO = realIO,+ passPieces: number = PASS_PIECES) {if (given.clips.length === 0) throw new Error("Nothing to export.");const codec = given.codec ?? "h264";@@ -935,6 +1059,47 @@ export async function runExport(}if (await io.exportSourceFiles(job, mediaPathFor, given.sourceSegments, codec, given.audioBitrate)) return;}++ // A long delivery renders in passes held to a memory budget, joined into+ // the one file at the end. A piece's cost follows its source's size and+ // frame rate.+ const pictures = new Map<string, SourcePicture>();+ const footage = [...given.clips, ...(given.overlayVideos ?? [])].filter((c) => c.file && !c.image && !c.removal);+ await Promise.all(+ [...new Set(footage.map((c) => c.file))].map(async (file) => {+ const at = await resolveMedia(io.stat, mediaPathFor, file);+ const [dims, perSecond] = await Promise.all([io.videoDimensions(at), io.videoDecodeCost(at)]);+ const pixels = dims ? dims.width * dims.height : 0;+ pictures.set(file, { pixels, fps: pixels > 0 && perSecond ? perSecond / (1.5 * pixels) : given.fps });+ })+ );+ const windows = passWindows(given, (file) => pictures.get(file) ?? { pixels: 0, fps: given.fps }, passPieces);+ if (windows.length === 1) {+ await renderPass(job, given, mediaPathFor, io, { kind: "delivery" });+ return;+ }+ await renderInPasses(job, given, windows, mediaPathFor, io);+}++/**+ * Where one pass's render lands: the delivery itself, or a piece of a+ * delivery rendered in passes — its picture encoded for the delivery, its mix+ * and stems as sound to lay end to end — with the share of the job's+ * progress the pass covers.+ */+type PassOutput =+ | { kind: "delivery" }+ | { kind: "piece"; video: string; mix: string; stems: string[]; progress: [number, number] };++/** One encode pass over `given`, composited whole and cut to its range. */+async function renderPass(+ job: RenderHandle,+ given: ExportSpec,+ mediaPathFor: (file: string) => string,+ io: ExportPipelineIO,+ out: PassOutput+) {+ const codec = given.codec ?? "h264";let spec = await withSpecColors(narrowSpecToRange(given), mediaPathFor);const { width: W, height: H, fps } = spec;@@ -1166,11 +1331,12 @@ export async function runExport(// plus one per uploaded overlay PNG.// Still images are excluded here: a plain `-i file` decodes one frame, so// each image clip/overlay gets its own looped input below instead. Gap- // spacers (empty file) reference no media at all — they render as black.+ // spacers (empty file) reference no media at all — they render as black,+ // and so do hidden clips outside a range, whose files stay closed.const mediaFiles = [...new Set([- ...spec.clips.filter((c) => !c.image),+ ...spec.clips.filter((c) => !c.image && !c.hidden),...spec.audio,...overlayVideos.filter((o) => !o.image),]@@ -1196,12 +1362,21 @@ export async function runExport(// Resolve paths in order first so ffmpeg input indices stay deterministic,// then probe every file's streams concurrently.const paths = await Promise.all(mediaFiles.map((f) => resolveMedia(io.stat, mediaPathFor, f)));- // file → the path ffmpeg opens, so a cut can open its own span below.- const pathOf = new Map<string, string>();+ // Each file opens at the earliest source second the graph reads of it, so+ // a pass deep into a long recording never decodes the file from its start.+ // Re-stamped by the same amount, the input carries the timestamps an+ // unseeked one does, and every trim below reads it unchanged.+ const readFrom = new Map<string, number>();+ const readers = [...spec.clips.filter((c) => !c.hidden && !c.image), ...overlayVideos.filter((o) => !o.image), ...spec.audio];+ for (const c of readers) {+ const back = "soundBack" in c ? (c.soundBack ?? 0) : 0;+ const lo = srcSpan(retimeOf(c), -back, 0).lo - SOURCE_PREROLL - READ_MARGIN_S;+ readFrom.set(c.file, Math.min(readFrom.get(c.file) ?? Infinity, lo));+ }mediaFiles.forEach((f, i) => {inputIndex.set(f, nInputs++);- pathOf.set(f, paths[i]);- inputs.push("-i", paths[i]);+ const at = Math.floor(readFrom.get(f) ?? 0);+ inputs.push(...(at >= READ_SEEK_MIN_S ? ["-ss", num(at), "-itsoffset", num(at)] : []), "-i", paths[i]);});await Promise.all(mediaFiles.map(async (f, i) => {@@ -1235,7 +1410,6 @@ export async function runExport(// The turned copies: already probed at the bake.for (const [file, streams] of turned) {inputIndex.set(file, nInputs++);- pathOf.set(file, file);inputs.push("-i", file);audioPresence.set(file, streams.audio);videoPresence.set(file, streams.video);@@ -1294,7 +1468,9 @@ export async function runExport(const imageClipInput = new Map<number, number>();for (let j = 0; j < spec.clips.length; j++) {const c = spec.clips[j];- if (!c.image || !c.file) continue;+ if (!c.image || !c.file || c.hidden) {+ continue;+ }const dur = spanLen(c);imageClipInput.set(j, nInputs++);inputs.push(@@ -1411,7 +1587,9 @@ export async function runExport(const clipRemovalInput = new Map<number, { rgb: number; a: number }>();for (let j = 0; j < spec.clips.length; j++) {const c = spec.clips[j];- if (c.removal) clipRemovalInput.set(j, removalInputs(c.removal));+ if (c.removal && !c.hidden) {+ clipRemovalInput.set(j, removalInputs(c.removal));+ }}const overlayRemovalInput = new Map<(typeof overlayVideos)[number], { rgb: number; a: number }>();for (const oc of overlayVideos) {@@ -1762,40 +1940,6 @@ export async function runExport(inputs.push("-i", file);baked.set(c, { ...bake, idx });}-- /**- * One input per footage cut, seeked to that cut's span, instead of every cut- * trimming one shared decoder.- *- * `[0:v]trim=a:b` written N times makes ffmpeg split one decoder N ways, and- * `concat` reads those branches in order — so every branch past the one being- * read holds the frames already handed to it. What it holds grows with how- * long the encode runs, which is why a longer timeline dies where a short one- * survives, and why a slower preset dies earlier at the same length. Measured- * on one 30-cut 104s render, 1080x1920, veryfast, no subtitle lane:- *- * one input, 30 trim branches 8,320MB- * an input per cut 3,311MB- *- * Seeking at the input also skips decoding the head of each span, so a cut- * late in a long file no longer walks the file to reach its first frame.- *- * Left trimming in the graph: stills (no source span), removal clips (their- * picture is the keyed pair, not this file), and clips whose source is read- * twice over (a sound twin), where a second opening buys nothing.- */- const cutVideoInput = new Map<number, number>();- spec.clips.forEach((c, j) => {- if (c.image || clipRemovalInput.get(j) || c.hidden) return;- if (!c.file || !videoPresence.get(c.file)) return;- const src = pathOf.get(c.file);- if (!src) return;- const from = spanStart(c);- const span = Math.max(1 / fps, c.out - from);- cutVideoInput.set(j, nInputs++);- inputs.push("-ss", num(from), "-t", num(span), "-i", src);- });-/*** The head of an audio chain reading a clip's sound over the timeline-local* window `[fromT, toT]` (seconds from the clip's head; negative reaches@@ -2339,12 +2483,9 @@ export async function runExport(alphaSet(`rvc${j}`, `rva${j}`, `rvs${j}`, `rv${j}`, ",tpad=stop_mode=clone:stop_duration=1");timebase = `[rvs${j}]null`;} else {- const cutIdx = cutVideoInput.get(j);timebase = c.image? `[${idx}:v]setpts=PTS-STARTPTS`- : cutIdx === undefined- ? sourceSpan(idx, c, rt)- : seekedSpan(cutIdx, c, rt);+ : sourceSpan(idx, c, rt);}if ((c.image || rmIn || videoPresence.get(c.file)) && !c.hidden) {const region = regionPx(c.frame, W, H);@@ -3262,7 +3403,12 @@ export async function runExport(const to = num(Math.min(spec.duration, spec.range.end));filters.push(`[${vLabel}]trim=start=${from}:end=${to},setpts=PTS-STARTPTS[vrange]`);vLabel = "vrange";- filters.push(`[${aLabel}]atrim=start=${from}:end=${to},asetpts=PTS-STARTPTS[arange]`);+ // The sound takes the delivery's rate before the cut, so a range starts+ // on the whole delivery's sample grid: ranges laid end to end are the+ // samples one pass would write.+ filters.push(+ `[${aLabel}]aresample=${spec.audioSampleRate ?? MIX_RATE},atrim=start=${from}:end=${to},asetpts=PTS-STARTPTS[arange]`+ );aLabel = "arange";}@@ -3280,13 +3426,16 @@ export async function runExport(}label = treatAudio(label, `s${i}_`);const window = spec.range- ? `,atrim=start=${num(Math.max(0, spec.range.start))}:end=${num(Math.min(spec.duration, spec.range.end))},asetpts=PTS-STARTPTS`+ ? `,aresample=${STEM_RATE},atrim=start=${num(Math.max(0, spec.range.start))}:end=${num(Math.min(spec.duration, spec.range.end))},asetpts=PTS-STARTPTS`: "";filters.push(`[${label}]apad=whole_dur=${num(spec.duration)},atrim=0:${num(spec.duration)}${window}[stem${i}]`);return `stem${i}`;});if (stemPlan.length > 0 && !job.stemsPath) throw new Error("This export has nowhere to put its stems.");- const stemFiles = stemPlan.map((s, i) => ({ path: path.join(job.tmpDir, `stem_${i}.wav`), name: s.file }));+ const stemFiles = stemPlan.map((s, i) => ({+ path: out.kind === "piece" ? out.stems[i] : path.join(job.tmpDir, `stem_${i}.wav`),+ name: s.file,+ }));const stemOutputs = stemLabels.flatMap((label, i) => ["-map", `[${label}]`,"-c:a", "pcm_s24le",@@ -3301,9 +3450,12 @@ export async function runExport(// The raw mix comes out in the delivery's own rate and layout, so the// master weighs the same channels the tab's master does: a mono delivery// is measured as mono, a 5.1 one with its surround weights.+ // A piece's mix always comes out raw: the pieces' samples laid end to end+ // are the delivery's mix, mastered or encoded once at the join.const master = spec.loudness !== undefined;if (master && spec.truePeakCeiling === undefined) throw new Error("A mastered export needs its true-peak ceiling.");- const mixPath = path.join(job.tmpDir, "mix.f32");+ const rawMix = master || out.kind === "piece";+ const mixPath = out.kind === "piece" ? out.mix : path.join(job.tmpDir, "mix.f32");const mixRate = spec.audioSampleRate ?? MIX_RATE;const mixChannels = spec.audioChannels ?? 2;@@ -3312,7 +3464,8 @@ export async function runExport(// Encode into the tmp dir, then re-emit the container to strip a stray output// rotation flag (see the strip pass below). Keeping the encode intermediate// lets the second pass own faststart.- const encodePath = path.join(job.tmpDir, `encode${containerExtension(spec)}`);+ const encodePath = out.kind === "piece" ? out.video : path.join(job.tmpDir, `encode${containerExtension(spec)}`);+ const [shareFrom, shareTo] = out.kind === "piece" ? out.progress : [0, 1];await io.runFfmpeg(job,[@@ -3320,20 +3473,26 @@ export async function runExport(...inputs,"-filter_complex", assertGraphSafe(fanOutInputs(filters).join(";")),"-map", `[${vLabel}]`,- ...(master ? [] : ["-map", `[${aLabel}]`]),+ ...(rawMix ? [] : ["-map", `[${aLabel}]`]),...videoCodecArgs(enc, spec),...colorTagArgs(spec),- ...(master ? [] : audioCodecArgs(spec)),+ ...(rawMix ? [] : audioCodecArgs(spec)),"-t", num(span),encodePath,- ...(master+ ...(rawMix? ["-map", `[${aLabel}]`, "-c:a", "pcm_f32le", "-ar", String(mixRate), "-ac", String(mixChannels), "-f", "f32le", "-t", num(span), mixPath]: []),...stemOutputs,],- (t) => (job.progress = Math.min(0.99, t / Math.max(0.1, span)))+ (t) => (job.progress = Math.min(0.99, shareFrom + (shareTo - shareFrom) * (t / Math.max(0.1, span)))));+ // A piece is done once its files are written; the join masters, packs and+ // muxes the whole.+ if (out.kind === "piece") {+ return;+ }+// The master and the stems pack run in this process, out of a cancel's// reach, so the job is checked after each: a canceled render never goes on// to write its file.@@ -3377,3 +3536,106 @@ export async function runExport(job.progress = 1;}++/** An ffconcat list of pieces laid end to end, each held for its window's+ * length so the next starts on the frame the last one stopped short of. */+function pieceList(pieces: { file: string; seconds: number }[]): string {+ const lines = ["ffconcat version 1.0"];+ for (const p of pieces) {+ lines.push(`file '${p.file}'`, `duration ${edge(p.seconds)}`);+ }+ return lines.join("\n") + "\n";+}++/**+ * Render `given` window by window and join the pieces into the delivery.+ *+ * Each pass composites one window — a range export — so it holds what that+ * window's pieces need and lets go of it when it ends. The pictures were+ * encoded for the delivery and join without a re-encode; the raw mixes and+ * stems lay end to end sample for sample, then are mastered, packed and+ * encoded once over the whole, as a single pass would.+ */+async function renderInPasses(+ job: RenderHandle,+ given: ExportSpec,+ windows: ExportRange[],+ mediaPathFor: (file: string) => string,+ io: ExportPipelineIO+) {+ const span = deliverySpan(given.range, given.duration);+ const ext = containerExtension(given);+ const stemPlan = given.stemPlan ?? [];+ const pieces = windows.map((w, k) => ({+ window: w,+ video: path.join(job.tmpDir, `piece_${k}${ext}`),+ mix: path.join(job.tmpDir, `piece_${k}.f32`),+ stems: stemPlan.map((_, i) => path.join(job.tmpDir, `piece_${k}_stem_${i}.wav`)),+ }));++ // The passes, in order; each takes its window's share of the progress.+ let done = 0;+ for (const p of pieces) {+ const seconds = p.window.end - p.window.start;+ const progress: [number, number] = [done / span, (done + seconds) / span];+ await renderPass(job, { ...given, range: p.window }, mediaPathFor, io, { kind: "piece", ...p, progress });+ stopIfCanceled(job);+ done += seconds;+ }++ // The mix: the pieces' raw samples read as one stream, mastered when the+ // delivery asks for it.+ const mixRate = given.audioSampleRate ?? MIX_RATE;+ const mixChannels = given.audioChannels ?? 2;+ const raw = ["-f", "f32le", "-ar", String(mixRate), "-ac", String(mixChannels)];+ let mix = `concat:${pieces.map((p) => p.mix).join("|")}`;+ if (given.loudness !== undefined) {+ if (given.truePeakCeiling === undefined) {+ throw new Error("A mastered export needs its true-peak ceiling.");+ }+ const mixPath = path.join(job.tmpDir, "mix.f32");+ const masteredPath = path.join(job.tmpDir, "master.f32");+ await io.runFfmpeg(job, ["-y", ...raw, "-i", mix, "-c", "copy", "-f", "f32le", mixPath]);+ await io.masterRawMix(mixPath, masteredPath, {+ sampleRate: mixRate,+ channels: mixChannels,+ targetLufs: given.loudness,+ ceilingDbtp: given.truePeakCeiling,+ });+ stopIfCanceled(job);+ mix = masteredPath;+ }++ // Each stem: its pieces joined into one file, then the pack.+ if (stemPlan.length > 0) {+ if (!job.stemsPath) {+ throw new Error("This export has nowhere to put its stems.");+ }+ const stemFiles = stemPlan.map((s, i) => ({ path: path.join(job.tmpDir, `stem_${i}.wav`), name: s.file }));+ for (const [i, stem] of stemFiles.entries()) {+ const list = path.join(job.tmpDir, `stem_${i}.ffconcat`);+ await io.writeFile(list, pieceList(pieces.map((p) => ({ file: p.stems[i], seconds: p.window.end - p.window.start }))));+ await io.runFfmpeg(job, ["-y", "-f", "concat", "-safe", "0", "-i", list, "-c", "copy", stem.path]);+ }+ await io.packStems(stemFiles, job.stemsPath);+ stopIfCanceled(job);+ }++ // The delivery: the pictures joined as encoded, the stray rotation flag+ // stripped as a single pass strips it, and the mix encoded under them.+ const list = path.join(job.tmpDir, "pieces.ffconcat");+ await io.writeFile(list, pieceList(pieces.map((p) => ({ file: p.video, seconds: p.window.end - p.window.start }))));+ await io.runFfmpeg(job, [+ "-y",+ "-display_rotation", "0",+ "-f", "concat", "-safe", "0", "-i", list,+ ...raw, "-i", mix,+ "-map", "0:v", "-map", "1:a",+ "-c:v", "copy", ...audioCodecArgs(given),+ "-t", num(span),+ "-movflags", "+faststart",+ job.outPath,+ ]);++ job.progress = 1;+}