Open source CapCut alternative.
TypeScript
feat(site): the export splices audio effects into its mix [rebuild]
David Le8afb2a365ffb
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.tsindex 5c781bae..9f0abed2 100644--- a/site/src/cut/server/exportPipeline.test.ts+++ b/site/src/cut/server/exportPipeline.test.ts@@ -491,3 +491,76 @@ describe("the project background in the filtergraph", () => {expect(g.join(";")).toContain("color=c=black:");});});++describe("audio effects in the filtergraph", () => {+ test("a windowed effect splices the mix and puts the untreated pieces back", async () => {+ const g = await graphFor({+ clips: [clip("a.mp4", { out: 10 })],+ effects: [{ effect: "echo", amount: 0.7, start: 2, end: 4 }],+ });+ const joined = g.join(";");+ expect(joined).toContain("asplit=3");+ expect(joined).toContain("aecho=");+ // The head, the treated window, and the tail, joined back in order.+ expect(joined).toContain("atrim=0:2.000,asetpts=PTS-STARTPTS[afxh0]");+ expect(joined).toContain("atrim=start=4.000,asetpts=PTS-STARTPTS[afxt0]");+ expect(joined).toContain("concat=n=3:v=0:a=1[afx0]");+ // The treated piece leaves at the length it went in at, so a chain that+ // rings past its window cannot push the rest of the sound late.+ expect(joined).toContain("apad=whole_dur=2.000,atrim=0:2.000");+ });++ test("an effect over the whole cut needs no splice", async () => {+ const g = await graphFor({+ clips: [clip("a.mp4", { out: 4 })],+ effects: [{ effect: "muffle", amount: 1, start: 0, end: 4 }],+ });+ const joined = g.join(";");+ expect(joined).toContain("lowpass=");+ expect(joined).not.toContain("asplit=");+ expect(joined).not.toContain("concat=n=");+ });++ test("an effect at the head splices in two pieces", async () => {+ const g = await graphFor({+ clips: [clip("a.mp4", { out: 6 })],+ effects: [{ effect: "telephone", start: 0, end: 2 }],+ });+ const joined = g.join(";");+ expect(joined).toContain("asplit=2");+ expect(joined).toContain("concat=n=2:v=0:a=1[afx0]");+ expect(joined).not.toContain("[afxh0]");+ });++ test("audio effects stay out of the picture chain, and picture effects out of the mix", async () => {+ const g = await graphFor({+ clips: [clip("a.mp4", { out: 6 })],+ effects: [+ { effect: "reverb", start: 1, end: 3 },+ { effect: "vignette", amount: 0.5, start: 1, end: 3 },+ ],+ });+ const joined = g.join(";");+ // The vignette gates the picture; the reverb never reaches a video label.+ expect(joined).toContain("vignette=");+ expect(joined).toContain("aecho=");+ expect(joined).not.toContain("aecho=1:1:23|41|59|79|101|127|151|181:0.35|0.223|0.142|0.091|0.058|0.037|0.024|0.015[vfx");+ });++ test("two audio effects run in series, each over its own window", async () => {+ const g = await graphFor({+ clips: [clip("a.mp4", { out: 12 })],+ effects: [+ { effect: "echo", start: 6, end: 8 },+ { effect: "crush", start: 1, end: 3 },+ ],+ });+ const joined = g.join(";");+ // Sorted by start: the crush splices first, and the echo splices what it+ // handed on.+ expect(joined).toContain("acrusher=");+ expect(joined.indexOf("acrusher=")).toBeLessThan(joined.indexOf("aecho="));+ expect(joined).toContain("[afx0]asplit=3");+ expect(joined).toContain("concat=n=3:v=0:a=1[afx1]");+ });+});diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.tsindex 8318eafb..b0697e2a 100644--- a/site/src/cut/server/exportPipeline.ts+++ b/site/src/cut/server/exportPipeline.ts@@ -4,7 +4,7 @@ import path from "node:path";import { atempoChain, hasStream, num, videoColorInfo } from "./util";import { assertGraphSafe, fexpr } from "./filterGraph";import { CLIP_MAX_ZOOM, projectFadeSeconds, regionPx, TRANSITION_XFADE, TRANSITION_ZOOM, type ColorGrade, type TransitionStyle } from "../lib/types";-import { buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, lookFilterLines, lutToCube, shortestTurn, sortedKeys, type OverlayKey } from "@donkeycut/effects-kit";+import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, shortestTurn, sortedKeys, type OverlayKey } from "@donkeycut/effects-kit";// The render pipeline itself: spec in, finished mp4 out. Shared by the local// engine's job registry (jobs.ts) and the cloud render worker, which stage@@ -196,9 +196,10 @@ export interface ExportSpec {* union of their windows. */behindMask?: { file: string; from: number };/** Time-ranged effect elements — the spec carries ids and knobs; the- * LGPL-safe chains are built here from the kit's recipes. Each one grades- * what plays under it: the video, the overlay tracks, and every element on- * a lane below its own. */+ * LGPL-safe chains are built here from the kit's recipes. A picture effect+ * grades what plays under it: the video, the overlay tracks, and every+ * element on a lane below its own. An audio effect treats the finished mix+ * over its window instead, and takes no lane. */effects?: {effect: string;amount?: number;@@ -1564,6 +1565,7 @@ export async function runExport(.sort((a, b) => laneOfEntry(b) - laneOfEntry(a));const effects = (spec.effects ?? []).map((e, i) => ({ e, i }))+ .filter(({ e }) => !isAudioEffect(e.effect)).sort((a, b) => (b.e.lane ?? 0) - (a.e.lane ?? 0));let placed = 0;const compositeDownTo = (lane: number) => {@@ -1680,6 +1682,55 @@ export async function runExport(aLabel = "amix";}+ // Audio effect elements treat the finished mix over their own windows: the+ // window is cut out of the stream, run through the effect's chain, and put+ // back between the untreated pieces. The treated piece is padded and+ // trimmed back to the length it went in at, so a chain that rings past its+ // input cannot push the rest of the sound late.+ (spec.effects ?? [])+ .filter((e) => isAudioEffect(e.effect))+ .sort((a, b) => a.start - b.start)+ .forEach((e, k) => {+ const chain = audioFxFilters(e.effect, e.amount);+ const from = Math.max(0, e.start);+ const to = Math.min(e.end, spec.duration);+ if (!chain || !(to > from)) return;+ const len = to - from;+ const fit = `${chain},apad=whole_dur=${num(len)},atrim=0:${num(len)},asetpts=PTS-STARTPTS`;+ const head = from > 0.001;+ const tail = to < spec.duration - 0.001;+ // An effect covering the whole mix is the chain itself; nothing to splice.+ if (!head && !tail) {+ filters.push(`[${aLabel}]${fit}[afx${k}]`);+ aLabel = `afx${k}`;+ return;+ }+ const branches = 1 + (head ? 1 : 0) + (tail ? 1 : 0);+ filters.push(+ `[${aLabel}]asplit=${branches}` ++ (head ? `[afxsh${k}]` : "") ++ `[afxsw${k}]` ++ (tail ? `[afxst${k}]` : "")+ );+ const parts: string[] = [];+ if (head) {+ filters.push(`[afxsh${k}]atrim=0:${num(from)},asetpts=PTS-STARTPTS[afxh${k}]`);+ parts.push(`afxh${k}`);+ }+ filters.push(+ `[afxsw${k}]atrim=${num(from)}:${num(to)},asetpts=PTS-STARTPTS,${fit}[afxw${k}]`+ );+ parts.push(`afxw${k}`);+ if (tail) {+ filters.push(`[afxst${k}]atrim=start=${num(to)},asetpts=PTS-STARTPTS[afxt${k}]`);+ parts.push(`afxt${k}`);+ }+ filters.push(+ `${parts.map((l) => `[${l}]`).join("")}concat=n=${parts.length}:v=0:a=1[afx${k}]`+ );+ aLabel = `afx${k}`;+ });+// Whole-video fades on the final composite and mix, so titles, captions,// overlays, and soundtrack all fade together.const fadeIn = projectFadeSeconds(spec.fadeIn, spec.duration);