Skip to content

Open source CapCut alternative.

TypeScript

feat(site): the export splices audio effects into its mix [rebuild]

David Le8afb2a365ffb
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.ts
index 5c781bae..9f0abed2 100644
--- a/site/src/cut/server/exportPipeline.test.ts
+++ b/site/src/cut/server/exportPipeline.test.ts
@@ -491,3 +491,76 @@ describe("the project background in the filtergraph", () => {
expect(g.join(";")).toContain("color=c=black:");
});
});
+
+describe("audio effects in the filtergraph", () => {
+ test("a windowed effect splices the mix and puts the untreated pieces back", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { out: 10 })],
+ effects: [{ effect: "echo", amount: 0.7, start: 2, end: 4 }],
+ });
+ const joined = g.join(";");
+ expect(joined).toContain("asplit=3");
+ expect(joined).toContain("aecho=");
+ // The head, the treated window, and the tail, joined back in order.
+ expect(joined).toContain("atrim=0:2.000,asetpts=PTS-STARTPTS[afxh0]");
+ expect(joined).toContain("atrim=start=4.000,asetpts=PTS-STARTPTS[afxt0]");
+ expect(joined).toContain("concat=n=3:v=0:a=1[afx0]");
+ // The treated piece leaves at the length it went in at, so a chain that
+ // rings past its window cannot push the rest of the sound late.
+ expect(joined).toContain("apad=whole_dur=2.000,atrim=0:2.000");
+ });
+
+ test("an effect over the whole cut needs no splice", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { out: 4 })],
+ effects: [{ effect: "muffle", amount: 1, start: 0, end: 4 }],
+ });
+ const joined = g.join(";");
+ expect(joined).toContain("lowpass=");
+ expect(joined).not.toContain("asplit=");
+ expect(joined).not.toContain("concat=n=");
+ });
+
+ test("an effect at the head splices in two pieces", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { out: 6 })],
+ effects: [{ effect: "telephone", start: 0, end: 2 }],
+ });
+ const joined = g.join(";");
+ expect(joined).toContain("asplit=2");
+ expect(joined).toContain("concat=n=2:v=0:a=1[afx0]");
+ expect(joined).not.toContain("[afxh0]");
+ });
+
+ test("audio effects stay out of the picture chain, and picture effects out of the mix", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { out: 6 })],
+ effects: [
+ { effect: "reverb", start: 1, end: 3 },
+ { effect: "vignette", amount: 0.5, start: 1, end: 3 },
+ ],
+ });
+ const joined = g.join(";");
+ // The vignette gates the picture; the reverb never reaches a video label.
+ expect(joined).toContain("vignette=");
+ expect(joined).toContain("aecho=");
+ expect(joined).not.toContain("aecho=1:1:23|41|59|79|101|127|151|181:0.35|0.223|0.142|0.091|0.058|0.037|0.024|0.015[vfx");
+ });
+
+ test("two audio effects run in series, each over its own window", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { out: 12 })],
+ effects: [
+ { effect: "echo", start: 6, end: 8 },
+ { effect: "crush", start: 1, end: 3 },
+ ],
+ });
+ const joined = g.join(";");
+ // Sorted by start: the crush splices first, and the echo splices what it
+ // handed on.
+ expect(joined).toContain("acrusher=");
+ expect(joined.indexOf("acrusher=")).toBeLessThan(joined.indexOf("aecho="));
+ expect(joined).toContain("[afx0]asplit=3");
+ expect(joined).toContain("concat=n=3:v=0:a=1[afx1]");
+ });
+});
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index 8318eafb..b0697e2a 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -4,7 +4,7 @@ import path from "node:path";
import { atempoChain, hasStream, num, videoColorInfo } from "./util";
import { assertGraphSafe, fexpr } from "./filterGraph";
import { CLIP_MAX_ZOOM, projectFadeSeconds, regionPx, TRANSITION_XFADE, TRANSITION_ZOOM, type ColorGrade, type TransitionStyle } from "../lib/types";
-import { buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, lookFilterLines, lutToCube, shortestTurn, sortedKeys, type OverlayKey } from "@donkeycut/effects-kit";
+import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, shortestTurn, sortedKeys, type OverlayKey } from "@donkeycut/effects-kit";
// The render pipeline itself: spec in, finished mp4 out. Shared by the local
// engine's job registry (jobs.ts) and the cloud render worker, which stage
@@ -196,9 +196,10 @@ export interface ExportSpec {
* union of their windows. */
behindMask?: { file: string; from: number };
/** Time-ranged effect elements — the spec carries ids and knobs; the
- * LGPL-safe chains are built here from the kit's recipes. Each one grades
- * what plays under it: the video, the overlay tracks, and every element on
- * a lane below its own. */
+ * LGPL-safe chains are built here from the kit's recipes. A picture effect
+ * grades what plays under it: the video, the overlay tracks, and every
+ * element on a lane below its own. An audio effect treats the finished mix
+ * over its window instead, and takes no lane. */
effects?: {
effect: string;
amount?: number;
@@ -1564,6 +1565,7 @@ export async function runExport(
.sort((a, b) => laneOfEntry(b) - laneOfEntry(a));
const effects = (spec.effects ?? [])
.map((e, i) => ({ e, i }))
+ .filter(({ e }) => !isAudioEffect(e.effect))
.sort((a, b) => (b.e.lane ?? 0) - (a.e.lane ?? 0));
let placed = 0;
const compositeDownTo = (lane: number) => {
@@ -1680,6 +1682,55 @@ export async function runExport(
aLabel = "amix";
}
+ // Audio effect elements treat the finished mix over their own windows: the
+ // window is cut out of the stream, run through the effect's chain, and put
+ // back between the untreated pieces. The treated piece is padded and
+ // trimmed back to the length it went in at, so a chain that rings past its
+ // input cannot push the rest of the sound late.
+ (spec.effects ?? [])
+ .filter((e) => isAudioEffect(e.effect))
+ .sort((a, b) => a.start - b.start)
+ .forEach((e, k) => {
+ const chain = audioFxFilters(e.effect, e.amount);
+ const from = Math.max(0, e.start);
+ const to = Math.min(e.end, spec.duration);
+ if (!chain || !(to > from)) return;
+ const len = to - from;
+ const fit = `${chain},apad=whole_dur=${num(len)},atrim=0:${num(len)},asetpts=PTS-STARTPTS`;
+ const head = from > 0.001;
+ const tail = to < spec.duration - 0.001;
+ // An effect covering the whole mix is the chain itself; nothing to splice.
+ if (!head && !tail) {
+ filters.push(`[${aLabel}]${fit}[afx${k}]`);
+ aLabel = `afx${k}`;
+ return;
+ }
+ const branches = 1 + (head ? 1 : 0) + (tail ? 1 : 0);
+ filters.push(
+ `[${aLabel}]asplit=${branches}` +
+ (head ? `[afxsh${k}]` : "") +
+ `[afxsw${k}]` +
+ (tail ? `[afxst${k}]` : "")
+ );
+ const parts: string[] = [];
+ if (head) {
+ filters.push(`[afxsh${k}]atrim=0:${num(from)},asetpts=PTS-STARTPTS[afxh${k}]`);
+ parts.push(`afxh${k}`);
+ }
+ filters.push(
+ `[afxsw${k}]atrim=${num(from)}:${num(to)},asetpts=PTS-STARTPTS,${fit}[afxw${k}]`
+ );
+ parts.push(`afxw${k}`);
+ if (tail) {
+ filters.push(`[afxst${k}]atrim=start=${num(to)},asetpts=PTS-STARTPTS[afxt${k}]`);
+ parts.push(`afxt${k}`);
+ }
+ filters.push(
+ `${parts.map((l) => `[${l}]`).join("")}concat=n=${parts.length}:v=0:a=1[afx${k}]`
+ );
+ aLabel = `afx${k}`;
+ });
+
// Whole-video fades on the final composite and mix, so titles, captions,
// overlays, and soundtrack all fade together.
const fadeIn = projectFadeSeconds(spec.fadeIn, spec.duration);