Skip to content

Open source CapCut alternative.

TypeScript

fix(site): ffmpeg export carries the preview's compressor makeup gain and mono level [rebuild]

David Le1435391bd182
diff --git a/site/packages/effects-kit/src/audioFx.test.ts b/site/packages/effects-kit/src/audioFx.test.ts
index 650941c1..f4b93c74 100644
--- a/site/packages/effects-kit/src/audioFx.test.ts
+++ b/site/packages/effects-kit/src/audioFx.test.ts
@@ -9,8 +9,7 @@ import {
SOUND_EQ_BANDS,
SOUND_PRESETS,
soundFilters,
- soundRecipe,
-} from "./audioFx";
+ soundRecipe, dynamicsMakeupGain } from "./audioFx";
import { ALL_EFFECT_IDS, EFFECT_IDS, EFFECT_LABELS } from "./effects";
// The bundled engine ffmpeg is LGPL and carries no GPL-only filters. A recipe
@@ -144,12 +143,28 @@ describe("clip sound", () => {
expect(names[0]).toBe("lowshelf");
// The shelves carry the slope Web Audio's shelf nodes run at.
expect(chain).toContain("lowshelf=f=100:width_type=s:w=1:g=2");
- expect(names[names.length - 3]).toBe("highshelf");
- expect(names[names.length - 2]).toBe("acompressor");
- expect(names[names.length - 1]).toBe("alimiter");
+ expect(names[names.length - 4]).toBe("highshelf");
+ expect(names[names.length - 3]).toBe("acompressor");
+ expect(names[names.length - 2]).toBe("alimiter");
+ expect(names[names.length - 1]).toBe("volume");
// Thresholds go to ffmpeg as linear amplitude.
expect(chain).toContain("acompressor=threshold=0.126:ratio=3:attack=10:release=80");
+ // The browser's dynamics node reads peaks, links the channels on the
+ // louder one, and adds a fixed makeup gain; the chain carries all three.
+ expect(chain).toContain(":knee=1.259:makeup=2.204:detection=peak:link=maximum");
expect(chain).toContain("alimiter=limit=0.891");
+ expect(chain).toContain("alimiter=limit=0.891:attack=5:release=50:level=false,volume=1.068");
+ });
+
+ test("the makeup gain is the browser kernel's: a full-scale signal's curve gain, inverted, to the 0.6", () => {
+ // A hard-knee 20:1 stage at −12 dB puts 0 dBFS at −11.4 dB; the makeup
+ // is (10^(11.4/20))^0.6. Measured off a real in-tab render at 2.197.
+ expect(dynamicsMakeupGain(-12, 0, 20)).toBeCloseTo(2.198, 2);
+ expect(dynamicsMakeupGain(-1, 0, 20)).toBeCloseTo(1.068, 2);
+ // A stage that never engages a full-scale signal adds nothing.
+ expect(dynamicsMakeupGain(0, 0, 4)).toBeCloseTo(1, 5);
+ // The soft knee lands the voice preset near +7 dB.
+ expect(20 * Math.log10(dynamicsMakeupGain(-18, 2, 3))).toBeCloseTo(6.86, 1);
});
});
diff --git a/site/packages/effects-kit/src/audioFx.ts b/site/packages/effects-kit/src/audioFx.ts
index 411c0a04..8d11b023 100644
--- a/site/packages/effects-kit/src/audioFx.ts
+++ b/site/packages/effects-kit/src/audioFx.ts
@@ -470,6 +470,47 @@ function bandFilter(b: AudioBandSpec): string {
/** dBFS as the linear amplitude ffmpeg's dynamics filters take. */
const linear = (db: number) => Math.pow(10, db / 20);
+const decibels = (x: number) => 20 * Math.log10(x);
+
+/** The soft knee the compressor stage runs with, in dB, on both renderers. */
+export const COMPRESSOR_KNEE_DB = 2;
+
+/**
+ * The fixed gain the browser's dynamics node puts on its output.
+ *
+ * Every browser ships the same kernel for that node, and the kernel raises
+ * whatever it outputs by a gain worked out from its own curve: the level a
+ * full-scale signal would come out at, inverted and taken to the 0.6 power.
+ * The preview and the in-tab fold hear a treated clip that much louder than
+ * its curve alone, so the ffmpeg chain carries the same number and an export
+ * lands at the level the preview played. This is that kernel's arithmetic:
+ * an exponential knee whose steepness is bisected until its slope at the
+ * knee's end meets the ratio, then a straight line at the ratio above it.
+ */
+export function dynamicsMakeupGain(thresholdDb: number, kneeDb: number, ratio: number): number {
+ const linearThreshold = linear(thresholdDb);
+ const kneeEndDb = thresholdDb + kneeDb;
+ const kneeEnd = linear(kneeEndDb);
+ const slope = 1 / ratio;
+ const kneeCurve = (x: number, k: number) =>
+ x < linearThreshold ? x : linearThreshold + (1 - Math.exp(-k * (x - linearThreshold))) / k;
+ const slopeAt = (x: number, k: number) => {
+ if (x < linearThreshold) return 1;
+ const x2 = x * 1.001;
+ return (decibels(kneeCurve(x2, k)) - decibels(kneeCurve(x, k))) / (decibels(x2) - decibels(x));
+ };
+ let minK = 0.1;
+ let maxK = 10000;
+ let k = 5;
+ for (let i = 0; i < 15; i++) {
+ if (slopeAt(kneeEnd, k) < slope) maxK = k;
+ else minK = k;
+ k = Math.sqrt(minK * maxK);
+ }
+ const fullRangeGain =
+ 1 < kneeEnd ? kneeCurve(1, k) : linear(decibels(kneeCurve(kneeEnd, k)) + slope * (0 - kneeEndDb));
+ return Math.pow(1 / fullRangeGain, 0.6);
+}
/**
* The effect as one ffmpeg audio chain — comma-joined filters, no stream
@@ -510,16 +551,25 @@ export function recipeFilters(r: AudioFxRecipe): string | null {
if (r.compressor) {
const c = r.compressor;
// acompressor's threshold is linear amplitude; attack and release are ms.
- // No makeup, so the two sides agree: the Web Audio node adds none either.
+ // The knee is the same width in dB as the browser node's, detection is
+ // peak with the channels linked on the louder one, which is how that
+ // node reads its input, and the makeup is the gain that node adds.
+ const makeup = dynamicsMakeupGain(c.thresholdDb, COMPRESSOR_KNEE_DB, c.ratio);
steps.push(
`acompressor=threshold=${fmt(linear(c.thresholdDb))}:ratio=${fmt(c.ratio)}` +
- `:attack=${fmt(c.attackMs)}:release=${fmt(c.releaseMs)}:knee=2:makeup=1`
+ `:attack=${fmt(c.attackMs)}:release=${fmt(c.releaseMs)}` +
+ `:knee=${fmt(linear(COMPRESSOR_KNEE_DB))}:makeup=${fmt(makeup)}:detection=peak:link=maximum`
);
}
if (r.limiter) {
// A lookahead of a few milliseconds, the same as the browser's dynamics
// node carries, and no automatic level: the ceiling is the whole point.
+ // The browser stands its limiter in with a hard-knee 20:1 compressor at
+ // the ceiling, and that node's makeup gain rides the whole signal, so it
+ // follows here as a level.
+ const makeup = dynamicsMakeupGain(r.limiter.ceilingDb, 0, 20);
steps.push(`alimiter=limit=${fmt(linear(r.limiter.ceilingDb))}:attack=5:release=50:level=false`);
+ if (Math.abs(makeup - 1) > 1e-3) steps.push(`volume=${fmt(makeup)}`);
}
if (r.gain !== undefined && r.gain !== 1) steps.push(`volume=${fmt(r.gain)}`);
return steps.length > 0 ? steps.join(",") : null;
@@ -632,7 +682,7 @@ export function buildAudioFx(
const comp = ctx.createDynamicsCompressor();
comp.threshold.value = c.thresholdDb;
comp.ratio.value = Math.max(1, Math.min(20, c.ratio));
- comp.knee.value = 2;
+ comp.knee.value = COMPRESSOR_KNEE_DB;
comp.attack.value = c.attackMs / 1000;
comp.release.value = c.releaseMs / 1000;
node.connect(comp);
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.ts
index 30695e47..8a0f82ed 100644
--- a/site/src/cut/server/exportPipeline.test.ts
+++ b/site/src/cut/server/exportPipeline.test.ts
@@ -23,6 +23,7 @@ const runsFor = async (over: Partial<ExportSpec>): Promise<string[][]> => {
readFile: (async () => new Uint8Array(0)) as unknown as ExportPipelineIO["readFile"],
unlink: (async () => {}) as unknown as ExportPipelineIO["unlink"],
hasStream: async () => true,
+ audioChannels: async (file) => (file.includes("mono") ? 1 : 2),
videoColorInfo: async () => null,
videoDecodeCost: async () => null,
mediaDuration: async (file) => produced.get(file) ?? 0,
@@ -822,3 +823,19 @@ describe("delivery", () => {
expect(arg(args, "-maxrate")).toBe("9000000");
});
});
+
+describe("channel layout", () => {
+ test("a mono source is laid into both channels at full level, a stereo one passes through", async () => {
+ const f = await graphFor({
+ clips: [clip("mono-voice.mp4"), clip("stereo.mp4")],
+ audio: [{ file: "mono-bed.m4a", in: 0, out: 2, start: 0, volume: 0.5 }],
+ });
+ const inputs = ["mono-voice.mp4", "stereo.mp4", "mono-bed.m4a"];
+ const stanza = (file: string) =>
+ f.find((x) => x.startsWith(`[${inputs.indexOf(file)}:a]`) && x.includes("aresample=44100"))!;
+ expect(stanza("mono-voice.mp4")).toContain("aresample=44100,pan=stereo|c0=c0|c1=c0,aformat=");
+ expect(stanza("mono-bed.m4a")).toContain("aresample=44100,pan=stereo|c0=c0|c1=c0,aformat=");
+ expect(stanza("stereo.mp4")).toContain("aresample=44100,aformat=");
+ expect(stanza("stereo.mp4")).not.toContain("pan=");
+ });
+});
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index d39a03db..0298588a 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -2,7 +2,7 @@ import { spawn, type ChildProcess } from "node:child_process";
import { deliveryContainer, videoBitrateFor } from "../lib/exportDelivery";
import { readFile, stat, unlink, writeFile } from "node:fs/promises";
import path from "node:path";
-import { atempoChain, hasStream, mediaDuration, num, videoColorInfo, videoDecodeCost } from "./util";
+import { atempoChain, audioChannels, hasStream, mediaDuration, num, videoColorInfo, videoDecodeCost } from "./util";
import { assertGraphSafe, fexpr } from "./filterGraph";
import { bakeRetimedAudio, setptsExpr, type BakedAudio } from "./retimeAudio";
import { bakeTurnedMedia } from "./turnMedia";
@@ -554,6 +554,7 @@ export interface ExportPipelineIO {
readFile: typeof readFile;
unlink: typeof unlink;
hasStream: typeof hasStream;
+ audioChannels: typeof audioChannels;
videoColorInfo: typeof videoColorInfo;
videoDecodeCost: typeof videoDecodeCost;
mediaDuration: typeof mediaDuration;
@@ -567,6 +568,7 @@ const realIO: ExportPipelineIO = {
readFile,
unlink,
hasStream,
+ audioChannels,
videoColorInfo,
videoDecodeCost,
mediaDuration,
@@ -592,7 +594,7 @@ export async function runExport(
// becomes a forward clip over that file, its curve's nodes carried along,
// and everything below reads it like any other footage. The copy joins the
// inputs by its own path, past the project-folder lookup.
- const turned = new Map<string, { video: boolean; audio: boolean }>();
+ const turned = new Map<string, { video: boolean; audio: boolean; mono: boolean }>();
const turn = async <
T extends {
file: string;
@@ -623,6 +625,7 @@ export async function runExport(
const hi = Math.max(lo + 0.001, reach.hi);
const video = !soundOnly && (await io.hasStream(src, "v"));
const audio = await io.hasStream(src, "a");
+ const mono = audio && (await io.audioChannels(src)) === 1;
const colorFix = video ? sdrConvert(await io.videoColorInfo(src)) : "";
const decodeCost = video ? ((await io.videoDecodeCost(src)) ?? 0) : 0;
const file = path.join(job.tmpDir, `turned_${tag}.${video ? "mov" : "wav"}`);
@@ -637,7 +640,7 @@ export async function runExport(
{ lo, hi, video, audio, colorFix, decodeCost },
file
);
- turned.set(file, { video, audio });
+ turned.set(file, { video, audio, mono });
return { ...mirrorRetimable(c, pivot), file };
};
const turnedClips: ExportSpec["clips"] = [];
@@ -706,6 +709,14 @@ export async function runExport(
];
const audioPresence = new Map<string, boolean>();
const videoPresence = new Map<string, boolean>();
+ // Files whose audio is one channel. ffmpeg's own mono-to-stereo lays the
+ // channel into each side 3 dB down; the browser's mixer, which the preview
+ // and the in-tab fold play through, copies it at full level. Every read of
+ // a mono file upmixes by hand so the export sits where the preview did.
+ const monoFiles = new Set<string>();
+ const toStereo = (file: string) =>
+ `aresample=44100,${monoFiles.has(file) ? "pan=stereo|c0=c0|c1=c0," : ""}` +
+ "aformat=sample_fmts=fltp:channel_layouts=stereo,";
// file → filter prefix folding a wide-gamut/HDR source down to BT.709 (or "").
const colorFix = new Map<string, string>();
const inputs: string[] = [];
@@ -729,6 +740,7 @@ export async function runExport(
let audioProbeFailed = false;
const hasAudio = await io.hasStream(paths[i], "a", () => (audioProbeFailed = true));
audioPresence.set(f, hasAudio || audioProbeFailed);
+ if (hasAudio && (await io.audioChannels(paths[i])) === 1) monoFiles.add(f);
let videoProbeFailed = false;
const hasVideo = await io.hasStream(paths[i], "v", () => (videoProbeFailed = true));
videoPresence.set(f, hasVideo || videoProbeFailed);
@@ -742,6 +754,7 @@ export async function runExport(
inputs.push("-i", file);
audioPresence.set(file, streams.audio);
videoPresence.set(file, streams.video);
+ if (streams.mono) monoFiles.add(file);
colorFix.set(file, "");
}
// Animated overlays: each is its own concat-demuxer slideshow (region-sized
@@ -1011,7 +1024,8 @@ export async function runExport(
rt,
c.soundBack ?? 0,
c.soundAhead ?? 0,
- file
+ file,
+ { mono: monoFiles.has(c.file) }
);
const idx = nInputs++;
inputs.push("-i", file);
@@ -1582,7 +1596,7 @@ export async function runExport(
]);
filters.push(
audioRead(c, 0, dur) +
- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,${vol}` +
+ `${toStereo(c.file)}${vol}` +
`apad=whole_dur=${num(dur)},atrim=0:${num(dur)}${afades}[a${j}]`
);
} else {
@@ -1684,7 +1698,7 @@ export async function runExport(
const lab = `xh${key}`;
filters.push(
audioRead(src, fromT, toT) +
- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` +
+ toStereo(src.file) +
`${soundChain(src.sound)}` +
`${(src.volume ?? 1) !== 1 ? `volume=${num(src.volume ?? 1)},` : ""}anull${crossFilters([expr])},` +
`adelay=${Math.max(0, Math.round(at * 1000))}:all=1[${lab}]`
@@ -1975,7 +1989,7 @@ export async function runExport(
const lab = `ovs${k}`;
filters.push(
audioRead(oc, -back, olen + ahead) +
- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` +
+ toStereo(oc.file) +
`${soundChain(oc.sound)}${vol}${afades}anull` +
`${crosses},adelay=${delayMs}:all=1[${lab}]`
);
@@ -2135,7 +2149,7 @@ export async function runExport(
fades.push(`afade=t=out:st=${num(Math.max(0, len - a.fadeOut))}:d=${num(a.fadeOut)}`);
filters.push(
audioRead(a, 0, len) +
- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` +
+ toStereo(a.file) +
`${soundChain(a.sound)}volume=${num(a.volume)},` +
(fades.length ? fades.join(",") + "," : "") +
`adelay=${delayMs}:all=1[snd${k}]`
diff --git a/site/src/cut/server/retimeAudio.ts b/site/src/cut/server/retimeAudio.ts
index ae8f2804..7428222e 100644
--- a/site/src/cut/server/retimeAudio.ts
+++ b/site/src/cut/server/retimeAudio.ts
@@ -49,7 +49,10 @@ export async function bakeRetimedAudio(
retime: Retime,
back: number,
ahead: number,
- outFile: string
+ outFile: string,
+ /** A mono source is laid into both channels at full level, the way the
+ * browser's mixer upmixes it; ffmpeg's own upmix would sit it 3 dB down. */
+ opts: { mono?: boolean } = {}
): Promise<BakedAudio> {
const rt = retime;
// A handle cannot reach before the source's first sample; the head reach
@@ -70,8 +73,7 @@ export async function bakeRetimedAudio(
"-t",
num(Math.max(0.001, to - from)),
"-vn",
- "-ac",
- String(CHANNELS),
+ ...(opts.mono ? ["-af", "pan=stereo|c0=c0|c1=c0"] : ["-ac", String(CHANNELS)]),
"-ar",
String(BAKE_RATE),
"-f",
diff --git a/site/src/cut/server/util.ts b/site/src/cut/server/util.ts
index 1b458384..4a228970 100644
--- a/site/src/cut/server/util.ts
+++ b/site/src/cut/server/util.ts
@@ -251,6 +251,40 @@ export function mediaDuration(file: string): Promise<number | null> {
});
}
+/**
+ * How many channels a file's first audio stream carries, or null when the
+ * probe fails or the file has no audio. A mono source is the one the mix
+ * upmixes by hand: ffmpeg's own mono-to-stereo lays the channel into each
+ * side 3 dB down, and the browser's mixer copies it at full level.
+ */
+export function audioChannels(file: string): Promise<number | null> {
+ return new Promise((resolve) => {
+ const p = spawn("ffprobe", [
+ "-v", "error",
+ "-select_streams", "a:0",
+ "-show_entries", "stream=channels",
+ "-of", "csv=p=0",
+ file,
+ ]);
+ let out = "";
+ const timer = setTimeout(() => {
+ p.kill("SIGKILL");
+ resolve(null);
+ }, 30_000);
+ timer.unref();
+ p.stdout.on("data", (d) => (out += d));
+ p.on("close", (code) => {
+ clearTimeout(timer);
+ const n = Number(out.trim().split("\n")[0]);
+ resolve(code === 0 && Number.isFinite(n) && n > 0 ? n : null);
+ });
+ p.on("error", () => {
+ clearTimeout(timer);
+ resolve(null);
+ });
+ });
+}
+
/**
* Whether a media file carries a stream of the given kind ("a" audio /
* "v" video). Resolves false only when ffprobe reports no such stream; a