Open source CapCut alternative.
TypeScript
fix(site): ffmpeg export carries the preview's compressor makeup gain and mono level [rebuild]
David Le1435391bd182
diff --git a/site/packages/effects-kit/src/audioFx.test.ts b/site/packages/effects-kit/src/audioFx.test.tsindex 650941c1..f4b93c74 100644--- a/site/packages/effects-kit/src/audioFx.test.ts+++ b/site/packages/effects-kit/src/audioFx.test.ts@@ -9,8 +9,7 @@ import {SOUND_EQ_BANDS,SOUND_PRESETS,soundFilters,- soundRecipe,-} from "./audioFx";+ soundRecipe, dynamicsMakeupGain } from "./audioFx";import { ALL_EFFECT_IDS, EFFECT_IDS, EFFECT_LABELS } from "./effects";// The bundled engine ffmpeg is LGPL and carries no GPL-only filters. A recipe@@ -144,12 +143,28 @@ describe("clip sound", () => {expect(names[0]).toBe("lowshelf");// The shelves carry the slope Web Audio's shelf nodes run at.expect(chain).toContain("lowshelf=f=100:width_type=s:w=1:g=2");- expect(names[names.length - 3]).toBe("highshelf");- expect(names[names.length - 2]).toBe("acompressor");- expect(names[names.length - 1]).toBe("alimiter");+ expect(names[names.length - 4]).toBe("highshelf");+ expect(names[names.length - 3]).toBe("acompressor");+ expect(names[names.length - 2]).toBe("alimiter");+ expect(names[names.length - 1]).toBe("volume");// Thresholds go to ffmpeg as linear amplitude.expect(chain).toContain("acompressor=threshold=0.126:ratio=3:attack=10:release=80");+ // The browser's dynamics node reads peaks, links the channels on the+ // louder one, and adds a fixed makeup gain; the chain carries all three.+ expect(chain).toContain(":knee=1.259:makeup=2.204:detection=peak:link=maximum");expect(chain).toContain("alimiter=limit=0.891");+ expect(chain).toContain("alimiter=limit=0.891:attack=5:release=50:level=false,volume=1.068");+ });++ test("the makeup gain is the browser kernel's: a full-scale signal's curve gain, inverted, to the 0.6", () => {+ // A hard-knee 20:1 stage at −12 dB puts 0 dBFS at −11.4 dB; the makeup+ // is (10^(11.4/20))^0.6. Measured off a real in-tab render at 2.197.+ expect(dynamicsMakeupGain(-12, 0, 20)).toBeCloseTo(2.198, 2);+ expect(dynamicsMakeupGain(-1, 0, 20)).toBeCloseTo(1.068, 2);+ // A stage that never engages a full-scale signal adds nothing.+ expect(dynamicsMakeupGain(0, 0, 4)).toBeCloseTo(1, 5);+ // The soft knee lands the voice preset near +7 dB.+ expect(20 * Math.log10(dynamicsMakeupGain(-18, 2, 3))).toBeCloseTo(6.86, 1);});});diff --git a/site/packages/effects-kit/src/audioFx.ts b/site/packages/effects-kit/src/audioFx.tsindex 411c0a04..8d11b023 100644--- a/site/packages/effects-kit/src/audioFx.ts+++ b/site/packages/effects-kit/src/audioFx.ts@@ -470,6 +470,47 @@ function bandFilter(b: AudioBandSpec): string {/** dBFS as the linear amplitude ffmpeg's dynamics filters take. */const linear = (db: number) => Math.pow(10, db / 20);+const decibels = (x: number) => 20 * Math.log10(x);++/** The soft knee the compressor stage runs with, in dB, on both renderers. */+export const COMPRESSOR_KNEE_DB = 2;++/**+ * The fixed gain the browser's dynamics node puts on its output.+ *+ * Every browser ships the same kernel for that node, and the kernel raises+ * whatever it outputs by a gain worked out from its own curve: the level a+ * full-scale signal would come out at, inverted and taken to the 0.6 power.+ * The preview and the in-tab fold hear a treated clip that much louder than+ * its curve alone, so the ffmpeg chain carries the same number and an export+ * lands at the level the preview played. This is that kernel's arithmetic:+ * an exponential knee whose steepness is bisected until its slope at the+ * knee's end meets the ratio, then a straight line at the ratio above it.+ */+export function dynamicsMakeupGain(thresholdDb: number, kneeDb: number, ratio: number): number {+ const linearThreshold = linear(thresholdDb);+ const kneeEndDb = thresholdDb + kneeDb;+ const kneeEnd = linear(kneeEndDb);+ const slope = 1 / ratio;+ const kneeCurve = (x: number, k: number) =>+ x < linearThreshold ? x : linearThreshold + (1 - Math.exp(-k * (x - linearThreshold))) / k;+ const slopeAt = (x: number, k: number) => {+ if (x < linearThreshold) return 1;+ const x2 = x * 1.001;+ return (decibels(kneeCurve(x2, k)) - decibels(kneeCurve(x, k))) / (decibels(x2) - decibels(x));+ };+ let minK = 0.1;+ let maxK = 10000;+ let k = 5;+ for (let i = 0; i < 15; i++) {+ if (slopeAt(kneeEnd, k) < slope) maxK = k;+ else minK = k;+ k = Math.sqrt(minK * maxK);+ }+ const fullRangeGain =+ 1 < kneeEnd ? kneeCurve(1, k) : linear(decibels(kneeCurve(kneeEnd, k)) + slope * (0 - kneeEndDb));+ return Math.pow(1 / fullRangeGain, 0.6);+}/*** The effect as one ffmpeg audio chain — comma-joined filters, no stream@@ -510,16 +551,25 @@ export function recipeFilters(r: AudioFxRecipe): string | null {if (r.compressor) {const c = r.compressor;// acompressor's threshold is linear amplitude; attack and release are ms.- // No makeup, so the two sides agree: the Web Audio node adds none either.+ // The knee is the same width in dB as the browser node's, detection is+ // peak with the channels linked on the louder one, which is how that+ // node reads its input, and the makeup is the gain that node adds.+ const makeup = dynamicsMakeupGain(c.thresholdDb, COMPRESSOR_KNEE_DB, c.ratio);steps.push(`acompressor=threshold=${fmt(linear(c.thresholdDb))}:ratio=${fmt(c.ratio)}` +- `:attack=${fmt(c.attackMs)}:release=${fmt(c.releaseMs)}:knee=2:makeup=1`+ `:attack=${fmt(c.attackMs)}:release=${fmt(c.releaseMs)}` ++ `:knee=${fmt(linear(COMPRESSOR_KNEE_DB))}:makeup=${fmt(makeup)}:detection=peak:link=maximum`);}if (r.limiter) {// A lookahead of a few milliseconds, the same as the browser's dynamics// node carries, and no automatic level: the ceiling is the whole point.+ // The browser stands its limiter in with a hard-knee 20:1 compressor at+ // the ceiling, and that node's makeup gain rides the whole signal, so it+ // follows here as a level.+ const makeup = dynamicsMakeupGain(r.limiter.ceilingDb, 0, 20);steps.push(`alimiter=limit=${fmt(linear(r.limiter.ceilingDb))}:attack=5:release=50:level=false`);+ if (Math.abs(makeup - 1) > 1e-3) steps.push(`volume=${fmt(makeup)}`);}if (r.gain !== undefined && r.gain !== 1) steps.push(`volume=${fmt(r.gain)}`);return steps.length > 0 ? steps.join(",") : null;@@ -632,7 +682,7 @@ export function buildAudioFx(const comp = ctx.createDynamicsCompressor();comp.threshold.value = c.thresholdDb;comp.ratio.value = Math.max(1, Math.min(20, c.ratio));- comp.knee.value = 2;+ comp.knee.value = COMPRESSOR_KNEE_DB;comp.attack.value = c.attackMs / 1000;comp.release.value = c.releaseMs / 1000;node.connect(comp);diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.tsindex 30695e47..8a0f82ed 100644--- a/site/src/cut/server/exportPipeline.test.ts+++ b/site/src/cut/server/exportPipeline.test.ts@@ -23,6 +23,7 @@ const runsFor = async (over: Partial<ExportSpec>): Promise<string[][]> => {readFile: (async () => new Uint8Array(0)) as unknown as ExportPipelineIO["readFile"],unlink: (async () => {}) as unknown as ExportPipelineIO["unlink"],hasStream: async () => true,+ audioChannels: async (file) => (file.includes("mono") ? 1 : 2),videoColorInfo: async () => null,videoDecodeCost: async () => null,mediaDuration: async (file) => produced.get(file) ?? 0,@@ -822,3 +823,19 @@ describe("delivery", () => {expect(arg(args, "-maxrate")).toBe("9000000");});});++describe("channel layout", () => {+ test("a mono source is laid into both channels at full level, a stereo one passes through", async () => {+ const f = await graphFor({+ clips: [clip("mono-voice.mp4"), clip("stereo.mp4")],+ audio: [{ file: "mono-bed.m4a", in: 0, out: 2, start: 0, volume: 0.5 }],+ });+ const inputs = ["mono-voice.mp4", "stereo.mp4", "mono-bed.m4a"];+ const stanza = (file: string) =>+ f.find((x) => x.startsWith(`[${inputs.indexOf(file)}:a]`) && x.includes("aresample=44100"))!;+ expect(stanza("mono-voice.mp4")).toContain("aresample=44100,pan=stereo|c0=c0|c1=c0,aformat=");+ expect(stanza("mono-bed.m4a")).toContain("aresample=44100,pan=stereo|c0=c0|c1=c0,aformat=");+ expect(stanza("stereo.mp4")).toContain("aresample=44100,aformat=");+ expect(stanza("stereo.mp4")).not.toContain("pan=");+ });+});diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.tsindex d39a03db..0298588a 100644--- a/site/src/cut/server/exportPipeline.ts+++ b/site/src/cut/server/exportPipeline.ts@@ -2,7 +2,7 @@ import { spawn, type ChildProcess } from "node:child_process";import { deliveryContainer, videoBitrateFor } from "../lib/exportDelivery";import { readFile, stat, unlink, writeFile } from "node:fs/promises";import path from "node:path";-import { atempoChain, hasStream, mediaDuration, num, videoColorInfo, videoDecodeCost } from "./util";+import { atempoChain, audioChannels, hasStream, mediaDuration, num, videoColorInfo, videoDecodeCost } from "./util";import { assertGraphSafe, fexpr } from "./filterGraph";import { bakeRetimedAudio, setptsExpr, type BakedAudio } from "./retimeAudio";import { bakeTurnedMedia } from "./turnMedia";@@ -554,6 +554,7 @@ export interface ExportPipelineIO {readFile: typeof readFile;unlink: typeof unlink;hasStream: typeof hasStream;+ audioChannels: typeof audioChannels;videoColorInfo: typeof videoColorInfo;videoDecodeCost: typeof videoDecodeCost;mediaDuration: typeof mediaDuration;@@ -567,6 +568,7 @@ const realIO: ExportPipelineIO = {readFile,unlink,hasStream,+ audioChannels,videoColorInfo,videoDecodeCost,mediaDuration,@@ -592,7 +594,7 @@ export async function runExport(// becomes a forward clip over that file, its curve's nodes carried along,// and everything below reads it like any other footage. The copy joins the// inputs by its own path, past the project-folder lookup.- const turned = new Map<string, { video: boolean; audio: boolean }>();+ const turned = new Map<string, { video: boolean; audio: boolean; mono: boolean }>();const turn = async <T extends {file: string;@@ -623,6 +625,7 @@ export async function runExport(const hi = Math.max(lo + 0.001, reach.hi);const video = !soundOnly && (await io.hasStream(src, "v"));const audio = await io.hasStream(src, "a");+ const mono = audio && (await io.audioChannels(src)) === 1;const colorFix = video ? sdrConvert(await io.videoColorInfo(src)) : "";const decodeCost = video ? ((await io.videoDecodeCost(src)) ?? 0) : 0;const file = path.join(job.tmpDir, `turned_${tag}.${video ? "mov" : "wav"}`);@@ -637,7 +640,7 @@ export async function runExport({ lo, hi, video, audio, colorFix, decodeCost },file);- turned.set(file, { video, audio });+ turned.set(file, { video, audio, mono });return { ...mirrorRetimable(c, pivot), file };};const turnedClips: ExportSpec["clips"] = [];@@ -706,6 +709,14 @@ export async function runExport(];const audioPresence = new Map<string, boolean>();const videoPresence = new Map<string, boolean>();+ // Files whose audio is one channel. ffmpeg's own mono-to-stereo lays the+ // channel into each side 3 dB down; the browser's mixer, which the preview+ // and the in-tab fold play through, copies it at full level. Every read of+ // a mono file upmixes by hand so the export sits where the preview did.+ const monoFiles = new Set<string>();+ const toStereo = (file: string) =>+ `aresample=44100,${monoFiles.has(file) ? "pan=stereo|c0=c0|c1=c0," : ""}` ++ "aformat=sample_fmts=fltp:channel_layouts=stereo,";// file → filter prefix folding a wide-gamut/HDR source down to BT.709 (or "").const colorFix = new Map<string, string>();const inputs: string[] = [];@@ -729,6 +740,7 @@ export async function runExport(let audioProbeFailed = false;const hasAudio = await io.hasStream(paths[i], "a", () => (audioProbeFailed = true));audioPresence.set(f, hasAudio || audioProbeFailed);+ if (hasAudio && (await io.audioChannels(paths[i])) === 1) monoFiles.add(f);let videoProbeFailed = false;const hasVideo = await io.hasStream(paths[i], "v", () => (videoProbeFailed = true));videoPresence.set(f, hasVideo || videoProbeFailed);@@ -742,6 +754,7 @@ export async function runExport(inputs.push("-i", file);audioPresence.set(file, streams.audio);videoPresence.set(file, streams.video);+ if (streams.mono) monoFiles.add(file);colorFix.set(file, "");}// Animated overlays: each is its own concat-demuxer slideshow (region-sized@@ -1011,7 +1024,8 @@ export async function runExport(rt,c.soundBack ?? 0,c.soundAhead ?? 0,- file+ file,+ { mono: monoFiles.has(c.file) });const idx = nInputs++;inputs.push("-i", file);@@ -1582,7 +1596,7 @@ export async function runExport(]);filters.push(audioRead(c, 0, dur) +- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,${vol}` ++ `${toStereo(c.file)}${vol}` +`apad=whole_dur=${num(dur)},atrim=0:${num(dur)}${afades}[a${j}]`);} else {@@ -1684,7 +1698,7 @@ export async function runExport(const lab = `xh${key}`;filters.push(audioRead(src, fromT, toT) +- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` ++ toStereo(src.file) +`${soundChain(src.sound)}` +`${(src.volume ?? 1) !== 1 ? `volume=${num(src.volume ?? 1)},` : ""}anull${crossFilters([expr])},` +`adelay=${Math.max(0, Math.round(at * 1000))}:all=1[${lab}]`@@ -1975,7 +1989,7 @@ export async function runExport(const lab = `ovs${k}`;filters.push(audioRead(oc, -back, olen + ahead) +- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` ++ toStereo(oc.file) +`${soundChain(oc.sound)}${vol}${afades}anull` +`${crosses},adelay=${delayMs}:all=1[${lab}]`);@@ -2135,7 +2149,7 @@ export async function runExport(fades.push(`afade=t=out:st=${num(Math.max(0, len - a.fadeOut))}:d=${num(a.fadeOut)}`);filters.push(audioRead(a, 0, len) +- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` ++ toStereo(a.file) +`${soundChain(a.sound)}volume=${num(a.volume)},` +(fades.length ? fades.join(",") + "," : "") +`adelay=${delayMs}:all=1[snd${k}]`diff --git a/site/src/cut/server/retimeAudio.ts b/site/src/cut/server/retimeAudio.tsindex ae8f2804..7428222e 100644--- a/site/src/cut/server/retimeAudio.ts+++ b/site/src/cut/server/retimeAudio.ts@@ -49,7 +49,10 @@ export async function bakeRetimedAudio(retime: Retime,back: number,ahead: number,- outFile: string+ outFile: string,+ /** A mono source is laid into both channels at full level, the way the+ * browser's mixer upmixes it; ffmpeg's own upmix would sit it 3 dB down. */+ opts: { mono?: boolean } = {}): Promise<BakedAudio> {const rt = retime;// A handle cannot reach before the source's first sample; the head reach@@ -70,8 +73,7 @@ export async function bakeRetimedAudio("-t",num(Math.max(0.001, to - from)),"-vn",- "-ac",- String(CHANNELS),+ ...(opts.mono ? ["-af", "pan=stereo|c0=c0|c1=c0"] : ["-ac", String(CHANNELS)]),"-ar",String(BAKE_RATE),"-f",diff --git a/site/src/cut/server/util.ts b/site/src/cut/server/util.tsindex 1b458384..4a228970 100644--- a/site/src/cut/server/util.ts+++ b/site/src/cut/server/util.ts@@ -251,6 +251,40 @@ export function mediaDuration(file: string): Promise<number | null> {});}+/**+ * How many channels a file's first audio stream carries, or null when the+ * probe fails or the file has no audio. A mono source is the one the mix+ * upmixes by hand: ffmpeg's own mono-to-stereo lays the channel into each+ * side 3 dB down, and the browser's mixer copies it at full level.+ */+export function audioChannels(file: string): Promise<number | null> {+ return new Promise((resolve) => {+ const p = spawn("ffprobe", [+ "-v", "error",+ "-select_streams", "a:0",+ "-show_entries", "stream=channels",+ "-of", "csv=p=0",+ file,+ ]);+ let out = "";+ const timer = setTimeout(() => {+ p.kill("SIGKILL");+ resolve(null);+ }, 30_000);+ timer.unref();+ p.stdout.on("data", (d) => (out += d));+ p.on("close", (code) => {+ clearTimeout(timer);+ const n = Number(out.trim().split("\n")[0]);+ resolve(code === 0 && Number.isFinite(n) && n > 0 ? n : null);+ });+ p.on("error", () => {+ clearTimeout(timer);+ resolve(null);+ });+ });+}+/*** Whether a media file carries a stream of the given kind ("a" audio /* "v" video). Resolves false only when ffprobe reports no such stream; a