Skip to content

Open source CapCut alternative.

TypeScript

feat(site): a clip carries its own sound treatment [rebuild]

David Le91ba78d85684
diff --git a/site/packages/effects-kit/src/audioFx.test.ts b/site/packages/effects-kit/src/audioFx.test.ts
index 67765dce..650941c1 100644
--- a/site/packages/effects-kit/src/audioFx.test.ts
+++ b/site/packages/effects-kit/src/audioFx.test.ts
@@ -5,6 +5,11 @@ import {
audioFxBars,
audioFxFilters,
audioFxRecipe,
+ normalizeSound,
+ SOUND_EQ_BANDS,
+ SOUND_PRESETS,
+ soundFilters,
+ soundRecipe,
} from "./audioFx";
import { ALL_EFFECT_IDS, EFFECT_IDS, EFFECT_LABELS } from "./effects";
@@ -20,6 +25,8 @@ const LGPL_AUDIO_FILTERS = new Set([
"equalizer",
"tremolo",
"acrusher",
+ "acompressor",
+ "alimiter",
"volume",
]);
@@ -93,6 +100,59 @@ describe("audio effect recipes", () => {
});
});
+describe("clip sound", () => {
+ test("a neutral treatment stores as absence", () => {
+ expect(normalizeSound(undefined)).toBeUndefined();
+ expect(normalizeSound({})).toBeUndefined();
+ expect(normalizeSound({ eq: [0, 0, 0, 0, 0, 0, 0] })).toBeUndefined();
+ expect(
+ normalizeSound({ compressor: { threshold: -18, ratio: 1, attack: 10, release: 80 } })
+ ).toBeUndefined();
+ expect(normalizeSound({ limiter: { ceiling: 0 } })).toBeUndefined();
+ expect(soundRecipe({})).toBeNull();
+ expect(soundFilters(undefined)).toBeNull();
+ });
+
+ test("the equalizer keeps one value per band and clamps it", () => {
+ const s = normalizeSound({ eq: [40, -40, 1] })!;
+ expect(s.eq).toHaveLength(SOUND_EQ_BANDS.length);
+ expect(s.eq![0]).toBe(12);
+ expect(s.eq![1]).toBe(-12);
+ expect(s.eq![2]).toBe(1);
+ expect(s.eq![6]).toBe(0);
+ // A flat band builds no section.
+ expect(soundRecipe(s)!.bands).toHaveLength(3);
+ });
+
+ test("every preset spells a chain from filters the shipped ffmpeg has", () => {
+ for (const p of SOUND_PRESETS) {
+ const chain = soundFilters(p.sound);
+ if (!p.sound) {
+ expect(chain).toBeNull();
+ continue;
+ }
+ expect(chain).toBeTruthy();
+ for (const name of filterNamesOf(chain!)) {
+ expect(LGPL_AUDIO_FILTERS.has(name)).toBe(true);
+ }
+ }
+ });
+
+ test("the voice preset runs a shelf, peaks, a shelf, then level control", () => {
+ const chain = soundFilters(SOUND_PRESETS.find((p) => p.id === "clear-voice")!.sound)!;
+ const names = filterNamesOf(chain);
+ expect(names[0]).toBe("lowshelf");
+ // The shelves carry the slope Web Audio's shelf nodes run at.
+ expect(chain).toContain("lowshelf=f=100:width_type=s:w=1:g=2");
+ expect(names[names.length - 3]).toBe("highshelf");
+ expect(names[names.length - 2]).toBe("acompressor");
+ expect(names[names.length - 1]).toBe("alimiter");
+ // Thresholds go to ffmpeg as linear amplitude.
+ expect(chain).toContain("acompressor=threshold=0.126:ratio=3:attack=10:release=80");
+ expect(chain).toContain("alimiter=limit=0.891");
+ });
+});
+
describe("audio effect tile figures", () => {
test("bars stay inside the tile whatever the effect and moment", () => {
for (const id of [...AUDIO_EFFECT_IDS, null]) {
diff --git a/site/packages/effects-kit/src/audioFx.ts b/site/packages/effects-kit/src/audioFx.ts
index 4732bbac..9ef79572 100644
--- a/site/packages/effects-kit/src/audioFx.ts
+++ b/site/packages/effects-kit/src/audioFx.ts
@@ -64,13 +64,22 @@ export interface AudioBandSpec {
db?: number;
}
+/** Downward compression: everything over `thresholdDb` is pulled toward it
+ * by `ratio`, reacting over `attackMs` and letting go over `releaseMs`. */
+export interface AudioCompressorSpec {
+ thresholdDb: number;
+ ratio: number;
+ attackMs: number;
+ releaseMs: number;
+}
+
/**
* What an audio effect does, in the terms both renderers can build.
*
* The delay taps are a finite echo: the dry signal plus each tap's delayed
* copy at its own gain, which is exactly what ffmpeg's `aecho` computes, so
- * one description drives both. Bands, tremolo, bit crush and makeup gain run
- * after the taps, in that order.
+ * one description drives both. Bands, tremolo, bit crush, the compressor, the
+ * limiter and makeup gain run after the taps, in that order.
*/
export interface AudioFxRecipe {
/** Gain on the untreated signal that the taps are added to. */
@@ -83,10 +92,163 @@ export interface AudioFxRecipe {
tremolo?: { hz: number; depth: number };
/** Bit-depth reduction, and how much of the crushed signal is heard. */
crush?: { bits: number; mix: number };
+ /** Level control after the filters. */
+ compressor?: AudioCompressorSpec;
+ /** A ceiling in dBFS the signal is held under. */
+ limiter?: { ceilingDb: number };
/** Makeup gain on the finished chain; absent = 1. */
gain?: number;
}
+/* ------------------------------------------------------- clip sound */
+
+/**
+ * A clip's own sound treatment — the Inspector's Sound quality section.
+ *
+ * Where an effect element treats the mix under a window, this rides one clip
+ * and runs on that clip's sound alone, before its fades and the duck. Every
+ * stage is optional and a fully neutral treatment is stored as absence, so a
+ * clip that was never touched carries nothing.
+ */
+export interface ClipSound {
+ /** Gain in dB per `SOUND_EQ_BANDS` entry, in that order. Absent = flat. */
+ eq?: number[];
+ /** dB, ratio (:1), attack ms, release ms. */
+ compressor?: { threshold: number; ratio: number; attack: number; release: number };
+ /** Ceiling in dBFS. */
+ limiter?: { ceiling: number };
+}
+
+/** The fixed band layout of the clip equalizer: a shelf at each end and
+ * peaks between, spaced where speech lives. Fixed frequencies keep a preset
+ * to seven numbers, which is what the sliders, the tool schema and a saved
+ * preset all carry. */
+export const SOUND_EQ_BANDS: {
+ hz: number;
+ type: AudioBandSpec["type"];
+ label: string;
+ /** The region of the spectrum the band sits in, named the way a mixer
+ * names it. */
+ group: "Low" | "Mids" | "Presence" | "Highs";
+}[] = [
+ { hz: 100, type: "lowshelf", label: "100 Hz", group: "Low" },
+ { hz: 200, type: "peaking", label: "200 Hz", group: "Low" },
+ { hz: 400, type: "peaking", label: "400 Hz", group: "Mids" },
+ { hz: 1000, type: "peaking", label: "1 kHz", group: "Mids" },
+ { hz: 3000, type: "peaking", label: "3 kHz", group: "Presence" },
+ { hz: 5000, type: "peaking", label: "5 kHz", group: "Presence" },
+ { hz: 10000, type: "highshelf", label: "10 kHz", group: "Highs" },
+];
+
+/** A peak's width: about an octave and a half, wide enough that two adjacent
+ * bands moved together read as one shape. */
+const SOUND_EQ_Q = 1.0;
+
+export const SOUND_EQ_DB_RANGE = 12;
+/** What the stage sliders offer. A ratio of 1:1 and a ceiling of 0 dBFS are
+ * the stage doing nothing, and `normalizeSound` stores them as absence, so
+ * the ranges stop short of both — dragging to an endpoint shapes the sound
+ * rather than deleting the stage mid-gesture. Switching the section off is
+ * how a stage goes away. */
+export const SOUND_COMPRESSOR_RANGE = {
+ threshold: { min: -60, max: -1 },
+ ratio: { min: 1.5, max: 20 },
+ attack: { min: 0.1, max: 200 },
+ release: { min: 5, max: 1000 },
+};
+export const SOUND_LIMITER_RANGE = { ceiling: { min: -20, max: -0.1 } };
+
+/** The settings a compressor or limiter opens at when its switch is turned
+ * on — the voice preset's numbers, since that is what the switch is for. */
+export const SOUND_COMPRESSOR_DEFAULT = { threshold: -18, ratio: 3, attack: 10, release: 80 };
+export const SOUND_LIMITER_DEFAULT = { ceiling: -1 };
+
+/** The treatments shipped with the editor. `sound` undefined is the flat one. */
+export const SOUND_PRESETS: { id: string; name: string; sound: ClipSound | undefined }[] = [
+ { id: "flat", name: "Flat", sound: undefined },
+ {
+ id: "clear-voice",
+ name: "Clear voice",
+ sound: {
+ eq: [2, 1, -2, -1, 1.5, 1, -1],
+ compressor: SOUND_COMPRESSOR_DEFAULT,
+ limiter: SOUND_LIMITER_DEFAULT,
+ },
+ },
+];
+
+export const soundPresetById = (id: string) => SOUND_PRESETS.find((p) => p.id === id);
+
+const clampTo = (v: number, r: { min: number; max: number }) =>
+ Math.max(r.min, Math.min(r.max, v));
+
+/**
+ * A sound with every stage clamped into range and every neutral stage
+ * dropped, or undefined when nothing is left: the one shape the doc stores.
+ * A flat equalizer, a compressor at 1:1 and a limiter at 0 dB all do nothing,
+ * so they are not kept.
+ */
+export function normalizeSound(sound: ClipSound | undefined | null): ClipSound | undefined {
+ if (!sound) return undefined;
+ const out: ClipSound = {};
+ if (sound.eq) {
+ const eq = SOUND_EQ_BANDS.map((_, i) => {
+ const v = Number(sound.eq![i] ?? 0);
+ return Number.isFinite(v) ? clampTo(v, { min: -SOUND_EQ_DB_RANGE, max: SOUND_EQ_DB_RANGE }) : 0;
+ });
+ if (eq.some((v) => Math.abs(v) > 1e-3)) out.eq = eq;
+ }
+ if (sound.compressor) {
+ const c = sound.compressor;
+ const r = SOUND_COMPRESSOR_RANGE;
+ if (Number.isFinite(c.ratio) && c.ratio > 1 + 1e-3) {
+ out.compressor = {
+ threshold: clampTo(c.threshold, r.threshold),
+ ratio: clampTo(c.ratio, r.ratio),
+ attack: clampTo(c.attack, r.attack),
+ release: clampTo(c.release, r.release),
+ };
+ }
+ }
+ if (sound.limiter) {
+ const raw = sound.limiter.ceiling;
+ if (Number.isFinite(raw) && raw < -1e-3) {
+ out.limiter = { ceiling: clampTo(raw, SOUND_LIMITER_RANGE.ceiling) };
+ }
+ }
+ return out.eq || out.compressor || out.limiter ? out : undefined;
+}
+
+/** The recipe a clip's sound builds, or null when it is neutral. */
+export function soundRecipe(sound: ClipSound | undefined | null): AudioFxRecipe | null {
+ const s = normalizeSound(sound);
+ if (!s) return null;
+ const bands: AudioBandSpec[] = [];
+ if (s.eq) {
+ SOUND_EQ_BANDS.forEach((b, i) => {
+ const db = s.eq![i];
+ if (Math.abs(db) < 1e-3) return;
+ bands.push({ type: b.type, hz: b.hz, db, ...(b.type === "peaking" ? { q: SOUND_EQ_Q } : {}) });
+ });
+ }
+ return {
+ dry: 1,
+ taps: [],
+ bands,
+ ...(s.compressor
+ ? {
+ compressor: {
+ thresholdDb: s.compressor.threshold,
+ ratio: s.compressor.ratio,
+ attackMs: s.compressor.attack,
+ releaseMs: s.compressor.release,
+ },
+ }
+ : {}),
+ ...(s.limiter ? { limiter: { ceilingDb: s.limiter.ceiling } } : {}),
+ };
+}
+
/** How long the treated signal takes to come in and go out at an effect
* window's edges, in seconds. Short enough to read as an instant switch, long
* enough that the switch cannot click. */
@@ -221,15 +383,21 @@ function bandFilter(b: AudioBandSpec): string {
return `lowpass=f=${fmt(b.hz)}:width_type=q:w=${fmt(q)}`;
case "highpass":
return `highpass=f=${fmt(b.hz)}:width_type=q:w=${fmt(q)}`;
+ // Web Audio's shelf nodes ignore Q and run at slope S=1; ffmpeg defaults
+ // to a Q width of 0.5, which is a different curve at the corner. Spelling
+ // the slope out is what keeps the preview and the export on one sound.
case "lowshelf":
- return `lowshelf=f=${fmt(b.hz)}:g=${fmt(b.db ?? 0)}`;
+ return `lowshelf=f=${fmt(b.hz)}:width_type=s:w=1:g=${fmt(b.db ?? 0)}`;
case "highshelf":
- return `highshelf=f=${fmt(b.hz)}:g=${fmt(b.db ?? 0)}`;
+ return `highshelf=f=${fmt(b.hz)}:width_type=s:w=1:g=${fmt(b.db ?? 0)}`;
case "peaking":
return `equalizer=f=${fmt(b.hz)}:width_type=q:w=${fmt(q)}:g=${fmt(b.db ?? 0)}`;
}
}
+/** dBFS as the linear amplitude ffmpeg's dynamics filters take. */
+const linear = (db: number) => Math.pow(10, db / 20);
+
/**
* The effect as one ffmpeg audio chain — comma-joined filters, no stream
* labels, so the caller splices it into whatever window it runs over. Null for
@@ -238,7 +406,17 @@ function bandFilter(b: AudioBandSpec): string {
*/
export function audioFxFilters(effect: string, amount?: number): string | null {
const r = audioFxRecipe(effect, amount);
- if (!r) return null;
+ return r ? recipeFilters(r) : null;
+}
+
+/** A clip's sound as one ffmpeg chain, or null when it is neutral. */
+export function soundFilters(sound: ClipSound | undefined | null): string | null {
+ const r = soundRecipe(sound);
+ return r ? recipeFilters(r) : null;
+}
+
+/** One recipe spelled as ffmpeg filters, in the order the graph builds them. */
+export function recipeFilters(r: AudioFxRecipe): string | null {
const steps: string[] = [];
if (r.taps.length > 0) {
// aecho computes in_gain*dry + out_gain*Σ decay_i·delayed_i, which is the
@@ -256,6 +434,20 @@ export function audioFxFilters(effect: string, amount?: number): string | null {
for (const b of r.bands) steps.push(bandFilter(b));
if (r.tremolo) steps.push(`tremolo=f=${fmt(r.tremolo.hz)}:d=${fmt(r.tremolo.depth)}`);
if (r.crush) steps.push(`acrusher=bits=${fmt(r.crush.bits)}:mix=${fmt(r.crush.mix)}:mode=lin`);
+ if (r.compressor) {
+ const c = r.compressor;
+ // acompressor's threshold is linear amplitude; attack and release are ms.
+ // No makeup, so the two sides agree: the Web Audio node adds none either.
+ steps.push(
+ `acompressor=threshold=${fmt(linear(c.thresholdDb))}:ratio=${fmt(c.ratio)}` +
+ `:attack=${fmt(c.attackMs)}:release=${fmt(c.releaseMs)}:knee=2:makeup=1`
+ );
+ }
+ if (r.limiter) {
+ // A lookahead of a few milliseconds, the same as the browser's dynamics
+ // node carries, and no automatic level: the ceiling is the whole point.
+ steps.push(`alimiter=limit=${fmt(linear(r.limiter.ceilingDb))}:attack=5:release=50:level=false`);
+ }
if (r.gain !== undefined && r.gain !== 1) steps.push(`volume=${fmt(r.gain)}`);
return steps.length > 0 ? steps.join(",") : null;
}
@@ -362,6 +554,31 @@ export function buildAudioFx(
node = mixed;
}
+ if (recipe.compressor) {
+ const c = recipe.compressor;
+ const comp = ctx.createDynamicsCompressor();
+ comp.threshold.value = c.thresholdDb;
+ comp.ratio.value = Math.max(1, Math.min(20, c.ratio));
+ comp.knee.value = 2;
+ comp.attack.value = c.attackMs / 1000;
+ comp.release.value = c.releaseMs / 1000;
+ node.connect(comp);
+ node = comp;
+ }
+
+ if (recipe.limiter) {
+ // Web Audio has no limiter node; a hard-knee compressor at the steepest
+ // ratio the node takes, reacting at once, holds the ceiling the same way.
+ const lim = ctx.createDynamicsCompressor();
+ lim.threshold.value = recipe.limiter.ceilingDb;
+ lim.ratio.value = 20;
+ lim.knee.value = 0;
+ lim.attack.value = 0.001;
+ lim.release.value = 0.05;
+ node.connect(lim);
+ node = lim;
+ }
+
if (recipe.gain !== undefined && recipe.gain !== 1) {
const makeup = ctx.createGain();
makeup.gain.value = recipe.gain;
diff --git a/site/packages/effects-kit/src/audioFxAudible.test.ts b/site/packages/effects-kit/src/audioFxAudible.test.ts
index c7fd5682..7905d257 100644
--- a/site/packages/effects-kit/src/audioFxAudible.test.ts
+++ b/site/packages/effects-kit/src/audioFxAudible.test.ts
@@ -1,6 +1,13 @@
import { describe, expect, test } from "bun:test";
import { AudioBuffer, OfflineAudioContext } from "node-web-audio-api";
-import { AUDIO_EFFECT_IDS, audioFxRecipe, buildAudioFx } from "./audioFx";
+import {
+ AUDIO_EFFECT_IDS,
+ audioFxRecipe,
+ buildAudioFx,
+ SOUND_PRESETS,
+ soundRecipe,
+ type AudioFxRecipe,
+} from "./audioFx";
/**
* What the recipes have to be: audible.
@@ -49,12 +56,13 @@ function voice(): AudioBuffer {
const SOURCE = voice();
-async function render(effect: string | null): Promise<Float32Array> {
+async function render(effect: string | AudioFxRecipe | null): Promise<Float32Array> {
const ctx = new OfflineAudioContext(1, Math.round(SECONDS * RATE), RATE);
const node = ctx.createBufferSource();
node.buffer = SOURCE;
if (effect) {
- const chain = buildAudioFx(ctx, audioFxRecipe(effect)!, 0);
+ const recipe = typeof effect === "string" ? audioFxRecipe(effect)! : effect;
+ const chain = buildAudioFx(ctx, recipe, 0);
node.connect(chain.input);
chain.output.connect(ctx.destination);
} else {
@@ -148,6 +156,22 @@ describe("every audio effect is audible at the strength a drop places", () => {
expect(swing(await render("wobble")) - swing(dry)).toBeGreaterThan(4);
});
+ test("the voice preset is audible and holds its ceiling", async () => {
+ const dry = await render(null);
+ const recipe = soundRecipe(SOUND_PRESETS.find((p) => p.id === "clear-voice")!.sound)!;
+ const wet = await render(recipe);
+ expect(change(wet, dry)).toBeGreaterThan(-26);
+ // The limiter's ceiling is −1 dBFS; nothing gets past it by more than the
+ // node's own overshoot.
+ let peak = 0;
+ for (const v of wet) peak = Math.max(peak, Math.abs(v));
+ expect(dB(peak)).toBeLessThan(0);
+ // A compressor flattens the syllables: the loud ones come down toward
+ // the quiet ones, so the level swing narrows.
+ const flat = await render({ ...recipe, limiter: undefined, bands: [] });
+ expect(swing(flat)).toBeLessThan(swing(dry));
+ });
+
test("the rooms are still sounding after the sound that made them stops", async () => {
// A burst, then silence: a reverb tail and an echo repeat both live in
// the silence, which is the part a spectrum of the whole take cannot show.
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.ts
index 77cf5ebd..cca062af 100644
--- a/site/src/cut/hooks/usePlayback.ts
+++ b/site/src/cut/hooks/usePlayback.ts
@@ -142,6 +142,7 @@ const voiceSpan = (sp: ClipSpan) => {
in: sp.clip.in - sp.soundBack * speed,
out: sp.clip.out + sp.soundAhead * speed,
speed,
+ sound: sp.clip.sound,
};
};
@@ -549,6 +550,7 @@ class Engine {
out: a.out,
speed,
gain: a.volume * g * dg,
+ sound: a.sound,
});
}
return out;
diff --git a/site/src/cut/lib/audioMix.ts b/site/src/cut/lib/audioMix.ts
index b2d8c519..21659b34 100644
--- a/site/src/cut/lib/audioMix.ts
+++ b/site/src/cut/lib/audioMix.ts
@@ -17,7 +17,13 @@
* words that the exported audio says at another time.
*/
-import { AUDIO_FX_RAMP, audioFxRecipe, buildAudioFx } from "@donkeycut/effects-kit";
+import {
+ AUDIO_FX_RAMP,
+ audioFxRecipe,
+ buildAudioFx,
+ soundRecipe,
+ type ClipSound,
+} from "@donkeycut/effects-kit";
import { decodeAudioSpan } from "./mediaRead";
import { timeStretch } from "./timeStretch";
@@ -31,6 +37,8 @@ export interface MixClip {
speed?: number;
/** Gain on the clip's own audio; absent = 1. */
volume?: number;
+ /** The clip's own treatment, run on its sound before the fades. */
+ sound?: ClipSound;
/** Cross-dissolve overlap into the next clip, timeline seconds. */
transition?: number;
/** Half the cross dissolve into the next clip, timeline seconds: the two
@@ -64,6 +72,7 @@ export interface MixItem {
muted?: boolean;
fadeIn?: number;
fadeOut?: number;
+ sound?: ClipSound;
/** While this item is audible, everything else drops to this gain. */
duck?: number;
}
@@ -377,18 +386,29 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB
// Each buffer holds exactly the span its entry plays, already fitted to the
// time that entry occupies, so it starts at its own zero and runs at 1.
+ // The entry's own treatment runs on its sound alone, between the source and
+ // the gain that carries its level, fades and crossings — the order the
+ // preview's voice and the ffmpeg stanza both keep.
const play = (
buf: AudioBuffer,
at: number,
duration: number,
into: AudioNode,
+ sound: ClipSound | undefined,
shape: (gain: GainNode) => void
) => {
const src = ctx.createBufferSource();
src.buffer = buf;
const gain = ctx.createGain();
shape(gain);
- src.connect(gain);
+ const recipe = soundRecipe(sound);
+ if (recipe) {
+ const fx = buildAudioFx(ctx, recipe, Math.max(0, at));
+ src.connect(fx.input);
+ fx.output.connect(gain);
+ } else {
+ src.connect(gain);
+ }
gain.connect(into);
src.start(Math.max(0, at), 0, Math.max(0, duration));
};
@@ -400,7 +420,7 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB
if (!buf) continue; // muted, silent, or a gap spacer: only shapes time
const from = g.at - g.back;
const end = g.at + g.dur;
- play(fitted(key, buf, speedOf(c.speed)), from, g.dur + g.back + g.ahead, ducked, (gain) => {
+ play(fitted(key, buf, speedOf(c.speed)), from, g.dur + g.back + g.ahead, ducked, c.sound, (gain) => {
const level = Math.max(0, c.volume ?? 1);
gain.gain.value = level;
// A crossing and a fade never share an edge — the cut carries one
@@ -431,7 +451,7 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB
// A voiceover that ducks everything else is not itself ducked; it still
// lands on the bus, so the effect elements treat it with the rest.
const into = a.duck !== undefined && a.duck < 1 ? bus : ducked;
- play(fitted(key, buf, speedOf(a.speed)), a.start - back, dur + back + ahead, into, (gain) => {
+ play(fitted(key, buf, speedOf(a.speed)), a.start - back, dur + back + ahead, into, a.sound, (gain) => {
const level = Math.max(0, a.volume);
gain.gain.value = level;
const fIn = Math.min(a.fadeIn ?? 0, dur);
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index a8870799..906ab4c9 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -594,6 +594,7 @@ export async function buildExportPayload(
out: sp.clip.out,
muted: sp.clip.muted,
volume: sp.clip.volume ?? 1,
+ sound: sp.clip.sound,
fit: sp.clip.fit ?? "fit",
zoom: clipZoom(sp.clip),
panX: sp.clip.panX ?? 0,
@@ -801,6 +802,7 @@ export async function buildExportPayload(
panY: c.panY ?? 0,
muted: c.muted,
volume: c.volume,
+ sound: c.sound,
speed: c.speed,
image: assetById.get(c.assetId)!.type === "image",
grade: normalizeGrade(c.grade),
@@ -899,6 +901,7 @@ export async function buildExportPayload(
fadeIn: a.fadeIn ?? 0,
fadeOut: a.fadeOut ?? 0,
speed: a.speed,
+ sound: a.sound,
duck: a.duck,
}));
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index 2b4bb9be..a0ee5038 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -509,6 +509,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
start: sp.start,
volume: sp.clip.volume ?? 1,
speed: sp.clip.speed,
+ sound: sp.clip.sound,
muted: sp.clip.muted || assetIsSilent(sp.asset),
fadeIn: ramps[i].head,
fadeOut: ramps[i].tail,
@@ -531,6 +532,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
speed: a.speed,
fadeIn: a.fadeIn,
fadeOut: a.fadeOut,
+ sound: a.sound,
duck: a.duck,
});
}
@@ -548,6 +550,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
muted: sp.clip.muted || !!sp.clip.hidden || assetIsSilent(sp.asset),
speed: sp.clip.speed,
volume: sp.clip.volume,
+ sound: sp.clip.sound,
transition: sp.transitionOut,
soundCross: sp.soundOut,
soundBack: sp.soundBack,
diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.ts
index ab12186a..073a263d 100644
--- a/site/src/cut/lib/previewMixer.ts
+++ b/site/src/cut/lib/previewMixer.ts
@@ -35,7 +35,13 @@
* chain runs them in.
*/
-import { audioFxRecipe, buildAudioFx, type AudioFxNodes } from "@donkeycut/effects-kit";
+import {
+ audioFxRecipe,
+ buildAudioFx,
+ soundRecipe,
+ type AudioFxNodes,
+ type ClipSound,
+} from "@donkeycut/effects-kit";
import type { WrappedAudioBuffer } from "mediabunny";
import { assembleAudio, decodeAudioSpan, openAudioWalk, type AudioWalk } from "./mediaRead";
import { timeStretch } from "./timeStretch";
@@ -80,6 +86,9 @@ export interface Voice {
speed: number;
/** Everything the frame plan says about how loud it is right now. */
gain: number;
+ /** The clip's own treatment, run on this voice alone before it joins the
+ * bus. */
+ sound?: ClipSound;
}
interface Scheduled {
@@ -94,7 +103,18 @@ interface LiveVoice {
in: number;
out: number;
start: number;
+ /** Where this voice's scheduled windows land. The clip's own treatment
+ * runs between here and the gain, so it hears the clip at its recorded
+ * level — the same place the fold and the ffmpeg stanza run it, ahead of
+ * the level, fades and duck the gain carries. */
+ head: GainNode;
gain: GainNode;
+ /** The treatment's chain, and what it was built from, so a frame that
+ * changes nothing about it touches no nodes. Rebuilt in place when the
+ * treatment changes: the windows already scheduled play on through the
+ * new chain, since they are connected to the head rather than to it. */
+ fx: AudioFxNodes | null;
+ soundKey: string;
/** Windows already handed to the context, oldest first. */
windows: Scheduled[];
/** Timeline time everything scheduled so far runs out at. */
@@ -553,6 +573,7 @@ export class PreviewMixer {
const live = this.voices.get(v.id) ?? this.open(v, t);
if (!live) continue;
live.gain.gain.value = Math.max(0, Math.min(3, v.gain));
+ this.treat(live, v.sound);
// A read that hangs holds the voice as surely as one that fails, and it
// holds it quietly: nothing else would ever notice, and the voice would
// wait on it for the rest of the play.
@@ -611,6 +632,7 @@ export class PreviewMixer {
const live = this.voices.get(v.id) ?? this.open(v, t);
if (!live) continue;
live.gain.gain.value = 0;
+ this.treat(live, v.sound);
const from = Math.max(v.start, t);
if (live.scheduled !== from) this.reaim(live, from);
const source = live.in + (from - live.start) * live.speed;
@@ -678,13 +700,18 @@ export class PreviewMixer {
const gain = ctx.createGain();
gain.gain.value = 0;
gain.connect(this.bus);
+ const head = ctx.createGain();
+ head.connect(gain);
const live: LiveVoice = {
url: v.url,
speed: v.speed,
in: v.in,
out: v.out,
start: v.start,
+ head,
gain,
+ fx: null,
+ soundKey: "",
windows: [],
// Scheduling begins where the playhead is: a voice opened mid-clip —
// playback started inside it, or the clip edited while audible — decodes
@@ -706,7 +733,43 @@ export class PreviewMixer {
private release(live: LiveVoice): void {
this.stopWindows(live);
this.moved(live);
+ live.head.disconnect();
live.gain.disconnect();
+ if (live.fx) {
+ live.fx.input.disconnect();
+ live.fx.output.disconnect();
+ live.fx = null;
+ }
+ }
+
+ /**
+ * Put the clip's sound treatment between the voice's gain and the bus, or
+ * take it out when the clip has none. Keyed on the treatment itself, so
+ * every frame that leaves it alone costs a string compare and nothing
+ * else, and a slider drag rebuilds a chain a few nodes long.
+ */
+ private treat(live: LiveVoice, sound: ClipSound | undefined): void {
+ const ctx = this.ctx;
+ const bus = this.bus;
+ if (!ctx || !bus) return;
+ const key = sound ? JSON.stringify(sound) : "";
+ if (key === live.soundKey) return;
+ live.soundKey = key;
+ live.head.disconnect();
+ if (live.fx) {
+ live.fx.input.disconnect();
+ live.fx.output.disconnect();
+ live.fx = null;
+ }
+ const recipe = soundRecipe(sound);
+ if (!recipe) {
+ live.head.connect(live.gain);
+ return;
+ }
+ const fx = buildAudioFx(ctx, recipe, ctx.currentTime);
+ live.head.connect(fx.input);
+ fx.output.connect(live.gain);
+ live.fx = fx;
}
private stopVoices(): void {
@@ -794,7 +857,7 @@ export class PreviewMixer {
}
const node = this.ctx.createBufferSource();
node.buffer = buffer;
- node.connect(live.gain);
+ node.connect(live.head);
const at = this.anchor.ctx + (from - this.anchor.timeline);
const now = this.ctx.currentTime;
if (at >= now) {
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index bb7809e4..21284ab3 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -3640,20 +3640,20 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
if (mi == null) continue;
// Track-0 clips re-materialize onto track 0 (asClip), so a template
// stands up its own video instead of an empty timeline.
- layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, track: 1, asClip: true });
+ layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, sound: sp.clip.sound, track: 1, asClip: true });
} else {
const c = s.clips.find((x) => x.id === sel.id);
if (!c) continue;
const mi = mediaFor(c.assetId);
if (mi == null) continue;
- layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, track: c.track + 1 });
+ layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, sound: c.sound, track: c.track + 1 });
}
} else if (sel.kind === "audio") {
const c = s.audioClips.find((x) => x.id === sel.id);
if (!c) continue;
const mi = mediaFor(c.assetId);
if (mi == null) continue;
- audio.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, volume: c.volume, fadeIn: c.fadeIn, fadeOut: c.fadeOut, speed: c.speed, duck: c.duck, lane: c.lane });
+ audio.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, volume: c.volume, fadeIn: c.fadeIn, fadeOut: c.fadeOut, speed: c.speed, sound: c.sound, duck: c.duck, lane: c.lane });
} else if (sel.kind === "overlay") {
const o = s.overlays.find((x) => x.id === sel.id);
// Asset-backed stickers stay out: a template copies only the media
@@ -3783,6 +3783,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
muted: l.muted,
...templateFraming(l),
...(l.speed ? { speed: l.speed } : {}),
+ ...(l.sound ? { sound: l.sound } : {}),
}));
const topTrack = Math.max(0, ...overlayLayers(get().clips).map((c) => c.track));
// Template layers store `track` as the source track + 1 (so a track-1
@@ -3800,6 +3801,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
muted: l.muted,
...templateFraming(l),
...(l.speed ? { speed: l.speed } : {}),
+ ...(l.sound ? { sound: l.sound } : {}),
}));
const newAudio: AudioClip[] = template.audio
.filter((a) => assetIds[a.media])
@@ -3813,6 +3815,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
...(a.fadeIn ? { fadeIn: a.fadeIn } : {}),
...(a.fadeOut ? { fadeOut: a.fadeOut } : {}),
...(a.speed ? { speed: a.speed } : {}),
+ ...(a.sound ? { sound: a.sound } : {}),
...(a.duck !== undefined && a.duck < 1 ? { duck: a.duck } : {}),
...(a.lane ? { lane: a.lane } : {}),
}));
diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.ts
index e32e086c..72881247 100644
--- a/site/src/cut/lib/types.ts
+++ b/site/src/cut/lib/types.ts
@@ -5,6 +5,7 @@ import {
stampOverlayKinds,
stripDefaultOverlayKinds,
type ClipRemoval,
+ type ClipSound,
type ColorGrade,
type EffectOverlay,
type LookStyle,
@@ -508,6 +509,9 @@ export interface VideoClip {
muted: boolean;
/** Gain on the clip's own audio, 0..3; absent = 1 (unchanged). */
volume?: number;
+ /** The clip's own sound treatment — equalizer, compressor, limiter — run
+ * on its sound alone, before its fades; absent = untouched. */
+ sound?: ClipSound;
/** How the clip meets its region: letterboxed ("fit", default) or scaled to
* cover it ("fill", cropping the overflow). */
fit?: "fit" | "fill";
@@ -576,7 +580,7 @@ export interface VideoClip {
// Color grading (the dual-renderer math) lives in the effects kit; the model
// types and ranges re-export here so doc-model consumers keep one import.
export { GRADE_BASIC_FIELDS, GRADE_HUE_MAX, GRADE_MAX, HSL_BANDS } from "@donkeycut/effects-kit";
-export type { ColorGrade, GradePresetRef, HslBand } from "@donkeycut/effects-kit";
+export type { ClipSound, ColorGrade, GradePresetRef, HslBand } from "@donkeycut/effects-kit";
// Background removal follows the same split: math and model in the kit, the
// doc-model types re-exported here.
@@ -969,6 +973,8 @@ export interface AudioClip {
volume: number; // 0..3
fadeIn?: number; // seconds, ramp up from the clip start
fadeOut?: number; // seconds, ramp down into the clip end
+ /** Equalizer, compressor, limiter on this clip's sound; absent = untouched. */
+ sound?: ClipSound;
/** Muted from the final mix but kept on the timeline (grayed). */
hidden?: boolean;
/** Playback rate, default 1 (absent). Set only when audio was detached from
@@ -1013,6 +1019,7 @@ export interface TemplateLayer {
opacity?: number;
muted: boolean;
speed?: number;
+ sound?: ClipSound;
track: number;
/** Came from video track 0 — re-materializes as a timeline clip, not an
* overlay, so a template stands up its own footage. */
@@ -1027,6 +1034,7 @@ export interface TemplateAudio {
fadeIn?: number;
fadeOut?: number;
speed?: number;
+ sound?: ClipSound;
duck?: number;
lane?: number;
}
@@ -1041,6 +1049,10 @@ export interface LibraryTemplate {
audio: TemplateAudio[];
texts: Overlay[];
cues: SubtitleCue[];
+ /** A saved sound treatment: a template carrying only this is a sound
+ * preset (see soundPresets.ts), listed in the audio inspector and kept
+ * off the template shelf. */
+ sound?: ClipSound;
}
/** What the client sends to save a selection (media are project file names). */
export type TemplateSaveInput = Omit<LibraryTemplate, "id" | "addedAt">;
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.ts
index 4b1f3f09..bf2af942 100644
--- a/site/src/cut/server/exportPipeline.test.ts
+++ b/site/src/cut/server/exportPipeline.test.ts
@@ -321,6 +321,53 @@ describe("export filtergraph timebases", () => {
});
});
+describe("clip sound in the filtergraph", () => {
+ const sound = {
+ eq: [2, 1, -2, -1, 1.5, 1, -1],
+ compressor: { threshold: -18, ratio: 3, attack: 10, release: 80 },
+ limiter: { ceiling: -1 },
+ };
+
+ test("a treated track-0 clip runs its chain before its level, pad and fades", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { sound, volume: 0.8, animOut: { style: "fade", seconds: 0.4 } })],
+ });
+ const stanza = g.find((f) => f.startsWith("[0:a]"))!;
+ expect(stanza).toBeTruthy();
+ const at = (s: string) => stanza.indexOf(s);
+ expect(at("lowshelf=f=100")).toBeGreaterThan(at("aformat="));
+ expect(at("acompressor=")).toBeGreaterThan(at("highshelf="));
+ expect(at("alimiter=")).toBeGreaterThan(at("acompressor="));
+ expect(at("volume=0.8")).toBeGreaterThan(at("alimiter="));
+ expect(at("apad=")).toBeGreaterThan(at("volume=0.8"));
+ expect(at("afade=t=out")).toBeGreaterThan(at("apad="));
+ });
+
+ test("a treated soundtrack clip and overlay clip carry the same chain", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4")],
+ overlayVideos: [
+ { file: "ov.mp4", in: 0, out: 2, start: 1, track: 1, muted: false, sound },
+ ],
+ audio: [{ file: "music.mp3", in: 0, out: 5, start: 0, volume: 0.8, sound }],
+ });
+ const treated = g.filter((f) => f.includes("alimiter="));
+ expect(treated).toHaveLength(2);
+ // Every stanza treats the resampled stream, so one clip's treatment
+ // sounds the same whichever list it came from.
+ for (const f of treated) {
+ expect(f.indexOf("alimiter=")).toBeLessThan(f.indexOf("adelay="));
+ expect(f.indexOf("aformat=")).toBeLessThan(f.indexOf("lowshelf="));
+ }
+ });
+
+ test("an untreated clip spells no dynamics filter", async () => {
+ const g = await graphFor({ clips: [clip("a.mp4", { sound: { eq: [0, 0, 0, 0, 0, 0, 0] } })] });
+ expect(g.join(";")).not.toContain("acompressor=");
+ expect(g.join(";")).not.toContain("equalizer=");
+ });
+});
+
describe("clip masks in the filtergraph", () => {
test("a masked track-0 clip trims onto a black base and keeps the join sound", async () => {
const g = await graphFor({
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index f41d057e..c8922200 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -4,7 +4,7 @@ import path from "node:path";
import { atempoChain, hasStream, num, videoColorInfo } from "./util";
import { assertGraphSafe, fexpr } from "./filterGraph";
import { CLIP_MAX_ZOOM, projectFadeSeconds, regionPx, TRANSITION_XFADE, TRANSITION_ZOOM, type ColorGrade, type TransitionStyle } from "../lib/types";
-import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, shortestTurn, sortedKeys, type OverlayKey } from "@donkeycut/effects-kit";
+import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, shortestTurn, sortedKeys, soundFilters, type ClipSound, type OverlayKey } from "@donkeycut/effects-kit";
// The render pipeline itself: spec in, finished mp4 out. Shared by the local
// engine's job registry (jobs.ts) and the cloud render worker, which stage
@@ -40,6 +40,14 @@ function crossFilters(exprs: string[]): string {
return `,asetnsamples=n=256:p=0,volume=volume=${fexpr(live.join("*"))}:eval=frame`;
}
+/** A clip's own sound treatment as a chain fragment ending in a comma, or
+ * nothing: spliced into the clip's stanza ahead of its level, fades and
+ * crossings — where the preview's voice and the browser fold run it too. */
+const soundChain = (sound: ClipSound | undefined): string => {
+ const chain = soundFilters(sound);
+ return chain ? `${chain},` : "";
+};
+
export interface ExportSpec {
projectId: string;
/** Where the render lands instead of a stamped file in exports/: "hls" is the
@@ -68,6 +76,8 @@ export interface ExportSpec {
muted: boolean;
/** Gain on the clip's own audio, 0..3; absent = 1 (unchanged). */
volume?: number;
+ /** The clip's own sound treatment — the kit builds the chain from it. */
+ sound?: ClipSound;
/** "fit" letterboxes (default); "fill" covers the region and crops. */
fit?: "fit" | "fill";
/** How far the picture zooms past the size its box fits it to (1 = none).
@@ -163,6 +173,7 @@ export interface ExportSpec {
muted: boolean;
/** Gain on the clip's own audio, 0..3; absent = 1 (unchanged). */
volume?: number;
+ sound?: ClipSound;
speed?: number;
/** Transition ramps, timeline seconds from this overlay's head/tail. On
* an upper track a fade is an alpha fade (the tracks beneath show
@@ -224,6 +235,7 @@ export interface ExportSpec {
fadeOut?: number;
/** Playback rate (detached-audio clips inherit their video clip's speed). */
speed?: number;
+ sound?: ClipSound;
/** Voiceover ducking: while this clip plays, every other sound drops to
* this gain (0..1). Ducking clips never duck each other. */
duck?: number;
@@ -1308,7 +1320,7 @@ export async function runExport(
}
if (!c.muted && !c.hidden && audioPresence.get(c.file)) {
const tempo = speed !== 1 ? `${atempoChain(speed)},` : "";
- const vol = (c.volume ?? 1) !== 1 ? `volume=${num(c.volume ?? 1)},` : "";
+ const vol = `${soundChain(c.sound)}${(c.volume ?? 1) !== 1 ? `volume=${num(c.volume ?? 1)},` : ""}`;
// The picture's fade edges carry the sound with them; zoom edges don't.
const afades =
(ahf > 0.01 ? `,afade=t=in:st=0:d=${num(ahf)}` : "") +
@@ -1426,8 +1438,9 @@ export async function runExport(
filters.push(
`[${inputIndex.get(src.file)!}:a]atrim=${num(from)}:${num(from + seconds * rate)},` +
`asetpts=PTS-STARTPTS,${rate !== 1 ? `${atempoChain(rate)},` : ""}` +
- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo` +
- `${(src.volume ?? 1) !== 1 ? `,volume=${num(src.volume ?? 1)}` : ""}${crossFilters([expr])},` +
+ `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` +
+ `${soundChain(src.sound)}` +
+ `${(src.volume ?? 1) !== 1 ? `volume=${num(src.volume ?? 1)},` : ""}anull${crossFilters([expr])},` +
`adelay=${Math.max(0, Math.round(at * 1000))}:all=1[${lab}]`
);
crossHandleLabels.push(lab);
@@ -1716,8 +1729,9 @@ export async function runExport(
const lab = `ovs${k}`;
filters.push(
`[${idx}:a]atrim=${num(oc.in - back * ospeed)}:${num(oc.out + ahead * ospeed)},` +
- `asetpts=PTS-STARTPTS,${tempo}${vol}${afades}` +
- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo` +
+ `asetpts=PTS-STARTPTS,${tempo}` +
+ `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` +
+ `${soundChain(oc.sound)}${vol}${afades}anull` +
`${crosses},adelay=${delayMs}:all=1[${lab}]`
);
overlaySoundLabels.push(lab);
@@ -1881,7 +1895,7 @@ export async function runExport(
filters.push(
`[${idx}:a]atrim=${num(a.in)}:${num(a.out)},asetpts=PTS-STARTPTS,${tempo}` +
`aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` +
- `volume=${num(a.volume)},` +
+ `${soundChain(a.sound)}volume=${num(a.volume)},` +
(fades.length ? fades.join(",") + "," : "") +
`adelay=${delayMs}:all=1[snd${k}]`
);