Open source CapCut alternative.
TypeScript
feat(site): a clip carries its own sound treatment [rebuild]
David Le91ba78d85684
diff --git a/site/packages/effects-kit/src/audioFx.test.ts b/site/packages/effects-kit/src/audioFx.test.tsindex 67765dce..650941c1 100644--- a/site/packages/effects-kit/src/audioFx.test.ts+++ b/site/packages/effects-kit/src/audioFx.test.ts@@ -5,6 +5,11 @@ import {audioFxBars,audioFxFilters,audioFxRecipe,+ normalizeSound,+ SOUND_EQ_BANDS,+ SOUND_PRESETS,+ soundFilters,+ soundRecipe,} from "./audioFx";import { ALL_EFFECT_IDS, EFFECT_IDS, EFFECT_LABELS } from "./effects";@@ -20,6 +25,8 @@ const LGPL_AUDIO_FILTERS = new Set(["equalizer","tremolo","acrusher",+ "acompressor",+ "alimiter","volume",]);@@ -93,6 +100,59 @@ describe("audio effect recipes", () => {});});+describe("clip sound", () => {+ test("a neutral treatment stores as absence", () => {+ expect(normalizeSound(undefined)).toBeUndefined();+ expect(normalizeSound({})).toBeUndefined();+ expect(normalizeSound({ eq: [0, 0, 0, 0, 0, 0, 0] })).toBeUndefined();+ expect(+ normalizeSound({ compressor: { threshold: -18, ratio: 1, attack: 10, release: 80 } })+ ).toBeUndefined();+ expect(normalizeSound({ limiter: { ceiling: 0 } })).toBeUndefined();+ expect(soundRecipe({})).toBeNull();+ expect(soundFilters(undefined)).toBeNull();+ });++ test("the equalizer keeps one value per band and clamps it", () => {+ const s = normalizeSound({ eq: [40, -40, 1] })!;+ expect(s.eq).toHaveLength(SOUND_EQ_BANDS.length);+ expect(s.eq![0]).toBe(12);+ expect(s.eq![1]).toBe(-12);+ expect(s.eq![2]).toBe(1);+ expect(s.eq![6]).toBe(0);+ // A flat band builds no section.+ expect(soundRecipe(s)!.bands).toHaveLength(3);+ });++ test("every preset spells a chain from filters the shipped ffmpeg has", () => {+ for (const p of SOUND_PRESETS) {+ const chain = soundFilters(p.sound);+ if (!p.sound) {+ expect(chain).toBeNull();+ continue;+ }+ expect(chain).toBeTruthy();+ for (const name of filterNamesOf(chain!)) {+ expect(LGPL_AUDIO_FILTERS.has(name)).toBe(true);+ }+ }+ });++ test("the voice preset runs a shelf, peaks, a shelf, then level control", () => {+ const chain = soundFilters(SOUND_PRESETS.find((p) => p.id === "clear-voice")!.sound)!;+ const names = filterNamesOf(chain);+ expect(names[0]).toBe("lowshelf");+ // The shelves carry the slope Web Audio's shelf nodes run at.+ expect(chain).toContain("lowshelf=f=100:width_type=s:w=1:g=2");+ expect(names[names.length - 3]).toBe("highshelf");+ expect(names[names.length - 2]).toBe("acompressor");+ expect(names[names.length - 1]).toBe("alimiter");+ // Thresholds go to ffmpeg as linear amplitude.+ expect(chain).toContain("acompressor=threshold=0.126:ratio=3:attack=10:release=80");+ expect(chain).toContain("alimiter=limit=0.891");+ });+});+describe("audio effect tile figures", () => {test("bars stay inside the tile whatever the effect and moment", () => {for (const id of [...AUDIO_EFFECT_IDS, null]) {diff --git a/site/packages/effects-kit/src/audioFx.ts b/site/packages/effects-kit/src/audioFx.tsindex 4732bbac..9ef79572 100644--- a/site/packages/effects-kit/src/audioFx.ts+++ b/site/packages/effects-kit/src/audioFx.ts@@ -64,13 +64,22 @@ export interface AudioBandSpec {db?: number;}+/** Downward compression: everything over `thresholdDb` is pulled toward it+ * by `ratio`, reacting over `attackMs` and letting go over `releaseMs`. */+export interface AudioCompressorSpec {+ thresholdDb: number;+ ratio: number;+ attackMs: number;+ releaseMs: number;+}+/*** What an audio effect does, in the terms both renderers can build.** The delay taps are a finite echo: the dry signal plus each tap's delayed* copy at its own gain, which is exactly what ffmpeg's `aecho` computes, so- * one description drives both. Bands, tremolo, bit crush and makeup gain run- * after the taps, in that order.+ * one description drives both. Bands, tremolo, bit crush, the compressor, the+ * limiter and makeup gain run after the taps, in that order.*/export interface AudioFxRecipe {/** Gain on the untreated signal that the taps are added to. */@@ -83,10 +92,163 @@ export interface AudioFxRecipe {tremolo?: { hz: number; depth: number };/** Bit-depth reduction, and how much of the crushed signal is heard. */crush?: { bits: number; mix: number };+ /** Level control after the filters. */+ compressor?: AudioCompressorSpec;+ /** A ceiling in dBFS the signal is held under. */+ limiter?: { ceilingDb: number };/** Makeup gain on the finished chain; absent = 1. */gain?: number;}+/* ------------------------------------------------------- clip sound */++/**+ * A clip's own sound treatment — the Inspector's Sound quality section.+ *+ * Where an effect element treats the mix under a window, this rides one clip+ * and runs on that clip's sound alone, before its fades and the duck. Every+ * stage is optional and a fully neutral treatment is stored as absence, so a+ * clip that was never touched carries nothing.+ */+export interface ClipSound {+ /** Gain in dB per `SOUND_EQ_BANDS` entry, in that order. Absent = flat. */+ eq?: number[];+ /** dB, ratio (:1), attack ms, release ms. */+ compressor?: { threshold: number; ratio: number; attack: number; release: number };+ /** Ceiling in dBFS. */+ limiter?: { ceiling: number };+}++/** The fixed band layout of the clip equalizer: a shelf at each end and+ * peaks between, spaced where speech lives. Fixed frequencies keep a preset+ * to seven numbers, which is what the sliders, the tool schema and a saved+ * preset all carry. */+export const SOUND_EQ_BANDS: {+ hz: number;+ type: AudioBandSpec["type"];+ label: string;+ /** The region of the spectrum the band sits in, named the way a mixer+ * names it. */+ group: "Low" | "Mids" | "Presence" | "Highs";+}[] = [+ { hz: 100, type: "lowshelf", label: "100 Hz", group: "Low" },+ { hz: 200, type: "peaking", label: "200 Hz", group: "Low" },+ { hz: 400, type: "peaking", label: "400 Hz", group: "Mids" },+ { hz: 1000, type: "peaking", label: "1 kHz", group: "Mids" },+ { hz: 3000, type: "peaking", label: "3 kHz", group: "Presence" },+ { hz: 5000, type: "peaking", label: "5 kHz", group: "Presence" },+ { hz: 10000, type: "highshelf", label: "10 kHz", group: "Highs" },+];++/** A peak's width: about an octave and a half, wide enough that two adjacent+ * bands moved together read as one shape. */+const SOUND_EQ_Q = 1.0;++export const SOUND_EQ_DB_RANGE = 12;+/** What the stage sliders offer. A ratio of 1:1 and a ceiling of 0 dBFS are+ * the stage doing nothing, and `normalizeSound` stores them as absence, so+ * the ranges stop short of both — dragging to an endpoint shapes the sound+ * rather than deleting the stage mid-gesture. Switching the section off is+ * how a stage goes away. */+export const SOUND_COMPRESSOR_RANGE = {+ threshold: { min: -60, max: -1 },+ ratio: { min: 1.5, max: 20 },+ attack: { min: 0.1, max: 200 },+ release: { min: 5, max: 1000 },+};+export const SOUND_LIMITER_RANGE = { ceiling: { min: -20, max: -0.1 } };++/** The settings a compressor or limiter opens at when its switch is turned+ * on — the voice preset's numbers, since that is what the switch is for. */+export const SOUND_COMPRESSOR_DEFAULT = { threshold: -18, ratio: 3, attack: 10, release: 80 };+export const SOUND_LIMITER_DEFAULT = { ceiling: -1 };++/** The treatments shipped with the editor. `sound` undefined is the flat one. */+export const SOUND_PRESETS: { id: string; name: string; sound: ClipSound | undefined }[] = [+ { id: "flat", name: "Flat", sound: undefined },+ {+ id: "clear-voice",+ name: "Clear voice",+ sound: {+ eq: [2, 1, -2, -1, 1.5, 1, -1],+ compressor: SOUND_COMPRESSOR_DEFAULT,+ limiter: SOUND_LIMITER_DEFAULT,+ },+ },+];++export const soundPresetById = (id: string) => SOUND_PRESETS.find((p) => p.id === id);++const clampTo = (v: number, r: { min: number; max: number }) =>+ Math.max(r.min, Math.min(r.max, v));++/**+ * A sound with every stage clamped into range and every neutral stage+ * dropped, or undefined when nothing is left: the one shape the doc stores.+ * A flat equalizer, a compressor at 1:1 and a limiter at 0 dB all do nothing,+ * so they are not kept.+ */+export function normalizeSound(sound: ClipSound | undefined | null): ClipSound | undefined {+ if (!sound) return undefined;+ const out: ClipSound = {};+ if (sound.eq) {+ const eq = SOUND_EQ_BANDS.map((_, i) => {+ const v = Number(sound.eq![i] ?? 0);+ return Number.isFinite(v) ? clampTo(v, { min: -SOUND_EQ_DB_RANGE, max: SOUND_EQ_DB_RANGE }) : 0;+ });+ if (eq.some((v) => Math.abs(v) > 1e-3)) out.eq = eq;+ }+ if (sound.compressor) {+ const c = sound.compressor;+ const r = SOUND_COMPRESSOR_RANGE;+ if (Number.isFinite(c.ratio) && c.ratio > 1 + 1e-3) {+ out.compressor = {+ threshold: clampTo(c.threshold, r.threshold),+ ratio: clampTo(c.ratio, r.ratio),+ attack: clampTo(c.attack, r.attack),+ release: clampTo(c.release, r.release),+ };+ }+ }+ if (sound.limiter) {+ const raw = sound.limiter.ceiling;+ if (Number.isFinite(raw) && raw < -1e-3) {+ out.limiter = { ceiling: clampTo(raw, SOUND_LIMITER_RANGE.ceiling) };+ }+ }+ return out.eq || out.compressor || out.limiter ? out : undefined;+}++/** The recipe a clip's sound builds, or null when it is neutral. */+export function soundRecipe(sound: ClipSound | undefined | null): AudioFxRecipe | null {+ const s = normalizeSound(sound);+ if (!s) return null;+ const bands: AudioBandSpec[] = [];+ if (s.eq) {+ SOUND_EQ_BANDS.forEach((b, i) => {+ const db = s.eq![i];+ if (Math.abs(db) < 1e-3) return;+ bands.push({ type: b.type, hz: b.hz, db, ...(b.type === "peaking" ? { q: SOUND_EQ_Q } : {}) });+ });+ }+ return {+ dry: 1,+ taps: [],+ bands,+ ...(s.compressor+ ? {+ compressor: {+ thresholdDb: s.compressor.threshold,+ ratio: s.compressor.ratio,+ attackMs: s.compressor.attack,+ releaseMs: s.compressor.release,+ },+ }+ : {}),+ ...(s.limiter ? { limiter: { ceilingDb: s.limiter.ceiling } } : {}),+ };+}+/** How long the treated signal takes to come in and go out at an effect* window's edges, in seconds. Short enough to read as an instant switch, long* enough that the switch cannot click. */@@ -221,15 +383,21 @@ function bandFilter(b: AudioBandSpec): string {return `lowpass=f=${fmt(b.hz)}:width_type=q:w=${fmt(q)}`;case "highpass":return `highpass=f=${fmt(b.hz)}:width_type=q:w=${fmt(q)}`;+ // Web Audio's shelf nodes ignore Q and run at slope S=1; ffmpeg defaults+ // to a Q width of 0.5, which is a different curve at the corner. Spelling+ // the slope out is what keeps the preview and the export on one sound.case "lowshelf":- return `lowshelf=f=${fmt(b.hz)}:g=${fmt(b.db ?? 0)}`;+ return `lowshelf=f=${fmt(b.hz)}:width_type=s:w=1:g=${fmt(b.db ?? 0)}`;case "highshelf":- return `highshelf=f=${fmt(b.hz)}:g=${fmt(b.db ?? 0)}`;+ return `highshelf=f=${fmt(b.hz)}:width_type=s:w=1:g=${fmt(b.db ?? 0)}`;case "peaking":return `equalizer=f=${fmt(b.hz)}:width_type=q:w=${fmt(q)}:g=${fmt(b.db ?? 0)}`;}}+/** dBFS as the linear amplitude ffmpeg's dynamics filters take. */+const linear = (db: number) => Math.pow(10, db / 20);+/*** The effect as one ffmpeg audio chain — comma-joined filters, no stream* labels, so the caller splices it into whatever window it runs over. Null for@@ -238,7 +406,17 @@ function bandFilter(b: AudioBandSpec): string {*/export function audioFxFilters(effect: string, amount?: number): string | null {const r = audioFxRecipe(effect, amount);- if (!r) return null;+ return r ? recipeFilters(r) : null;+}++/** A clip's sound as one ffmpeg chain, or null when it is neutral. */+export function soundFilters(sound: ClipSound | undefined | null): string | null {+ const r = soundRecipe(sound);+ return r ? recipeFilters(r) : null;+}++/** One recipe spelled as ffmpeg filters, in the order the graph builds them. */+export function recipeFilters(r: AudioFxRecipe): string | null {const steps: string[] = [];if (r.taps.length > 0) {// aecho computes in_gain*dry + out_gain*Σ decay_i·delayed_i, which is the@@ -256,6 +434,20 @@ export function audioFxFilters(effect: string, amount?: number): string | null {for (const b of r.bands) steps.push(bandFilter(b));if (r.tremolo) steps.push(`tremolo=f=${fmt(r.tremolo.hz)}:d=${fmt(r.tremolo.depth)}`);if (r.crush) steps.push(`acrusher=bits=${fmt(r.crush.bits)}:mix=${fmt(r.crush.mix)}:mode=lin`);+ if (r.compressor) {+ const c = r.compressor;+ // acompressor's threshold is linear amplitude; attack and release are ms.+ // No makeup, so the two sides agree: the Web Audio node adds none either.+ steps.push(+ `acompressor=threshold=${fmt(linear(c.thresholdDb))}:ratio=${fmt(c.ratio)}` ++ `:attack=${fmt(c.attackMs)}:release=${fmt(c.releaseMs)}:knee=2:makeup=1`+ );+ }+ if (r.limiter) {+ // A lookahead of a few milliseconds, the same as the browser's dynamics+ // node carries, and no automatic level: the ceiling is the whole point.+ steps.push(`alimiter=limit=${fmt(linear(r.limiter.ceilingDb))}:attack=5:release=50:level=false`);+ }if (r.gain !== undefined && r.gain !== 1) steps.push(`volume=${fmt(r.gain)}`);return steps.length > 0 ? steps.join(",") : null;}@@ -362,6 +554,31 @@ export function buildAudioFx(node = mixed;}+ if (recipe.compressor) {+ const c = recipe.compressor;+ const comp = ctx.createDynamicsCompressor();+ comp.threshold.value = c.thresholdDb;+ comp.ratio.value = Math.max(1, Math.min(20, c.ratio));+ comp.knee.value = 2;+ comp.attack.value = c.attackMs / 1000;+ comp.release.value = c.releaseMs / 1000;+ node.connect(comp);+ node = comp;+ }++ if (recipe.limiter) {+ // Web Audio has no limiter node; a hard-knee compressor at the steepest+ // ratio the node takes, reacting at once, holds the ceiling the same way.+ const lim = ctx.createDynamicsCompressor();+ lim.threshold.value = recipe.limiter.ceilingDb;+ lim.ratio.value = 20;+ lim.knee.value = 0;+ lim.attack.value = 0.001;+ lim.release.value = 0.05;+ node.connect(lim);+ node = lim;+ }+if (recipe.gain !== undefined && recipe.gain !== 1) {const makeup = ctx.createGain();makeup.gain.value = recipe.gain;diff --git a/site/packages/effects-kit/src/audioFxAudible.test.ts b/site/packages/effects-kit/src/audioFxAudible.test.tsindex c7fd5682..7905d257 100644--- a/site/packages/effects-kit/src/audioFxAudible.test.ts+++ b/site/packages/effects-kit/src/audioFxAudible.test.ts@@ -1,6 +1,13 @@import { describe, expect, test } from "bun:test";import { AudioBuffer, OfflineAudioContext } from "node-web-audio-api";-import { AUDIO_EFFECT_IDS, audioFxRecipe, buildAudioFx } from "./audioFx";+import {+ AUDIO_EFFECT_IDS,+ audioFxRecipe,+ buildAudioFx,+ SOUND_PRESETS,+ soundRecipe,+ type AudioFxRecipe,+} from "./audioFx";/*** What the recipes have to be: audible.@@ -49,12 +56,13 @@ function voice(): AudioBuffer {const SOURCE = voice();-async function render(effect: string | null): Promise<Float32Array> {+async function render(effect: string | AudioFxRecipe | null): Promise<Float32Array> {const ctx = new OfflineAudioContext(1, Math.round(SECONDS * RATE), RATE);const node = ctx.createBufferSource();node.buffer = SOURCE;if (effect) {- const chain = buildAudioFx(ctx, audioFxRecipe(effect)!, 0);+ const recipe = typeof effect === "string" ? audioFxRecipe(effect)! : effect;+ const chain = buildAudioFx(ctx, recipe, 0);node.connect(chain.input);chain.output.connect(ctx.destination);} else {@@ -148,6 +156,22 @@ describe("every audio effect is audible at the strength a drop places", () => {expect(swing(await render("wobble")) - swing(dry)).toBeGreaterThan(4);});+ test("the voice preset is audible and holds its ceiling", async () => {+ const dry = await render(null);+ const recipe = soundRecipe(SOUND_PRESETS.find((p) => p.id === "clear-voice")!.sound)!;+ const wet = await render(recipe);+ expect(change(wet, dry)).toBeGreaterThan(-26);+ // The limiter's ceiling is −1 dBFS; nothing gets past it by more than the+ // node's own overshoot.+ let peak = 0;+ for (const v of wet) peak = Math.max(peak, Math.abs(v));+ expect(dB(peak)).toBeLessThan(0);+ // A compressor flattens the syllables: the loud ones come down toward+ // the quiet ones, so the level swing narrows.+ const flat = await render({ ...recipe, limiter: undefined, bands: [] });+ expect(swing(flat)).toBeLessThan(swing(dry));+ });+test("the rooms are still sounding after the sound that made them stops", async () => {// A burst, then silence: a reverb tail and an echo repeat both live in// the silence, which is the part a spectrum of the whole take cannot show.diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.tsindex 77cf5ebd..cca062af 100644--- a/site/src/cut/hooks/usePlayback.ts+++ b/site/src/cut/hooks/usePlayback.ts@@ -142,6 +142,7 @@ const voiceSpan = (sp: ClipSpan) => {in: sp.clip.in - sp.soundBack * speed,out: sp.clip.out + sp.soundAhead * speed,speed,+ sound: sp.clip.sound,};};@@ -549,6 +550,7 @@ class Engine {out: a.out,speed,gain: a.volume * g * dg,+ sound: a.sound,});}return out;diff --git a/site/src/cut/lib/audioMix.ts b/site/src/cut/lib/audioMix.tsindex b2d8c519..21659b34 100644--- a/site/src/cut/lib/audioMix.ts+++ b/site/src/cut/lib/audioMix.ts@@ -17,7 +17,13 @@* words that the exported audio says at another time.*/-import { AUDIO_FX_RAMP, audioFxRecipe, buildAudioFx } from "@donkeycut/effects-kit";+import {+ AUDIO_FX_RAMP,+ audioFxRecipe,+ buildAudioFx,+ soundRecipe,+ type ClipSound,+} from "@donkeycut/effects-kit";import { decodeAudioSpan } from "./mediaRead";import { timeStretch } from "./timeStretch";@@ -31,6 +37,8 @@ export interface MixClip {speed?: number;/** Gain on the clip's own audio; absent = 1. */volume?: number;+ /** The clip's own treatment, run on its sound before the fades. */+ sound?: ClipSound;/** Cross-dissolve overlap into the next clip, timeline seconds. */transition?: number;/** Half the cross dissolve into the next clip, timeline seconds: the two@@ -64,6 +72,7 @@ export interface MixItem {muted?: boolean;fadeIn?: number;fadeOut?: number;+ sound?: ClipSound;/** While this item is audible, everything else drops to this gain. */duck?: number;}@@ -377,18 +386,29 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB// Each buffer holds exactly the span its entry plays, already fitted to the// time that entry occupies, so it starts at its own zero and runs at 1.+ // The entry's own treatment runs on its sound alone, between the source and+ // the gain that carries its level, fades and crossings — the order the+ // preview's voice and the ffmpeg stanza both keep.const play = (buf: AudioBuffer,at: number,duration: number,into: AudioNode,+ sound: ClipSound | undefined,shape: (gain: GainNode) => void) => {const src = ctx.createBufferSource();src.buffer = buf;const gain = ctx.createGain();shape(gain);- src.connect(gain);+ const recipe = soundRecipe(sound);+ if (recipe) {+ const fx = buildAudioFx(ctx, recipe, Math.max(0, at));+ src.connect(fx.input);+ fx.output.connect(gain);+ } else {+ src.connect(gain);+ }gain.connect(into);src.start(Math.max(0, at), 0, Math.max(0, duration));};@@ -400,7 +420,7 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioBif (!buf) continue; // muted, silent, or a gap spacer: only shapes timeconst from = g.at - g.back;const end = g.at + g.dur;- play(fitted(key, buf, speedOf(c.speed)), from, g.dur + g.back + g.ahead, ducked, (gain) => {+ play(fitted(key, buf, speedOf(c.speed)), from, g.dur + g.back + g.ahead, ducked, c.sound, (gain) => {const level = Math.max(0, c.volume ?? 1);gain.gain.value = level;// A crossing and a fade never share an edge — the cut carries one@@ -431,7 +451,7 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB// A voiceover that ducks everything else is not itself ducked; it still// lands on the bus, so the effect elements treat it with the rest.const into = a.duck !== undefined && a.duck < 1 ? bus : ducked;- play(fitted(key, buf, speedOf(a.speed)), a.start - back, dur + back + ahead, into, (gain) => {+ play(fitted(key, buf, speedOf(a.speed)), a.start - back, dur + back + ahead, into, a.sound, (gain) => {const level = Math.max(0, a.volume);gain.gain.value = level;const fIn = Math.min(a.fadeIn ?? 0, dur);diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.tsindex a8870799..906ab4c9 100644--- a/site/src/cut/lib/exportClient.ts+++ b/site/src/cut/lib/exportClient.ts@@ -594,6 +594,7 @@ export async function buildExportPayload(out: sp.clip.out,muted: sp.clip.muted,volume: sp.clip.volume ?? 1,+ sound: sp.clip.sound,fit: sp.clip.fit ?? "fit",zoom: clipZoom(sp.clip),panX: sp.clip.panX ?? 0,@@ -801,6 +802,7 @@ export async function buildExportPayload(panY: c.panY ?? 0,muted: c.muted,volume: c.volume,+ sound: c.sound,speed: c.speed,image: assetById.get(c.assetId)!.type === "image",grade: normalizeGrade(c.grade),@@ -899,6 +901,7 @@ export async function buildExportPayload(fadeIn: a.fadeIn ?? 0,fadeOut: a.fadeOut ?? 0,speed: a.speed,+ sound: a.sound,duck: a.duck,}));diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.tsindex 2b4bb9be..a0ee5038 100644--- a/site/src/cut/lib/exportRender.ts+++ b/site/src/cut/lib/exportRender.ts@@ -509,6 +509,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strinstart: sp.start,volume: sp.clip.volume ?? 1,speed: sp.clip.speed,+ sound: sp.clip.sound,muted: sp.clip.muted || assetIsSilent(sp.asset),fadeIn: ramps[i].head,fadeOut: ramps[i].tail,@@ -531,6 +532,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strinspeed: a.speed,fadeIn: a.fadeIn,fadeOut: a.fadeOut,+ sound: a.sound,duck: a.duck,});}@@ -548,6 +550,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strinmuted: sp.clip.muted || !!sp.clip.hidden || assetIsSilent(sp.asset),speed: sp.clip.speed,volume: sp.clip.volume,+ sound: sp.clip.sound,transition: sp.transitionOut,soundCross: sp.soundOut,soundBack: sp.soundBack,diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.tsindex ab12186a..073a263d 100644--- a/site/src/cut/lib/previewMixer.ts+++ b/site/src/cut/lib/previewMixer.ts@@ -35,7 +35,13 @@* chain runs them in.*/-import { audioFxRecipe, buildAudioFx, type AudioFxNodes } from "@donkeycut/effects-kit";+import {+ audioFxRecipe,+ buildAudioFx,+ soundRecipe,+ type AudioFxNodes,+ type ClipSound,+} from "@donkeycut/effects-kit";import type { WrappedAudioBuffer } from "mediabunny";import { assembleAudio, decodeAudioSpan, openAudioWalk, type AudioWalk } from "./mediaRead";import { timeStretch } from "./timeStretch";@@ -80,6 +86,9 @@ export interface Voice {speed: number;/** Everything the frame plan says about how loud it is right now. */gain: number;+ /** The clip's own treatment, run on this voice alone before it joins the+ * bus. */+ sound?: ClipSound;}interface Scheduled {@@ -94,7 +103,18 @@ interface LiveVoice {in: number;out: number;start: number;+ /** Where this voice's scheduled windows land. The clip's own treatment+ * runs between here and the gain, so it hears the clip at its recorded+ * level — the same place the fold and the ffmpeg stanza run it, ahead of+ * the level, fades and duck the gain carries. */+ head: GainNode;gain: GainNode;+ /** The treatment's chain, and what it was built from, so a frame that+ * changes nothing about it touches no nodes. Rebuilt in place when the+ * treatment changes: the windows already scheduled play on through the+ * new chain, since they are connected to the head rather than to it. */+ fx: AudioFxNodes | null;+ soundKey: string;/** Windows already handed to the context, oldest first. */windows: Scheduled[];/** Timeline time everything scheduled so far runs out at. */@@ -553,6 +573,7 @@ export class PreviewMixer {const live = this.voices.get(v.id) ?? this.open(v, t);if (!live) continue;live.gain.gain.value = Math.max(0, Math.min(3, v.gain));+ this.treat(live, v.sound);// A read that hangs holds the voice as surely as one that fails, and it// holds it quietly: nothing else would ever notice, and the voice would// wait on it for the rest of the play.@@ -611,6 +632,7 @@ export class PreviewMixer {const live = this.voices.get(v.id) ?? this.open(v, t);if (!live) continue;live.gain.gain.value = 0;+ this.treat(live, v.sound);const from = Math.max(v.start, t);if (live.scheduled !== from) this.reaim(live, from);const source = live.in + (from - live.start) * live.speed;@@ -678,13 +700,18 @@ export class PreviewMixer {const gain = ctx.createGain();gain.gain.value = 0;gain.connect(this.bus);+ const head = ctx.createGain();+ head.connect(gain);const live: LiveVoice = {url: v.url,speed: v.speed,in: v.in,out: v.out,start: v.start,+ head,gain,+ fx: null,+ soundKey: "",windows: [],// Scheduling begins where the playhead is: a voice opened mid-clip —// playback started inside it, or the clip edited while audible — decodes@@ -706,7 +733,43 @@ export class PreviewMixer {private release(live: LiveVoice): void {this.stopWindows(live);this.moved(live);+ live.head.disconnect();live.gain.disconnect();+ if (live.fx) {+ live.fx.input.disconnect();+ live.fx.output.disconnect();+ live.fx = null;+ }+ }++ /**+ * Put the clip's sound treatment between the voice's gain and the bus, or+ * take it out when the clip has none. Keyed on the treatment itself, so+ * every frame that leaves it alone costs a string compare and nothing+ * else, and a slider drag rebuilds a chain a few nodes long.+ */+ private treat(live: LiveVoice, sound: ClipSound | undefined): void {+ const ctx = this.ctx;+ const bus = this.bus;+ if (!ctx || !bus) return;+ const key = sound ? JSON.stringify(sound) : "";+ if (key === live.soundKey) return;+ live.soundKey = key;+ live.head.disconnect();+ if (live.fx) {+ live.fx.input.disconnect();+ live.fx.output.disconnect();+ live.fx = null;+ }+ const recipe = soundRecipe(sound);+ if (!recipe) {+ live.head.connect(live.gain);+ return;+ }+ const fx = buildAudioFx(ctx, recipe, ctx.currentTime);+ live.head.connect(fx.input);+ fx.output.connect(live.gain);+ live.fx = fx;}private stopVoices(): void {@@ -794,7 +857,7 @@ export class PreviewMixer {}const node = this.ctx.createBufferSource();node.buffer = buffer;- node.connect(live.gain);+ node.connect(live.head);const at = this.anchor.ctx + (from - this.anchor.timeline);const now = this.ctx.currentTime;if (at >= now) {diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex bb7809e4..21284ab3 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -3640,20 +3640,20 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {if (mi == null) continue;// Track-0 clips re-materialize onto track 0 (asClip), so a template// stands up its own video instead of an empty timeline.- layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, track: 1, asClip: true });+ layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, sound: sp.clip.sound, track: 1, asClip: true });} else {const c = s.clips.find((x) => x.id === sel.id);if (!c) continue;const mi = mediaFor(c.assetId);if (mi == null) continue;- layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, track: c.track + 1 });+ layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, sound: c.sound, track: c.track + 1 });}} else if (sel.kind === "audio") {const c = s.audioClips.find((x) => x.id === sel.id);if (!c) continue;const mi = mediaFor(c.assetId);if (mi == null) continue;- audio.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, volume: c.volume, fadeIn: c.fadeIn, fadeOut: c.fadeOut, speed: c.speed, duck: c.duck, lane: c.lane });+ audio.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, volume: c.volume, fadeIn: c.fadeIn, fadeOut: c.fadeOut, speed: c.speed, sound: c.sound, duck: c.duck, lane: c.lane });} else if (sel.kind === "overlay") {const o = s.overlays.find((x) => x.id === sel.id);// Asset-backed stickers stay out: a template copies only the media@@ -3783,6 +3783,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {muted: l.muted,...templateFraming(l),...(l.speed ? { speed: l.speed } : {}),+ ...(l.sound ? { sound: l.sound } : {}),}));const topTrack = Math.max(0, ...overlayLayers(get().clips).map((c) => c.track));// Template layers store `track` as the source track + 1 (so a track-1@@ -3800,6 +3801,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {muted: l.muted,...templateFraming(l),...(l.speed ? { speed: l.speed } : {}),+ ...(l.sound ? { sound: l.sound } : {}),}));const newAudio: AudioClip[] = template.audio.filter((a) => assetIds[a.media])@@ -3813,6 +3815,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {...(a.fadeIn ? { fadeIn: a.fadeIn } : {}),...(a.fadeOut ? { fadeOut: a.fadeOut } : {}),...(a.speed ? { speed: a.speed } : {}),+ ...(a.sound ? { sound: a.sound } : {}),...(a.duck !== undefined && a.duck < 1 ? { duck: a.duck } : {}),...(a.lane ? { lane: a.lane } : {}),}));diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.tsindex e32e086c..72881247 100644--- a/site/src/cut/lib/types.ts+++ b/site/src/cut/lib/types.ts@@ -5,6 +5,7 @@ import {stampOverlayKinds,stripDefaultOverlayKinds,type ClipRemoval,+ type ClipSound,type ColorGrade,type EffectOverlay,type LookStyle,@@ -508,6 +509,9 @@ export interface VideoClip {muted: boolean;/** Gain on the clip's own audio, 0..3; absent = 1 (unchanged). */volume?: number;+ /** The clip's own sound treatment — equalizer, compressor, limiter — run+ * on its sound alone, before its fades; absent = untouched. */+ sound?: ClipSound;/** How the clip meets its region: letterboxed ("fit", default) or scaled to* cover it ("fill", cropping the overflow). */fit?: "fit" | "fill";@@ -576,7 +580,7 @@ export interface VideoClip {// Color grading (the dual-renderer math) lives in the effects kit; the model// types and ranges re-export here so doc-model consumers keep one import.export { GRADE_BASIC_FIELDS, GRADE_HUE_MAX, GRADE_MAX, HSL_BANDS } from "@donkeycut/effects-kit";-export type { ColorGrade, GradePresetRef, HslBand } from "@donkeycut/effects-kit";+export type { ClipSound, ColorGrade, GradePresetRef, HslBand } from "@donkeycut/effects-kit";// Background removal follows the same split: math and model in the kit, the// doc-model types re-exported here.@@ -969,6 +973,8 @@ export interface AudioClip {volume: number; // 0..3fadeIn?: number; // seconds, ramp up from the clip startfadeOut?: number; // seconds, ramp down into the clip end+ /** Equalizer, compressor, limiter on this clip's sound; absent = untouched. */+ sound?: ClipSound;/** Muted from the final mix but kept on the timeline (grayed). */hidden?: boolean;/** Playback rate, default 1 (absent). Set only when audio was detached from@@ -1013,6 +1019,7 @@ export interface TemplateLayer {opacity?: number;muted: boolean;speed?: number;+ sound?: ClipSound;track: number;/** Came from video track 0 — re-materializes as a timeline clip, not an* overlay, so a template stands up its own footage. */@@ -1027,6 +1034,7 @@ export interface TemplateAudio {fadeIn?: number;fadeOut?: number;speed?: number;+ sound?: ClipSound;duck?: number;lane?: number;}@@ -1041,6 +1049,10 @@ export interface LibraryTemplate {audio: TemplateAudio[];texts: Overlay[];cues: SubtitleCue[];+ /** A saved sound treatment: a template carrying only this is a sound+ * preset (see soundPresets.ts), listed in the audio inspector and kept+ * off the template shelf. */+ sound?: ClipSound;}/** What the client sends to save a selection (media are project file names). */export type TemplateSaveInput = Omit<LibraryTemplate, "id" | "addedAt">;diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.tsindex 4b1f3f09..bf2af942 100644--- a/site/src/cut/server/exportPipeline.test.ts+++ b/site/src/cut/server/exportPipeline.test.ts@@ -321,6 +321,53 @@ describe("export filtergraph timebases", () => {});});+describe("clip sound in the filtergraph", () => {+ const sound = {+ eq: [2, 1, -2, -1, 1.5, 1, -1],+ compressor: { threshold: -18, ratio: 3, attack: 10, release: 80 },+ limiter: { ceiling: -1 },+ };++ test("a treated track-0 clip runs its chain before its level, pad and fades", async () => {+ const g = await graphFor({+ clips: [clip("a.mp4", { sound, volume: 0.8, animOut: { style: "fade", seconds: 0.4 } })],+ });+ const stanza = g.find((f) => f.startsWith("[0:a]"))!;+ expect(stanza).toBeTruthy();+ const at = (s: string) => stanza.indexOf(s);+ expect(at("lowshelf=f=100")).toBeGreaterThan(at("aformat="));+ expect(at("acompressor=")).toBeGreaterThan(at("highshelf="));+ expect(at("alimiter=")).toBeGreaterThan(at("acompressor="));+ expect(at("volume=0.8")).toBeGreaterThan(at("alimiter="));+ expect(at("apad=")).toBeGreaterThan(at("volume=0.8"));+ expect(at("afade=t=out")).toBeGreaterThan(at("apad="));+ });++ test("a treated soundtrack clip and overlay clip carry the same chain", async () => {+ const g = await graphFor({+ clips: [clip("a.mp4")],+ overlayVideos: [+ { file: "ov.mp4", in: 0, out: 2, start: 1, track: 1, muted: false, sound },+ ],+ audio: [{ file: "music.mp3", in: 0, out: 5, start: 0, volume: 0.8, sound }],+ });+ const treated = g.filter((f) => f.includes("alimiter="));+ expect(treated).toHaveLength(2);+ // Every stanza treats the resampled stream, so one clip's treatment+ // sounds the same whichever list it came from.+ for (const f of treated) {+ expect(f.indexOf("alimiter=")).toBeLessThan(f.indexOf("adelay="));+ expect(f.indexOf("aformat=")).toBeLessThan(f.indexOf("lowshelf="));+ }+ });++ test("an untreated clip spells no dynamics filter", async () => {+ const g = await graphFor({ clips: [clip("a.mp4", { sound: { eq: [0, 0, 0, 0, 0, 0, 0] } })] });+ expect(g.join(";")).not.toContain("acompressor=");+ expect(g.join(";")).not.toContain("equalizer=");+ });+});+describe("clip masks in the filtergraph", () => {test("a masked track-0 clip trims onto a black base and keeps the join sound", async () => {const g = await graphFor({diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.tsindex f41d057e..c8922200 100644--- a/site/src/cut/server/exportPipeline.ts+++ b/site/src/cut/server/exportPipeline.ts@@ -4,7 +4,7 @@ import path from "node:path";import { atempoChain, hasStream, num, videoColorInfo } from "./util";import { assertGraphSafe, fexpr } from "./filterGraph";import { CLIP_MAX_ZOOM, projectFadeSeconds, regionPx, TRANSITION_XFADE, TRANSITION_ZOOM, type ColorGrade, type TransitionStyle } from "../lib/types";-import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, shortestTurn, sortedKeys, type OverlayKey } from "@donkeycut/effects-kit";+import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, shortestTurn, sortedKeys, soundFilters, type ClipSound, type OverlayKey } from "@donkeycut/effects-kit";// The render pipeline itself: spec in, finished mp4 out. Shared by the local// engine's job registry (jobs.ts) and the cloud render worker, which stage@@ -40,6 +40,14 @@ function crossFilters(exprs: string[]): string {return `,asetnsamples=n=256:p=0,volume=volume=${fexpr(live.join("*"))}:eval=frame`;}+/** A clip's own sound treatment as a chain fragment ending in a comma, or+ * nothing: spliced into the clip's stanza ahead of its level, fades and+ * crossings — where the preview's voice and the browser fold run it too. */+const soundChain = (sound: ClipSound | undefined): string => {+ const chain = soundFilters(sound);+ return chain ? `${chain},` : "";+};+export interface ExportSpec {projectId: string;/** Where the render lands instead of a stamped file in exports/: "hls" is the@@ -68,6 +76,8 @@ export interface ExportSpec {muted: boolean;/** Gain on the clip's own audio, 0..3; absent = 1 (unchanged). */volume?: number;+ /** The clip's own sound treatment — the kit builds the chain from it. */+ sound?: ClipSound;/** "fit" letterboxes (default); "fill" covers the region and crops. */fit?: "fit" | "fill";/** How far the picture zooms past the size its box fits it to (1 = none).@@ -163,6 +173,7 @@ export interface ExportSpec {muted: boolean;/** Gain on the clip's own audio, 0..3; absent = 1 (unchanged). */volume?: number;+ sound?: ClipSound;speed?: number;/** Transition ramps, timeline seconds from this overlay's head/tail. On* an upper track a fade is an alpha fade (the tracks beneath show@@ -224,6 +235,7 @@ export interface ExportSpec {fadeOut?: number;/** Playback rate (detached-audio clips inherit their video clip's speed). */speed?: number;+ sound?: ClipSound;/** Voiceover ducking: while this clip plays, every other sound drops to* this gain (0..1). Ducking clips never duck each other. */duck?: number;@@ -1308,7 +1320,7 @@ export async function runExport(}if (!c.muted && !c.hidden && audioPresence.get(c.file)) {const tempo = speed !== 1 ? `${atempoChain(speed)},` : "";- const vol = (c.volume ?? 1) !== 1 ? `volume=${num(c.volume ?? 1)},` : "";+ const vol = `${soundChain(c.sound)}${(c.volume ?? 1) !== 1 ? `volume=${num(c.volume ?? 1)},` : ""}`;// The picture's fade edges carry the sound with them; zoom edges don't.const afades =(ahf > 0.01 ? `,afade=t=in:st=0:d=${num(ahf)}` : "") +@@ -1426,8 +1438,9 @@ export async function runExport(filters.push(`[${inputIndex.get(src.file)!}:a]atrim=${num(from)}:${num(from + seconds * rate)},` +`asetpts=PTS-STARTPTS,${rate !== 1 ? `${atempoChain(rate)},` : ""}` +- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo` +- `${(src.volume ?? 1) !== 1 ? `,volume=${num(src.volume ?? 1)}` : ""}${crossFilters([expr])},` ++ `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` ++ `${soundChain(src.sound)}` ++ `${(src.volume ?? 1) !== 1 ? `volume=${num(src.volume ?? 1)},` : ""}anull${crossFilters([expr])},` +`adelay=${Math.max(0, Math.round(at * 1000))}:all=1[${lab}]`);crossHandleLabels.push(lab);@@ -1716,8 +1729,9 @@ export async function runExport(const lab = `ovs${k}`;filters.push(`[${idx}:a]atrim=${num(oc.in - back * ospeed)}:${num(oc.out + ahead * ospeed)},` +- `asetpts=PTS-STARTPTS,${tempo}${vol}${afades}` +- `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo` ++ `asetpts=PTS-STARTPTS,${tempo}` ++ `aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` ++ `${soundChain(oc.sound)}${vol}${afades}anull` +`${crosses},adelay=${delayMs}:all=1[${lab}]`);overlaySoundLabels.push(lab);@@ -1881,7 +1895,7 @@ export async function runExport(filters.push(`[${idx}:a]atrim=${num(a.in)}:${num(a.out)},asetpts=PTS-STARTPTS,${tempo}` +`aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` +- `volume=${num(a.volume)},` ++ `${soundChain(a.sound)}volume=${num(a.volume)},` +(fades.length ? fades.join(",") + "," : "") +`adelay=${delayMs}:all=1[snd${k}]`);