Open source CapCut alternative.
TypeScript
"use client";
/**
* The cut's audible mix, rendered in the browser.
*
* One `OfflineAudioContext` fold that mirrors the engine's ffmpeg audio graph:
* track-0 clip audio folds sequentially with cross-dissolves overlapping,
* upper-track clips and soundtrack items land at their absolute starts with
* their own volumes and fades, a ducking voiceover pulls everything else down
* while it speaks, and the whole-project fade rides over the sum.
*
* Two callers want this and want it at different shapes. Transcription needs a
* 16 kHz mono buffer, because that is what a speech model reads; an export
* needs full-rate stereo, because that is what goes in the file. Same fold,
* different rate and channel count — which is why those are parameters and the
* fold is written once. A mix that drifted between them would put captions on
* words that the exported audio says at another time.
*/
import {
AUDIO_FX_RAMP,
audioFxRecipe,
buildAudioFx,
retimeOf,
soundRecipe,
type ClipSound,
type SpeedNode,
} from "@donkeycut/effects-kit";
import { decodeAudioSpan } from "./mediaRead";
import { CLIP_MIN_SECONDS } from "./types";
import { fitSpan, retimeFits } from "./retimeFit";
import { OWN_SOUND, shiftSpan, soundWindow, type SoundWindow, type SpecSound } from "./soundSource";
/** One track-0 clip's audio in the sequential fold. A spacer (no file) only
* shapes time. */
export interface MixClip {
file: string;
in: number;
out: number;
muted: boolean;
speed?: number;
speedCurve?: SpeedNode[];
reverse?: boolean;
/** Gain on the clip's own audio; absent = 1. */
volume?: number;
/** The clip's own treatment, run on its sound before the fades. */
sound?: ClipSound;
/** Head and tail ramps of the clip's own fade animation, timeline seconds:
* the sound goes down to silence with the picture, the gain the preview
* plays. An edge a transition owns carries none. */
fadeIn?: number;
fadeOut?: number;
/** Cross-dissolve overlap into the next clip, timeline seconds. */
transition?: number;
/** Half the cross dissolve into the next clip, timeline seconds: the two
* clips ramp equal-power past each other over that long either side of the
* cut. Set instead of `transition` when the handover is on the sound. */
soundCross?: number;
/** Source seconds past `out` this clip keeps sounding, into the crossing at
* its tail, and source seconds before `in` it starts sounding, for the
* crossing at its head. Both are handle — media the trim left behind — and
* are what make the two clips audible at once over a hard picture cut. */
soundAhead?: number;
soundBack?: number;
/** A split edit: seconds of `soundBack` the clip's sound plays ahead of its
* picture and of `soundAhead` past its end, each ramping from silence over
* `splitFade` at its outer end. */
soundLead?: number;
soundTail?: number;
splitFade?: number;
/** The video's sound recorded apart from it: the sound reads this file,
* `offset` seconds later in the source (see soundSource.ts). */
soundFrom?: SpecSound;
}
/** A clip placed at an absolute time: an upper-track video's audio, a
* soundtrack bed, or a voiceover. */
export interface MixItem {
file: string;
in: number;
out: number;
start: number;
volume: number;
speed?: number;
speedCurve?: SpeedNode[];
reverse?: boolean;
/** Half the cross dissolve at this item's head and tail, with the handle
* each side reaches into — an upper-track cut crosses the same way track
* 0's does. */
crossIn?: number;
crossOut?: number;
soundAhead?: number;
soundBack?: number;
muted?: boolean;
fadeIn?: number;
fadeOut?: number;
sound?: ClipSound;
/** While this item is audible, everything else drops to this gain. */
duck?: number;
soundLead?: number;
soundTail?: number;
splitFade?: number;
soundFrom?: SpecSound;
/** The soundtrack lane this item sits on; absent for a video clip's own
* sound. What a stem render picks items by. */
lane?: number;
}
/** One audio effect element: the treatment it names, and the stretch of
* timeline it treats. */
export interface MixEffect {
effect: string;
amount?: number;
start: number;
end: number;
}
export interface MixSpec {
duration: number;
clips: MixClip[];
/** Soundtrack, voiceovers, and upper-track clip audio. */
items: MixItem[];
/** Audio effect elements over the finished mix, in timeline order. */
effects?: MixEffect[];
/** The duck envelope, when it comes from items outside this spec: a stem
* holds some of the mix's items and still ducks under all of its
* voiceovers. Absent = read from `items`. */
ducks?: { start: number; end: number; gain: number }[];
}
export interface MixOptions {
sampleRate: number;
channels: number;
/** Resolves a spec file name to something the reader can open. */
resolve: (file: string) => string;
}
/** Timeline length of one clip: its source span through its rate. A gap
* spacer keeps its exact length — flooring it would land everything after it
* late. */
const clipDur = (c: MixClip) =>
c.file ? Math.max(CLIP_MIN_SECONDS, retimeOf(c).len) : Math.max(0, c.out - c.in);
/** Timeline footprint of a placed item. */
const itemDur = (a: MixItem) => Math.max(0, retimeOf(a).len);
/**
* Where the sequential fold puts each track-0 clip, and how long its
* transition edges are. Clips abut — a transition claims no layout — and its
* sound is the outgoing clip fading across the blend window while the next
* enters clean at the cut, exactly like the engine's tail-fade + hard join.
*
* A cross dissolve is the other shape: the two clips ramp past each other
* across the cut, each reaching into its handle so both are really sounding
* while the picture hard-joins. Those ramps are equal-power and are kept apart
* from the fades, which are linear and end at silence.
*/
export function foldClips(clips: MixClip[]) {
const geo = clips.map((clip) => ({
clip,
at: 0,
dur: clipDur(clip),
fadeIn: 0,
fadeOut: 0,
crossIn: 0,
crossOut: 0,
}));
let acc = 0;
geo.forEach((g, j) => {
g.at = acc;
acc += g.dur;
// The clip's own fade animation ramps its sound; the tail's room is what
// the head left, the way the engine's stanza splits them.
const own = Math.min(Math.max(0, g.clip.fadeIn ?? 0), g.dur);
if (own > 0.01) g.fadeIn = own;
const ownOut = Math.min(Math.max(0, g.clip.fadeOut ?? 0), Math.max(0, g.dur - g.fadeIn));
if (ownOut > 0.01) g.fadeOut = ownOut;
const next = geo[j + 1];
if (!next) return;
const fade = Math.min(g.clip.transition ?? 0, g.dur * 0.9);
if (fade > 0.01) g.fadeOut = Math.max(g.fadeOut, fade);
const cross = Math.min(g.clip.soundCross ?? 0, g.dur * 0.9, next.dur * 0.9);
if (cross > 0.01) {
g.crossOut = cross;
next.crossIn = cross;
}
});
return geo;
}
/** How far each side of a clip reaches into its handle for a crossing, capped
* by the ramp that crossing actually got, or for a split edit, which an edge
* with a crossing never carries. */
const handles = (g: { clip: MixClip; crossIn: number; crossOut: number }) => {
const rt = retimeOf(g.clip);
return {
// Never past the head of the source: a handle is what the trim left
// behind. A reversed clip's head reaches toward the source's end, and
// `soundBack` already stops at it; a bound recording's reach was measured
// on the recording.
back: Math.min(
g.clip.soundBack ?? 0,
Math.max(g.crossIn, g.clip.soundLead ?? 0),
rt.reverse || g.clip.soundFrom ? Infinity : Math.max(0, -rt.tAt(0))
),
ahead: Math.min(g.clip.soundAhead ?? 0, Math.max(g.crossOut, g.clip.soundTail ?? 0)),
};
};
type Sounded = {
file: string;
in: number;
out: number;
speed?: number;
speedCurve?: SpeedNode[];
reverse?: boolean;
soundFrom?: SpecSound;
};
/** What an entry decodes: its own trim widened by the reach at its edges,
* on the file its sound reads — a bound recording through its offset, the
* window narrowed to what that recording holds. */
const soundRead = (c: Sounded, back = 0, ahead = 0): (SoundWindow & { file: string }) | null => {
const len = retimeOf(c).len;
const from = c.soundFrom;
const w = soundWindow(c, from ? { offset: from.offset, limit: from.duration } : OWN_SOUND, -back, len + ahead);
return w && { ...w, file: from ? from.file : c.file };
};
/** The ramps a split edit puts on an entry's gain, in timeline seconds: up
* from silence at the head of the sound that leads the picture, down to
* silence at the end of the sound that trails it. Inside the picture the
* clip's own fades and crossings shape the level, and they never reach out
* here — so the trailing sound comes back up to `level` at the picture's end.
* Scheduled after those fades: two events at the picture's end keep the order
* they were set in, and the last one is the level the tail plays at. */
function scheduleSplit(
param: AudioParam,
level: number,
head: number,
end: number,
lead: number,
tail: number,
fade: number
) {
if (lead > 0) {
const f = Math.min(fade, lead);
const from = Math.max(0, head - lead);
param.setValueAtTime(f > 0 ? 0 : level, from);
if (f > 0) param.linearRampToValueAtTime(level, from + f);
}
if (tail > 0) {
const f = Math.min(fade, tail);
param.setValueAtTime(level, end);
if (f > 0) {
param.setValueAtTime(level, end + tail - f);
param.linearRampToValueAtTime(0, end + tail);
}
}
}
/**
* The windows during which a ducking item is audible, merged into a single
* envelope: each entry is a stretch of timeline where everything that is not
* itself ducking drops to `gain`.
*
* Overlapping voiceovers take the lowest gain of the ones speaking, which is
* what the preview does frame by frame — computed here once for the whole
* timeline instead, because a scheduled ramp cannot ask "what is live now?".
*/
export function duckWindows(items: MixItem[]): { start: number; end: number; gain: number }[] {
const edges = items
.filter((a) => a.duck !== undefined && a.duck < 1)
.map((a) => ({ start: a.start, end: a.start + itemDur(a), gain: Math.max(0, a.duck!) }))
.filter((w) => w.end > w.start)
.sort((a, b) => a.start - b.start);
const merged: { start: number; end: number; gain: number }[] = [];
for (const w of edges) {
const last = merged[merged.length - 1];
if (last && w.start <= last.end) {
last.end = Math.max(last.end, w.end);
last.gain = Math.min(last.gain, w.gain);
} else {
merged.push({ ...w });
}
}
return merged;
}
/**
* Schedule one side of a cross dissolve: an equal-power ramp between silence
* and `level` across `[from, to]`, rising for the incoming clip and falling
* for the outgoing one. The two meet at 0.707 in the middle of the window —
* the cut — which is what keeps the mix at one loudness through the join.
*
* Written as short linear steps rather than `setValueCurveAtTime`, which is
* not carried by every context this fold runs in (the headless renderer's
* among them).
*/
const CROSS_STEPS = 24;
export function scheduleCross(
param: AudioParam,
level: number,
from: number,
to: number,
rising: boolean
) {
for (let i = 0; i <= CROSS_STEPS; i++) {
const p = i / CROSS_STEPS;
const v = level * (rising ? Math.sin((p * Math.PI) / 2) : Math.cos((p * Math.PI) / 2));
const at = Math.max(0, from + (to - from) * p);
if (i === 0) param.setValueAtTime(v, at);
else param.linearRampToValueAtTime(v, at);
}
}
/** Schedule a duck envelope onto a gain node: 1 everywhere except inside the
* windows, where it steps down and back. The steps are ramps a few
* milliseconds long so the drop doesn't click. */
const DUCK_RAMP = 0.08;
function scheduleDuck(gain: GainNode, windows: { start: number; end: number; gain: number }[]) {
gain.gain.value = 1;
for (const w of windows) {
gain.gain.setValueAtTime(1, Math.max(0, w.start - DUCK_RAMP));
gain.gain.linearRampToValueAtTime(w.gain, w.start);
gain.gain.setValueAtTime(w.gain, Math.max(w.start, w.end - DUCK_RAMP));
gain.gain.linearRampToValueAtTime(1, w.end);
}
}
/**
* Render `spec` to a buffer, or null when nothing in it makes a sound.
*
* A clip that plays at a speed other than 1 is time-stretched to fit, so it
* keeps its pitch — the same thing ffmpeg's atempo does for the engine and the
* browser's `preservesPitch` does for the preview.
*/
export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioBuffer | null> {
const { sampleRate, channels, resolve } = opts;
const length = Math.max(1, Math.ceil(spec.duration * sampleRate));
const ctx = new OfflineAudioContext(channels, length, sampleRate);
// Only the spans that are actually heard are decoded, and only for entries
// that can be heard at all. Decoding whole sources instead would mean a
// ten-second trim of a ninety-minute recording pulling the entire recording
// into memory — around 1.4 GB an hour at 48 kHz stereo — and several of them
// at once.
// The track-0 fold, with the source span each clip really decodes: a
// crossing at either edge widens it into the handle it reaches for.
const folded = foldClips(spec.clips).map((g) => {
const { back, ahead } = handles(g);
const read = g.clip.file && !g.clip.muted ? soundRead(g.clip, back, ahead) : null;
return { ...g, back, ahead, read };
});
// Each placed item's reach: a crossing's handle, or the split edit an edge
// without one carries.
const placed = spec.items.map((a) => {
const back = Math.min(a.soundBack ?? 0, Math.max(a.crossIn ?? 0, a.soundLead ?? 0));
const ahead = Math.min(a.soundAhead ?? 0, Math.max(a.crossOut ?? 0, a.soundTail ?? 0));
const read = a.file && !a.muted ? soundRead(a, back, ahead) : null;
return { a, back, ahead, read };
});
const wanted = new Map<string, { file: string; from: number; to: number }>();
const spanKey = (c: { file: string; lo: number; hi: number }) =>
`${c.file}|${c.lo.toFixed(3)}|${c.hi.toFixed(3)}`;
for (const { read } of [...folded, ...placed]) {
if (read) wanted.set(spanKey(read), { file: read.file, from: read.lo, to: read.hi });
}
const buffers = new Map<string, AudioBuffer>();
await Promise.all(
[...wanted].map(async ([key, span]) => {
// A source with no audio track at all — a silent recording, a still —
// reads back null and mixes as silence, like the engine's hasStream
// check. A source that could not be *read* is a different thing and is
// allowed to fail the render: silently dropping it would hand the user a
// finished file with a clip, or a whole voiceover, missing its sound.
const buf = await decodeAudioSpan(resolve(span.file), span.from, span.to);
if (buf) buffers.set(key, buf);
})
);
if (buffers.size === 0) return null;
// Everything that can be ducked passes through one node; the voiceovers doing
// the ducking connect past it, so they never duck themselves.
const ducked = ctx.createGain();
scheduleDuck(ducked, spec.ducks ?? duckWindows(spec.items));
// Every sound meets on the bus, which is what the effect elements treat —
// the ffmpeg graph applies them to the mixed stream in the same place.
const bus = ctx.createGain();
ducked.connect(bus);
// The effects run in series off the bus, each crossed in over its own
// window: inside it the treated signal is heard, outside it the untouched
// one, with the edges ramping so the switch cannot click.
let treated: AudioNode = bus;
for (const fx of spec.effects ?? []) {
const recipe = audioFxRecipe(fx.effect, fx.amount);
const from = Math.max(0, fx.start);
const to = Math.min(fx.end, spec.duration);
if (!recipe || !(to > from)) continue;
const nodes = buildAudioFx(ctx, recipe, from);
const wet = ctx.createGain();
const dry = ctx.createGain();
const out = ctx.createGain();
const ramp = Math.min(AUDIO_FX_RAMP, Math.max(0.001, (to - from) / 2));
const cross = (gain: GainNode, inside: number) => {
const outside = 1 - inside;
gain.gain.setValueAtTime(outside, 0);
gain.gain.setValueAtTime(outside, Math.max(0, from - ramp));
gain.gain.linearRampToValueAtTime(inside, from + ramp);
gain.gain.setValueAtTime(inside, Math.max(from + ramp, to - ramp));
gain.gain.linearRampToValueAtTime(outside, to + ramp);
};
cross(wet, 1);
cross(dry, 0);
treated.connect(nodes.input);
nodes.output.connect(wet);
wet.connect(out);
treated.connect(dry);
dry.connect(out);
treated = out;
}
treated.connect(ctx.destination);
/**
* Fit a span to the time its clip occupies, keeping its pitch.
*
* `playbackRate` would do the fitting for free, and resample while it did it —
* a clip at 1.5× would come back seven semitones high, which neither the
* preview nor the engine's `atempo` does. So the buffer is stretched to the
* length it needs first, and then played at its own rate.
*/
const stretched = new Map<string, { buffer: AudioBuffer; shift: number }>();
const fitted = (
key: string,
buf: AudioBuffer,
entry: { in: number; out: number; speed?: number; speedCurve?: SpeedNode[]; reverse?: boolean },
from: number,
to: number
): { buffer: AudioBuffer; shift: number } => {
const rt = retimeOf(entry);
if (!retimeFits(rt)) return { buffer: buf, shift: 0 };
// A span used by more than one clip at the same rate is stretched once.
const memo = `${key}@${rt.key}`;
const hit = stretched.get(memo);
if (hit) return hit;
const channels = Array.from({ length: buf.numberOfChannels }, (_, c) =>
buf.getChannelData(c)
);
// `from` is the source second the buffer starts at; the fit follows the
// clip's own map from there and lands exactly on its length.
const fit = fitSpan(channels, buf.sampleRate, rt, from, to);
const out = new AudioBuffer({
length: Math.max(1, fit.channels[0].length),
numberOfChannels: buf.numberOfChannels,
sampleRate: buf.sampleRate,
});
fit.channels.forEach((data, c) => out.getChannelData(c).set(data));
const fitted = { buffer: out, shift: fit.shift };
stretched.set(memo, fitted);
return fitted;
};
// Each buffer holds exactly the span its entry plays, already fitted to the
// time that entry occupies, so it starts at its own zero and runs at 1.
// The entry's own treatment runs on its sound alone, between the source and
// the gain that carries its level, fades and crossings — the order the
// preview's voice and the ffmpeg stanza both keep.
const play = (
buf: AudioBuffer,
at: number,
duration: number,
into: AudioNode,
sound: ClipSound | undefined,
shape: (gain: GainNode) => void
) => {
const src = ctx.createBufferSource();
src.buffer = buf;
const gain = ctx.createGain();
shape(gain);
const recipe = soundRecipe(sound);
if (recipe) {
const fx = buildAudioFx(ctx, recipe, Math.max(0, at));
src.connect(fx.input);
fx.output.connect(gain);
} else {
src.connect(gain);
}
gain.connect(into);
// A sound reaching before the timeline's start joins part-way in.
const skip = Math.max(0, -at);
src.start(Math.max(0, at), skip, Math.max(0, duration - skip));
};
for (const g of folded) {
const c = g.clip;
const read = g.read;
const key = read ? spanKey(read) : "";
const buf = read ? buffers.get(key) : undefined;
if (!read || !buf) continue; // muted, silent, or a gap spacer: only shapes time
// Laid through the clip's own map, moved onto the file the sound reads.
const fit = fitted(key, buf, shiftSpan(c, c.soundFrom?.offset ?? 0), read.lo, read.hi);
const from = g.at - g.back + read.at + fit.shift;
const end = g.at + g.dur;
play(fit.buffer, from, read.len, ducked, c.sound, (gain) => {
const level = Math.max(0, c.volume ?? 1);
gain.gain.value = level;
// A crossing and a fade never share an edge — the cut carries one
// handover — so each side takes whichever it has.
if (g.crossIn > 0) {
scheduleCross(gain.gain, level, g.at - g.crossIn, g.at + g.crossIn, true);
} else if (g.fadeIn > 0) {
gain.gain.setValueAtTime(0, g.at);
gain.gain.linearRampToValueAtTime(level, g.at + g.fadeIn);
}
if (g.crossOut > 0) {
scheduleCross(gain.gain, level, end - g.crossOut, end + g.crossOut, false);
} else if (g.fadeOut > 0) {
gain.gain.setValueAtTime(level, end - g.fadeOut);
gain.gain.linearRampToValueAtTime(0, end);
}
scheduleSplit(
gain.gain,
level,
g.at,
end,
g.crossIn > 0 ? 0 : Math.min(c.soundLead ?? 0, g.back),
g.crossOut > 0 ? 0 : Math.min(c.soundTail ?? 0, g.ahead),
c.splitFade ?? 0
);
});
}
for (const { a, back, ahead, read } of placed) {
const key = read ? spanKey(read) : "";
const buf = read ? buffers.get(key) : undefined;
if (!read || !buf) continue;
const dur = itemDur(a);
const end = a.start + dur;
// A voiceover that ducks everything else is not itself ducked; it still
// lands on the bus, so the effect elements treat it with the rest.
const into = a.duck !== undefined && a.duck < 1 ? bus : ducked;
const fit = fitted(key, buf, shiftSpan(a, a.soundFrom?.offset ?? 0), read.lo, read.hi);
play(fit.buffer, a.start - back + read.at + fit.shift, read.len, into, a.sound, (gain) => {
const level = Math.max(0, a.volume);
gain.gain.value = level;
const fIn = Math.min(a.fadeIn ?? 0, dur);
const fOut = Math.min(a.fadeOut ?? 0, dur);
if (a.crossIn) {
scheduleCross(gain.gain, level, a.start - a.crossIn, a.start + a.crossIn, true);
} else if (fIn > 0) {
gain.gain.setValueAtTime(0, a.start);
gain.gain.linearRampToValueAtTime(level, a.start + fIn);
}
if (a.crossOut) {
scheduleCross(gain.gain, level, end - a.crossOut, end + a.crossOut, false);
} else if (fOut > 0) {
gain.gain.setValueAtTime(level, end - fOut);
gain.gain.linearRampToValueAtTime(0, end);
}
scheduleSplit(
gain.gain,
level,
a.start,
end,
a.crossIn ? 0 : Math.min(a.soundLead ?? 0, back),
a.crossOut ? 0 : Math.min(a.soundTail ?? 0, ahead),
a.splitFade ?? 0
);
});
}
return ctx.startRendering();
}