Open source CapCut alternative.
TypeScript
/**
* The sound of a clip whose rate changes through its footage, laid for
* ffmpeg.
*
* `atempo` takes one constant, and the bundled build (LGPL) carries nothing
* that takes a schedule. So a curved clip's audio is baked ahead of the graph:
* the trimmed span — widened by the handles a crossing reaches into — decodes
* to float PCM through ffmpeg, the same WSOLA the preview and the in-tab export
* run stretches it along the clip's own map, and the result lands as a WAV the
* graph reads like any other input, in timeline seconds and needing no tempo.
* One stretch implementation serves every renderer, which is the parity the
* export path promises.
*/
import { srcSpan, type Retime } from "@donkeycut/effects-kit";
import { fitSpan } from "../lib/retimeFit";
import { wavHeader } from "../lib/wav";
import { num } from "./util";
/** The bake's sample rate: what the graphs resample every input to. */
export const BAKE_RATE = 44100;
const CHANNELS = 2;
export interface BakedAudio {
/** The WAV on disk. */
file: string;
/** Timeline seconds the file holds before the clip's head (the back handle
* it was baked with, clamped to the source's start). */
back: number;
/** Timeline seconds the file runs. */
len: number;
}
export interface BakeIO {
/** Run ffmpeg with these args to completion. */
ffmpeg: (args: string[]) => Promise<void>;
readFile: (path: string) => Promise<Uint8Array>;
writeFile: (path: string, data: Uint8Array) => Promise<void>;
/** Drop the intermediate PCM once it is read. */
unlink: (path: string) => Promise<void>;
}
/**
* Bake `src`'s sound over the clip's span, plus `back` timeline seconds
* before its head and `ahead` after its tail, into `outFile`.
*/
export async function bakeRetimedAudio(
io: BakeIO,
src: string,
retime: Retime,
back: number,
ahead: number,
outFile: string,
/** A mono source is laid into both channels at full level, the way the
* browser's mixer upmixes it; ffmpeg's own upmix would sit it 3 dB down. */
opts: { mono?: boolean } = {}
): Promise<BakedAudio> {
const rt = retime;
// A handle cannot reach before the source's first sample; the head reach
// is whatever the source really had.
const reach = srcSpan(rt, -Math.max(0, back), rt.len + Math.max(0, ahead));
const from = Math.max(0, reach.lo);
const to = Math.max(from, reach.hi);
const pcm = `${outFile}.f32`;
await io.ffmpeg([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-ss",
num(from),
"-i",
src,
"-t",
num(Math.max(0.001, to - from)),
"-vn",
...(opts.mono ? ["-af", "pan=stereo|c0=c0|c1=c0"] : ["-ac", String(CHANNELS)]),
"-ar",
String(BAKE_RATE),
"-f",
"f32le",
pcm,
]);
let raw: Uint8Array | null = await io.readFile(pcm);
await io.unlink(pcm).catch(() => {});
// A DataView reads the samples wherever the buffer starts, so the span is
// held once rather than copied to an aligned array first — this is minutes
// of audio on a long clip.
const view = new DataView(raw.buffer, raw.byteOffset, raw.byteLength);
const frames = Math.floor(raw.byteLength / 4 / CHANNELS);
const channels = Array.from({ length: CHANNELS }, () => new Float32Array(frames));
for (let i = 0; i < frames; i++)
for (let c = 0; c < CHANNELS; c++)
channels[c][i] = view.getFloat32((i * CHANNELS + c) * 4, true);
// The read buffer is dead once the channels hold the samples; letting go of
// it here keeps the stretch from running beside a second copy of the span.
raw = null;
// The file may run short of `to` — the source ended first — so the output
// length follows the samples actually read, through the same map, and so
// does the head the fit reports: `from` on a forward clip, the far end of
// what was read on a reversed one.
const out = fitSpan(channels, BAKE_RATE, rt, from);
const tHead = rt.tAt(out.head);
const outLength = Math.max(1, out.channels[0]?.length ?? 1);
await io.writeFile(outFile, wavFloat32(out.channels, BAKE_RATE));
return { file: outFile, back: -tHead, len: outLength / BAKE_RATE };
}
/** A WAV container around float samples: what ffmpeg reads back as pcm_f32le. */
export function wavFloat32(channels: Float32Array[], sampleRate: number): Uint8Array {
const n = channels[0]?.length ?? 0;
const ch = Math.max(1, channels.length);
const buf = new ArrayBuffer(44 + n * ch * 4);
new Uint8Array(buf).set(wavHeader(n, ch, sampleRate, 32, "float"));
const samples = new Float32Array(buf, 44, n * ch);
for (let i = 0; i < n; i++)
for (let c = 0; c < ch; c++) samples[i * ch + c] = channels[c]?.[i] ?? 0;
return new Uint8Array(buf);
}
/**
* The `setpts` expression that lays a clip's frames along its map: the
* reduced polyline of the retime as a sum of clipped ramps, so nothing nests
* however many knots the curve has. `T - from` is the frame's seconds past
* the clip's in point; the sum is its timeline second, over the timebase. A
* frame from before the in point lands at 0.
*/
export function setptsExpr(retime: Retime, from: number): string {
// Slopes carry six places: a rounded slope drifts by its error times the
// knot's width, and three places would put a long ramp a frame off.
const fnum = (n: number) => String(Math.round(n * 1e6) / 1e6);
const k = retime.knots;
const terms: string[] = [];
for (let i = 0; i + 1 < k.length; i++) {
const [s0, t0] = k[i];
const [s1, t1] = k[i + 1];
const w = s1 - s0;
if (w <= 0) continue;
const slope = (t1 - t0) / w;
const x = `T-${fnum(from + s0)}`;
terms.push(`clip(${x},0,${fnum(w)})*${fnum(slope)}`);
}
if (terms.length === 0) return `PTS-${fnum(from)}/TB`;
return `(${terms.join("+")})/TB`;
}