Open source CapCut alternative.
TypeScript
import { spawn } from "node:child_process";
import { mkdir, mkdtemp, readFile, rm, stat, unlink, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { alignCues } from "../lib/cueAlign";
import { assertGraphSafe, fanOutInputs } from "./filterGraph";
import { assertLocalRuntime } from "./local-only";
import { createJobRegistry } from "./jobRegistry";
import { mediaPath, readProject } from "./projects";
import { retimeOf, type SpeedNode } from "@donkeycut/effects-kit";
import { CLIP_MIN_SECONDS } from "../lib/types";
import { bakeRetimedAudio, type BakedAudio } from "./retimeAudio";
import { atempoChain, findOnPath, hasStream, num, round, errorRecord } from "./util";
import { UserError, type ErrorFields } from "../lib/userError";
/** The audible slice of the cut, in timeline time (mirrors ExportSpec). */
export interface TranscribeSpec {
projectId: string;
duration: number;
locale?: string;
clips: {
file: string;
in: number;
out: number;
muted: boolean;
speed?: number;
speedCurve?: SpeedNode[];
reverse?: boolean;
/** Cross-dissolve overlap into the next clip, timeline seconds. */
transition?: number;
}[];
audio: {
file: string;
in: number;
out: number;
start: number;
volume: number;
speed?: number;
speedCurve?: SpeedNode[];
reverse?: boolean;
}[];
}
interface Word {
t0: number;
t1: number;
w: string;
}
export interface Cue {
id: string;
start: number;
end: number;
text: string;
words: Word[];
}
export interface TranscribeJob extends ErrorFields {
id: string;
projectId: string;
status: "running" | "done" | "error";
stage: "audio" | "model" | "transcribe";
cues?: Cue[];
}
// A speech run gets a fixed allowance plus twice the audio's length, so a run
// that hangs gives the queue back in minutes, e.g. 2m20s for a 10s listen. The
// whole-run cap holds for audio of unknown length.
const STT_BASE_MS = 120_000;
const STT_PER_AUDIO_SECOND_MS = 2_000;
const STT_MAX_MS = 600_000;
const sttTimeoutMs = (seconds: number) =>
Number.isFinite(seconds) ? Math.min(STT_MAX_MS, STT_BASE_MS + seconds * STT_PER_AUDIO_SECOND_MS) : STT_MAX_MS;
// Survives dev-server module reloads; caps the terminal backlog.
const { jobs, retire } = createJobRegistry<TranscribeJob>("__veditorSttJobs");
// On-device transcription is heavy, so jobs take the speech engine one at a
// time. A job waiting its turn reports running: a caller waits it out the
// same way it waits out the work itself. A chat waiting on the words goes
// "next", ahead of captions and background sweep chunks already queued.
type SpeechPlace = "next" | "last";
const speechQueue: (() => void)[] = [];
let speechBusy = false;
function inSpeechTurn<T>(work: () => Promise<T>, place: SpeechPlace = "last"): Promise<T> {
return new Promise<T>((resolve, reject) => {
const start = () => {
work()
.then(resolve, reject)
.finally(() => {
const following = speechQueue.shift();
if (following) {
following();
} else {
speechBusy = false;
}
});
};
if (!speechBusy) {
speechBusy = true;
start();
return;
}
if (place === "next") {
speechQueue.unshift(start);
} else {
speechQueue.push(start);
}
});
}
export function getTranscribeJob(id: string) {
return jobs.get(id);
}
function run(cmd: string, args: string[], notFound?: string, timeoutMs = 600_000): Promise<string> {
return new Promise((resolve, reject) => {
const p = spawn(cmd, args);
let out = "";
let err = "";
const timer = setTimeout(() => {
p.kill("SIGKILL");
const seconds = Math.round(timeoutMs / 1000);
reject(new UserError("tool_timed_out", `${cmd} timed out after ${seconds}s.`, { tool: cmd, seconds }));
}, timeoutMs);
timer.unref();
p.stdout.on("data", (d) => (out += d));
p.stderr.on("data", (d) => (err += d));
p.on("error", (e) => {
clearTimeout(timer);
reject(e.message.includes("ENOENT") && notFound ? new Error(notFound) : e);
});
p.on("close", (code) => {
clearTimeout(timer);
if (code === 0) {
return resolve(out);
}
// The tool's own complaint stays in its words; a silent exit names itself.
const tail = err.trim().split("\n").slice(-4).join("\n");
reject(tail ? new Error(tail) : new UserError("tool_exited", `${cmd} exited ${code}`, { tool: cmd, code: String(code) }));
});
});
}
/**
* The speech engine: Apple's on-device SpeechAnalyzer (macOS 26+), wrapped by
* a tiny Swift CLI. The packaged engine ships cut-stt beside its own binary
* (that directory leads PATH) and resolves it exclusively from PATH, so a dev
* machine exercises the exact prod lookup; the plain dev server compiles the
* source on first use into ~/.cache/cut. Nothing ever leaves the machine.
*/
export async function ensureStt(): Promise<string> {
const prebuilt = await findOnPath("cut-stt");
if (prebuilt) return prebuilt;
if (process.env.DONKEY_CUT_ENGINE) {
throw new UserError("speech_tool_missing", "The speech tool is missing. Update Donkey to restore transcription.");
}
const src = path.join(process.cwd(), "src", "cut", "server", "native", "cut-stt.swift");
const bin = path.join(os.homedir(), ".cache", "cut", "cut-stt");
const [b, s] = await Promise.all([stat(bin).catch(() => null), stat(src).catch(() => null)]);
if (b?.isFile() && (!s || b.mtimeMs >= s.mtimeMs)) return bin;
if (!s) {
throw new UserError("speech_tool_missing", "The speech tool is missing. Update Donkey to restore transcription.");
}
await mkdir(path.dirname(bin), { recursive: true });
await run(
"swiftc",
["-O", "-parse-as-library", src, "-o", bin],
"Swift compiler not found. Install the Xcode Command Line Tools: xcode-select --install"
).catch((e: Error) => {
throw new UserError("speech_engine_build_failed", `Could not build the on-device speech engine (needs macOS 26+).\n${e.message}`, { reason: e.message });
});
return bin;
}
/** Group word timings into short caption-sized cues — a few words at a time,
* the way captions read on a phone. */
export function groupWords(words: Word[]): Cue[] {
const MAX_CHARS = 38;
const MAX_DUR = 3.5;
const GAP = 0.6;
const cues: Cue[] = [];
let cur: Word[] = [];
const flush = () => {
if (cur.length === 0) return;
const start = round(cur[0].t0);
cues.push({
id: crypto.randomUUID().slice(0, 8),
start,
end: round(Math.max(cur[cur.length - 1].t1, start + 0.3)),
text: cur.map((w) => w.w).join(" "),
words: cur.map((w) => ({ t0: round(w.t0), t1: round(w.t1), w: w.w })),
});
cur = [];
};
for (const w of words) {
if (cur.length > 0) {
const last = cur[cur.length - 1];
const chars = cur.reduce((n, x) => n + x.w.length + 1, 0) + w.w.length;
if (
w.t0 - last.t1 > GAP ||
chars > MAX_CHARS ||
w.t1 - cur[0].t0 > MAX_DUR ||
/[.!?…]$/.test(last.w)
)
flush();
}
cur.push(w);
}
flush();
return cues;
}
/** Mono 16-bit samples out of a canonical PCM wav (what the mix render and the
* browser's uploads both are), for reading where the speech actually sits.
* Null for anything else — alignment is an improvement, so an unreadable file
* costs the caller nothing. */
async function readWavMono(file: string): Promise<{ samples: Int16Array; rate: number } | null> {
const buf = await readFile(file);
if (buf.length < 44 || buf.toString("ascii", 0, 4) !== "RIFF") return null;
if (buf.toString("ascii", 8, 12) !== "WAVE") return null;
let pos = 12;
let rate = 0;
let channels = 0;
let bits = 0;
let data: Uint8Array | null = null;
while (pos + 8 <= buf.length) {
const id = buf.toString("ascii", pos, pos + 4);
const size = buf.readUInt32LE(pos + 4);
const body = pos + 8;
if (id === "fmt " && body + 16 <= buf.length) {
channels = buf.readUInt16LE(body + 2);
rate = buf.readUInt32LE(body + 4);
bits = buf.readUInt16LE(body + 14);
} else if (id === "data") {
data = buf.subarray(body, Math.min(buf.length, body + size));
break;
}
pos = body + size + (size % 2); // chunks are word-aligned
}
if (!data || bits !== 16 || channels < 1 || rate <= 0) return null;
// An odd byte offset can't back an Int16Array view; copying realigns it.
const bytes = data.byteOffset % 2 === 0 ? data : new Uint8Array(data);
const pcm = new Int16Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.length / 2));
if (channels === 1) return { samples: pcm, rate };
const mono = new Int16Array(Math.floor(pcm.length / channels));
for (let i = 0; i < mono.length; i++) {
let sum = 0;
for (let c = 0; c < channels; c++) sum += pcm[i * channels + c];
mono[i] = sum / channels;
}
return { samples: mono, rate };
}
/** The finished cues for a run: grouped from the model's words, then snapped
* onto the speech in the very wav it transcribed, so captions land on the
* audio instead of a few frames off it. */
async function cuesFromRun(out: string, wav: string, duration: number): Promise<Cue[]> {
const cues = cuesFromStt(out, duration);
const pcm = await readWavMono(wav).catch(() => null);
return pcm ? alignCues(cues, pcm.samples, pcm.rate) : cues;
}
/** Cues from one cut-stt run: drop what the model emits degenerate, clamp to
* the audio's own length, and group the words into caption-sized cues. */
function cuesFromStt(out: string, duration: number): Cue[] {
const parsed = JSON.parse(out) as { words: Word[] };
const words = parsed.words
.filter((w) => w.w.trim().length > 0 && w.t1 > w.t0 - 0.001)
.map((w) => ({
t0: Math.max(0, Math.min(w.t0, duration)),
t1: Math.max(0, Math.min(w.t1, duration)),
w: w.w.trim(),
}))
.filter((w) => w.t1 > w.t0);
return groupWords(words);
}
/**
* Transcribe audio the caller hands over, for a project this engine does not
* store. A cloud project's media lives in R2, so the browser renders its
* audible mix (the same graph runTranscribe builds with ffmpeg) and posts the
* wav here — on-device speech then works whatever holds the project, and its
* real word timings beat the hosted route's interpolated ones. Shares the
* speech turn with the project path, so heavy speech still runs one at a time.
*/
export async function createAudioTranscribeJob(
audio: Uint8Array,
opts: { locale?: string; duration?: number } = {}
): Promise<TranscribeJob> {
assertLocalRuntime();
const job: TranscribeJob = { id: crypto.randomUUID().slice(0, 12), projectId: "", status: "running", stage: "model" };
jobs.set(job.id, job);
void inSpeechTurn(() => runAudioTranscribe(job, audio, opts))
.catch((err: unknown) => {
job.status = "error";
Object.assign(job, errorRecord(err, String(err)));
})
.finally(() => retire(job));
return job;
}
/** Transcribe audio in hand and wait for its words, taking the speech
* engine's next turn. The chat bridge reads a listen this way for a model
* that takes text and pictures only. `fileName` carries the container the
* speech tool reads the bytes as, e.g. "span.aac" for the engine's own cut. */
export function transcribeAudio(
audio: Uint8Array,
opts: { locale?: string; duration?: number; fileName?: string } = {}
): Promise<Cue[]> {
assertLocalRuntime();
const job: TranscribeJob = { id: crypto.randomUUID().slice(0, 12), projectId: "", status: "running", stage: "model" };
return inSpeechTurn(async () => {
await runAudioTranscribe(job, audio, opts);
return job.cues ?? [];
}, "next");
}
async function runAudioTranscribe(
job: TranscribeJob,
audio: Uint8Array,
{ locale, duration, fileName = "mix.wav" }: { locale?: string; duration?: number; fileName?: string }
) {
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "veditor-stt-"));
try {
const file = path.join(tmpDir, fileName);
await writeFile(file, audio);
const bin = await ensureStt();
job.stage = "transcribe";
const length = duration && duration > 0 ? duration : Infinity;
const out = await run(bin, [file, locale ?? "en-US"], undefined, sttTimeoutMs(length));
job.cues = await cuesFromRun(out, file, length);
job.status = "done";
} finally {
void rm(tmpDir, { recursive: true, force: true });
}
}
export async function createTranscribeJob(spec: TranscribeSpec): Promise<TranscribeJob> {
assertLocalRuntime();
const job: TranscribeJob = {
id: crypto.randomUUID().slice(0, 12),
projectId: spec.projectId,
status: "running",
stage: "audio",
};
jobs.set(job.id, job);
void inSpeechTurn(() => runTranscribe(job, spec))
.catch((err: unknown) => {
job.status = "error";
Object.assign(job, errorRecord(err, String(err)));
})
.finally(() => retire(job));
return job;
}
async function runTranscribe(job: TranscribeJob, spec: TranscribeSpec) {
if (!(await readProject(spec.projectId))) throw new UserError("project_not_found", "Project not found.");
// Video clips or a soundtrack clip are both valid speech sources (the render
// below mixes either), so an audio-only cut — a voiceover with no video, or a
// brief-to-video run transcribing its audio spine — transcribes fine.
if (spec.clips.length === 0 && spec.audio.length === 0)
throw new UserError("timeline_media_required", "Add audio or video to the timeline first.");
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "veditor-stt-"));
try {
// Render the cut's audible mix (clip audio + soundtrack, in timeline
// time) to a 16 kHz mono wav — the same graph the export uses, minus
// video, so cue times line up with the timeline exactly. Gap spacers
// (empty file) reference no media — they mix in as silence.
const files = [...new Set([...spec.clips, ...spec.audio].map((c) => c.file).filter(Boolean))];
const inputs: string[] = [];
const inputIndex = new Map<string, number>();
const audible = new Map<string, boolean>();
const paths = files.map((f) => mediaPath(spec.projectId, f));
await Promise.all(
paths.map(async (p, i) => {
if (!(await stat(p).catch(() => null)))
throw new UserError("media_file_missing_named", `Media file missing: ${files[i]}`, { name: files[i] });
})
);
files.forEach((f, i) => {
inputIndex.set(f, inputs.length / 2);
inputs.push("-i", paths[i]);
});
await Promise.all(
files.map(async (f, i) => audible.set(f, await hasStream(paths[i], "a")))
);
const hasSpeechSource =
spec.clips.some((c) => !c.muted && audible.get(c.file)) ||
spec.audio.some((a) => audible.get(a.file));
if (!hasSpeechSource) {
job.cues = [];
job.status = "done";
return;
}
const filters: string[] = [];
// Per-clip timeline length: a sped-up clip is shorter and time-stretched,
// so the transcript's cue times line up with what the user sees. A gap
// spacer (no file) keeps its exact length — flooring it would
// land every cue after the gap late.
const clipDur = (c: TranscribeSpec["clips"][number]) =>
c.file ? Math.max(CLIP_MIN_SECONDS, retimeOf(c).len) : Math.max(0, c.out - c.in);
// A clip whose rate changes through its footage is baked to a WAV in
// timeline seconds first (the same stretch the export and the preview
// use), so its words land where they play.
type Sounding = { file: string; in: number; out: number; speed?: number; speedCurve?: SpeedNode[]; reverse?: boolean };
const baked = new Map<Sounding, BakedAudio & { idx: number }>();
const bakeIO = {
ffmpeg: async (args: string[]) => {
await run("ffmpeg", args);
},
readFile: (p: string) => readFile(p),
writeFile: (p: string, data: Uint8Array) => writeFile(p, data),
unlink: (p: string) => unlink(p),
};
const curved: { c: Sounding; tag: string }[] = [
...spec.clips.filter((c) => !c.muted).map((c, j) => ({ c, tag: `clip_${j}` })),
...spec.audio.map((a, k) => ({ c: a, tag: `snd_${k}` })),
];
for (const { c, tag } of curved) {
const rt = retimeOf(c);
if (rt.uniform || !c.file || !audible.get(c.file)) continue;
const file = path.join(tmpDir, `retime_${tag}.wav`);
const bake = await bakeRetimedAudio(bakeIO, mediaPath(spec.projectId, c.file), rt, 0, 0, file);
const idx = inputs.length / 2;
inputs.push("-i", file);
baked.set(c, { ...bake, idx });
}
/** The head of a chain reading a clip's sound over its whole span. */
const audioRead = (c: Sounding, len: number) => {
const b = baked.get(c);
if (b) return `[${b.idx}:a]atrim=${num(b.back)}:${num(b.back + len)},asetpts=PTS-STARTPTS,`;
const rate = retimeOf(c).rate;
const tempo = rate !== 1 ? `${atempoChain(rate)},` : "";
return `[${inputIndex.get(c.file)}:a]atrim=${num(c.in)}:${num(c.out)},asetpts=PTS-STARTPTS,${tempo}`;
};
spec.clips.forEach((c, j) => {
const dur = clipDur(c);
if (!c.muted && audible.get(c.file)) {
filters.push(
audioRead(c, dur) +
`aresample=16000,aformat=sample_fmts=s16:channel_layouts=mono,` +
`apad=whole_dur=${num(dur)},atrim=0:${num(dur)}[a${j}]`
);
} else {
filters.push(`anullsrc=r=16000:cl=mono,atrim=0:${num(dur)},asetpts=PTS-STARTPTS[a${j}]`);
}
});
// Join clip audio exactly as the timeline (and export) do: clips abut, a
// transition fades the outgoing clip's tail while the next enters clean at
// the cut, so the chain is a flat concat with tail fades — the mix runs
// exactly the timeline's length and every cue lands where it plays. A
// cross dissolve's own fades stay out of it: they change no cue's time,
// and words are transcribed best at the level they were spoken.
// Skipped entirely for an audio-only cut (no clips) — there is no clip-audio
// chain to build, only the soundtrack, mixed below. (`spec.clips[0]` would be
// undefined otherwise.)
let aAcc: string | null = null;
if (spec.clips.length > 0) {
aAcc = "a0";
let acc = clipDur(spec.clips[0]);
for (let j = 1; j < spec.clips.length; j++) {
const prev = spec.clips[j - 1];
const durJ = clipDur(spec.clips[j]);
const d = Math.min(prev.transition ?? 0, acc * 0.9, durJ * 0.9);
const out = `aj${j}`;
if (d > 0.01) {
filters.push(`[${aAcc}]afade=t=out:st=${num(acc - d)}:d=${num(d)}[af${j}]`);
filters.push(`[af${j}][a${j}]concat=n=2:v=0:a=1[${out}]`);
} else {
filters.push(`[${aAcc}][a${j}]concat=n=2:v=0:a=1[${out}]`);
}
acc = acc + durJ;
aAcc = out;
}
}
const soundLabels: string[] = [];
spec.audio.forEach((a, k) => {
if (!audible.get(a.file)) return;
filters.push(
audioRead(a, Math.max(CLIP_MIN_SECONDS, retimeOf(a).len)) +
`aresample=16000,aformat=sample_fmts=s16:channel_layouts=mono,` +
`volume=${num(a.volume)},adelay=${Math.max(0, Math.round(a.start * 1000))}:all=1[snd${k}]`
);
soundLabels.push(`snd${k}`);
});
// Mix the clip chain (when present) with the soundtrack. A single source
// needs no amix; hasSpeechSource above guarantees at least one input here.
// duration=longest spans every input (a late soundtrack clip must not be
// cut off — with no video clip chain, `first` would truncate the mix to the
// earliest-finishing clip and drop later speech); the outer `-t
// spec.duration` then trims to the timeline.
const mixInputs = [...(aAcc ? [aAcc] : []), ...soundLabels];
let aLabel: string;
if (mixInputs.length <= 1) {
aLabel = mixInputs[0];
} else {
filters.push(
mixInputs.map((l) => `[${l}]`).join("") +
`amix=inputs=${mixInputs.length}:duration=longest:dropout_transition=0:normalize=0[amix]`
);
aLabel = "amix";
}
const wav = path.join(tmpDir, "mix.wav");
await run(
"ffmpeg",
[
"-y",
...inputs,
"-filter_complex", assertGraphSafe(fanOutInputs(filters).join(";")),
"-map", `[${aLabel}]`,
"-ac", "1",
"-ar", "16000",
"-t", num(spec.duration),
wav,
],
"ffmpeg was not found. Install it with: brew install ffmpeg"
);
job.stage = "model";
const bin = await ensureStt();
job.stage = "transcribe";
const out = await run(bin, [wav, spec.locale ?? "en-US"], undefined, sttTimeoutMs(spec.duration));
job.cues = await cuesFromRun(out, wav, spec.duration);
job.status = "done";
} finally {
void rm(tmpDir, { recursive: true, force: true });
}
}