Open source CapCut alternative.
TypeScript
import { spawn } from "node:child_process";
import { mkdtemp, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { h264Encoder, vtQuality } from "./exportPipeline";
import { mediaPath } from "./projects";
import { assertGraphSafe, fexpr } from "./filterGraph";
import { num, round } from "./util";
import { UserError } from "../lib/userError";
function run(cmd: string, args: string[]): Promise<void> {
return new Promise((resolve, reject) => {
const p = spawn(cmd, args);
let err = "";
const timer = setTimeout(() => {
p.kill("SIGKILL");
reject(new Error(`${cmd} timed out after 60s.`));
}, 60_000);
timer.unref();
p.stderr.on("data", (d) => (err = (err + d.toString()).slice(-2000)));
p.on("error", (e) => {
clearTimeout(timer);
reject(
e.message.includes("ENOENT")
? new UserError("ffmpeg_not_found", "ffmpeg was not found. Install it with: brew install ffmpeg")
: e
);
});
p.on("close", (code) => {
clearTimeout(timer);
code === 0 ? resolve() : reject(new Error(err.split("\n").slice(-3).join("\n")));
});
});
}
export function probeDims(file: string): Promise<{ width: number; height: number }> {
return new Promise((resolve) => {
const p = spawn("ffprobe", [
"-v", "error",
"-select_streams", "v:0",
"-show_entries", "stream=width,height",
"-of", "csv=p=0",
file,
]);
let out = "";
const timer = setTimeout(() => {
p.kill("SIGKILL");
resolve({ width: 1080, height: 1920 });
}, 30_000);
timer.unref();
p.stdout.on("data", (d) => (out += d));
p.on("close", () => {
clearTimeout(timer);
const [w, h] = out.trim().split(",").map(Number);
resolve({ width: w || 1080, height: h || 1920 });
});
p.on("error", () => {
clearTimeout(timer);
resolve({ width: 1080, height: 1920 });
});
});
}
export interface FreezeFraming {
fit: "fit" | "fill";
/** How far the picture zooms past the size the frame fits it to (1 = none). */
zoom?: number;
panX: number;
panY: number;
/** Mirror the framed picture left for right / top for bottom. */
flipH?: boolean;
flipV?: boolean;
}
/** A freeze's quality on VideoToolbox, as the crf libx264's default gives. */
const FREEZE_CRF = 23;
/**
* Render a still-video clip from one frame of a project media file
* (a freeze frame), written into the project's media folder.
*
* When `frame` is given the still is composited exactly as the preview shows
* it — letterboxed (fit) or crop-panned (fill) into the project frame — so
* the capture is locked to the aspect at capture time. Switching the project
* aspect later letterboxes the baked still; capture another to re-fit.
*/
export async function makeFreezeFrame(
projectId: string,
sourceFile: string,
srcTime: number,
duration: number,
frame?: { w: number; h: number },
framing?: FreezeFraming
): Promise<{ fileName: string; duration: number; width: number; height: number }> {
const src = mediaPath(projectId, sourceFile);
const dur = Math.min(10, Math.max(0.5, duration));
const stampTime = Math.max(0, srcTime);
const tmp = await mkdtemp(path.join(os.tmpdir(), "veditor-freeze-"));
const d = new Date();
const pad = (n: number) => String(n).padStart(2, "0");
const fileName = `freeze-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}-${crypto.randomUUID().slice(0, 4)}.mp4`;
try {
const png = path.join(tmp, "frame.png");
await run("ffmpeg", ["-y", "-ss", stampTime.toFixed(3), "-i", src, "-frames:v", "1", png]);
let vf: string | null = null;
if (frame) {
const { w, h } = frame;
// The clip's own framing, baked: the picture meets the frame the way the
// preview shows it — covering or fitted, zoomed, panned. Same chain as
// the export graph's `boxFraming`.
const cover = framing?.fit === "fill";
const z = Math.max(1, Math.min(4, framing?.zoom ?? 1));
const even = (n: number) => 2 * Math.round(n / 2);
const kx = (0.5 + (framing?.panX ?? 0) / 2).toFixed(4);
const ky = (0.5 + (framing?.panY ?? 0) / 2).toFixed(4);
const crop = `,crop=${fexpr(`min(iw,${w})`)}:${fexpr(`min(ih,${h})`)}:(iw-ow)*${kx}:(ih-oh)*${ky}`;
const mirror = (framing?.flipH ? ",hflip" : "") + (framing?.flipV ? ",vflip" : "");
vf = cover
? `scale=${even(w * z)}:${even(h * z)}:force_original_aspect_ratio=increase${crop}${mirror}`
: `scale=${even(w * z)}:${even(h * z)}:force_original_aspect_ratio=decrease` +
(z > 1.0001 ? crop : "") +
mirror +
`,pad=${w}:${h}:(ow-iw)/2:(oh-ih)/2:black`;
}
// The shipped LGPL ffmpeg has no libx264: the still encodes with what
// the build carries, as every other render here does.
const enc = await h264Encoder();
await run("ffmpeg", [
"-y",
"-loop", "1",
"-i", png,
"-t", dur.toFixed(3),
"-r", "30",
...(vf ? ["-vf", assertGraphSafe(vf)] : []),
...(enc === "libx264"
? ["-c:v", "libx264", "-preset", "veryfast"]
: ["-c:v", "h264_videotoolbox", "-q:v", String(vtQuality(FREEZE_CRF)), "-allow_sw", "1"]),
"-pix_fmt", "yuv420p",
mediaPath(projectId, fileName),
]);
if (frame) return { fileName, duration: dur, width: frame.w, height: frame.h };
const dims = await probeDims(png);
return { fileName, duration: dur, width: dims.width, height: dims.height };
} finally {
void rm(tmp, { recursive: true, force: true });
}
}
/** Media duration in seconds via ffprobe (0 when unknown). */
export function probeDuration(file: string): Promise<number> {
return new Promise((resolve) => {
const p = spawn("ffprobe", [
"-v", "error",
"-show_entries", "format=duration",
"-of", "csv=p=0",
file,
]);
let out = "";
const timer = setTimeout(() => {
p.kill("SIGKILL");
resolve(0);
}, 30_000);
timer.unref();
p.stdout.on("data", (d) => (out += d));
p.on("close", () => {
clearTimeout(timer);
resolve(parseFloat(out) || 0);
});
p.on("error", () => {
clearTimeout(timer);
resolve(0);
});
});
}
interface CaptureRun {
ok: boolean;
timedOut: boolean;
stderr: string;
}
/** Run ffmpeg keeping the whole stderr — the frame/silence reports stream
* there. A timeout resolves instead of rejecting so callers can salvage
* partial output; a missing binary still maps to the install hint. */
function runCapture(args: string[], timeoutMs: number): Promise<CaptureRun> {
return new Promise((resolve, reject) => {
const p = spawn("ffmpeg", args);
let err = "";
let timedOut = false;
const timer = setTimeout(() => {
timedOut = true;
p.kill("SIGKILL");
}, timeoutMs);
timer.unref();
p.stderr.on("data", (d) => {
if (err.length < 2_000_000) err += d.toString();
});
p.on("error", (e) => {
clearTimeout(timer);
reject(
e.message.includes("ENOENT")
? new UserError("ffmpeg_not_found", "ffmpeg was not found. Install it with: brew install ffmpeg")
: e
);
});
p.on("close", (code) => {
clearTimeout(timer);
resolve({ ok: code === 0 && !timedOut, timedOut, stderr: err });
});
});
}
const errTail = (stderr: string) => stderr.trim().split("\n").slice(-3).join("\n");
export interface SilenceRange {
start: number;
end: number;
duration: number;
}
/** Find silent stretches in a media file's audio (ffmpeg silencedetect).
* Times are absolute source seconds. */
export async function detectSilence(
projectId: string,
sourceFile: string,
opts: { from: number; to: number; thresholdDb: number; minSilence: number }
): Promise<SilenceRange[]> {
const { from, to, thresholdDb, minSilence } = opts;
const r = await runCapture(
[
"-hide_banner", "-nostats", "-loglevel", "info",
...(from > 0 ? ["-ss", num(from)] : []),
"-t", num(to - from),
"-i", mediaPath(projectId, sourceFile),
"-vn", "-sn", "-dn",
"-af", `silencedetect=n=${thresholdDb}dB:d=${num(minSilence)}`,
"-f", "null", "-",
],
60_000
);
if (!r.ok) {
if (/does not contain any stream|matches no streams|Cannot find a matching stream/i.test(r.stderr))
throw new UserError("media_no_audio_track", "This file has no audio track.");
throw new Error(
r.timedOut ? "Silence detection timed out — try a shorter range." : errTail(r.stderr)
);
}
const silences: SilenceRange[] = [];
let open: number | null = null;
for (const line of r.stderr.split("\n")) {
const s = /silence_start: (-?[\d.]+)/.exec(line);
if (s) {
open = Math.max(from, from + parseFloat(s[1]));
continue;
}
const e = /silence_end: (-?[\d.]+)/.exec(line);
if (e && open !== null) {
const end = Math.min(to, from + parseFloat(e[1]));
if (end > open) silences.push({ start: round(open), end: round(end), duration: round(end - open) });
open = null;
}
}
// Silence still open at the end of the range closes there.
if (open !== null && to > open)
silences.push({ start: round(open), end: round(to), duration: round(to - open) });
return silences;
}
/** Pull a media file's audio track off as a small mono AAC stream so the
* assistant can hear it inline. Works for audio and video sources alike — the
* video image never travels, and 32 kbps keeps ~50 min under the model's
* inline cap. Times are absolute source seconds; an empty `to` runs to the end. */
export function extractAudio(
projectId: string,
sourceFile: string,
opts: { from: number; to?: number },
): Promise<Buffer> {
const { from, to } = opts;
return new Promise((resolve, reject) => {
const p = spawn("ffmpeg", [
"-hide_banner", "-nostats", "-loglevel", "error",
...(from > 0 ? ["-ss", num(from)] : []),
...(to !== undefined ? ["-t", num(to - from)] : []),
"-i", mediaPath(projectId, sourceFile),
"-vn", "-sn", "-dn",
"-ac", "1", "-ar", "24000", "-c:a", "aac", "-b:a", "32k",
"-f", "adts", "-",
]);
const chunks: Buffer[] = [];
let err = "";
const timer = setTimeout(() => {
p.kill("SIGKILL");
reject(new Error("Reading the audio timed out — try a shorter from/to range."));
}, 120_000);
timer.unref();
p.stdout.on("data", (d: Buffer) => chunks.push(d));
p.stderr.on("data", (d) => (err = (err + d.toString()).slice(-4000)));
p.on("error", (e) =>
reject(
e.message.includes("ENOENT")
? new UserError("ffmpeg_not_found", "ffmpeg was not found. Install it with: brew install ffmpeg")
: e,
),
);
p.on("close", (code) => {
clearTimeout(timer);
if (code === 0 && chunks.length > 0) return resolve(Buffer.concat(chunks));
if (/does not contain any stream|matches no streams|Cannot find a matching stream/i.test(err))
return reject(new UserError("media_no_audio_track", "This file has no audio track."));
reject(new Error(errTail(err) || "Could not read the audio."));
});
});
}