Skip to content

Open source CapCut alternative.

TypeScript

feat(site): a speed curve renders in ffmpeg [rebuild]

David Lec4dffd63376e
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.ts
index bf2af942..64eaec41 100644
--- a/site/src/cut/server/exportPipeline.test.ts
+++ b/site/src/cut/server/exportPipeline.test.ts
@@ -1,4 +1,5 @@
import { describe, expect, test } from "bun:test";
+import { retimeOf } from "@donkeycut/effects-kit";
import {
runExport,
type ExportPipelineIO,
@@ -14,6 +15,8 @@ const graphFor = async (over: Partial<ExportSpec>): Promise<string[]> => {
const io: ExportPipelineIO = {
stat: (async () => ({ isFile: () => true })) as unknown as ExportPipelineIO["stat"],
writeFile: (async () => {}) as unknown as ExportPipelineIO["writeFile"],
+ readFile: (async () => new Uint8Array(0)) as unknown as ExportPipelineIO["readFile"],
+ unlink: (async () => {}) as unknown as ExportPipelineIO["unlink"],
hasStream: async () => true,
videoColorInfo: async () => null,
h264Encoder: async () => "libx264",
@@ -29,10 +32,7 @@ const graphFor = async (over: Partial<ExportSpec>): Promise<string[]> => {
fps: 30,
crf: 24,
preset: "veryfast",
- duration: clips.reduce(
- (s, c) => s + (c.out - c.in) / (c.speed && c.speed > 0 ? c.speed : 1),
- 0
- ),
+ duration: clips.reduce((s, c) => s + retimeOf(c).len, 0),
clips,
audio: [],
overlays: [],
@@ -45,7 +45,7 @@ const graphFor = async (over: Partial<ExportSpec>): Promise<string[]> => {
log: [],
};
await runExport(job, spec, (f) => `/media/${f}`, io);
- const enc = ffmpegCalls[0];
+ const enc = ffmpegCalls.find((a) => a.includes("-filter_complex"))!;
return enc[enc.indexOf("-filter_complex") + 1].split(";");
};
@@ -321,6 +321,24 @@ describe("export filtergraph timebases", () => {
});
});
+describe("a speed curve in the filtergraph", () => {
+ test("a curved clip lays its picture through the map and reads baked sound", async () => {
+ const curved = clip("a.mp4", { speedCurve: [[0, 1], [4, 4]] });
+ const g = await graphFor({ clips: [curved, clip("b.mp4")] });
+ const video = g.find((f) => f.startsWith("[0:v]trim="))!;
+ expect(video).toContain("setpts='(clip(T-STARTT");
+ expect(video).not.toContain("(PTS-STARTPTS)/");
+ // The baked WAV joins the inputs after the two media files, and the
+ // clip's sound is read from it in timeline seconds with no tempo.
+ const audio = g.find((f) => f.startsWith("[2:a]atrim="))!;
+ expect(audio).toBeTruthy();
+ expect(audio).not.toContain("atempo");
+ expect(audio).toContain(`apad=whole_dur=${retimeOf(curved).len.toFixed(3)}`);
+ // The plain clip still reads its own input at source seconds.
+ expect(g.some((f) => f.startsWith("[1:a]atrim=0.000:4.000"))).toBe(true);
+ });
+});
+
describe("clip sound in the filtergraph", () => {
const sound = {
eq: [2, 1, -2, -1, 1.5, 1, -1],
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index c8922200..d9e34c76 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -1,10 +1,11 @@
import { spawn, type ChildProcess } from "node:child_process";
-import { stat, writeFile } from "node:fs/promises";
+import { readFile, stat, unlink, writeFile } from "node:fs/promises";
import path from "node:path";
import { atempoChain, hasStream, num, videoColorInfo } from "./util";
import { assertGraphSafe, fexpr } from "./filterGraph";
+import { bakeRetimedAudio, setptsExpr, type BakedAudio } from "./retimeAudio";
import { CLIP_MAX_ZOOM, projectFadeSeconds, regionPx, TRANSITION_XFADE, TRANSITION_ZOOM, type ColorGrade, type TransitionStyle } from "../lib/types";
-import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, shortestTurn, sortedKeys, soundFilters, type ClipSound, type OverlayKey } from "@donkeycut/effects-kit";
+import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, retimeOf, shortestTurn, sortedKeys, soundFilters, type ClipSound, type OverlayKey, type Retime, type SpeedNode } from "@donkeycut/effects-kit";
// The render pipeline itself: spec in, finished mp4 out. Shared by the local
// engine's job registry (jobs.ts) and the cloud render worker, which stage
@@ -88,6 +89,9 @@ export interface ExportSpec {
/** Region of the frame this clip fills; absent = full frame. */
frame?: { x: number; y: number; w: number; h: number };
speed?: number; // playback rate, default 1
+ /** A rate that changes through the footage: [source second, rate] nodes
+ * (see retimeOf); present, `speed` is ignored. */
+ speedCurve?: SpeedNode[];
/** Transition into the next clip, in timeline seconds (overlap). */
transition?: number;
/** Half the cross dissolve into the next clip, in timeline seconds: the
@@ -175,6 +179,9 @@ export interface ExportSpec {
volume?: number;
sound?: ClipSound;
speed?: number;
+ /** A rate that changes through the footage: [source second, rate] nodes
+ * (see retimeOf); present, `speed` is ignored. */
+ speedCurve?: SpeedNode[];
/** Transition ramps, timeline seconds from this overlay's head/tail. On
* an upper track a fade is an alpha fade (the tracks beneath show
* through); a cross transition arrives as the incoming clip's headFade
@@ -235,6 +242,9 @@ export interface ExportSpec {
fadeOut?: number;
/** Playback rate (detached-audio clips inherit their video clip's speed). */
speed?: number;
+ /** A rate that changes through the footage: [source second, rate] nodes
+ * (see retimeOf); present, `speed` is ignored. */
+ speedCurve?: SpeedNode[];
sound?: ClipSound;
/** Voiceover ducking: while this clip plays, every other sound drops to
* this gain (0..1). Ducking clips never duck each other. */
@@ -377,9 +387,14 @@ function ffColor(hex: string | undefined): string {
return m ? `0x${m[1].toUpperCase()}` : "black";
}
-function clipRate(c: ExportSpec["clips"][number]) {
- return c.speed && c.speed > 0 ? c.speed : 1;
-}
+/** The `setpts` that lays a trimmed span at its rate: the plain division for
+ * one rate, the map's polyline for a curve (quoted: it carries commas). */
+const retimedPts = (rt: Retime) =>
+ rt.uniform ? `(PTS-STARTPTS)/${num(rt.rate)}` : fexpr(setptsExpr(rt));
+
+/** Timeline length of a media clip's span through its rate or curve. */
+const spanLen = (c: { in: number; out: number; speed?: number; speedCurve?: SpeedNode[] }) =>
+ Math.max(0.1, retimeOf(c).len);
/** A clip's frame region in even pixels, or null when it fills the whole frame
* (the common case, which keeps the plain full-frame filter path). */
@@ -446,13 +461,15 @@ export function runFfmpeg(
export interface ExportPipelineIO {
stat: typeof stat;
writeFile: typeof writeFile;
+ readFile: typeof readFile;
+ unlink: typeof unlink;
hasStream: typeof hasStream;
videoColorInfo: typeof videoColorInfo;
h264Encoder: () => Promise<"libx264" | "h264_videotoolbox">;
runFfmpeg: typeof runFfmpeg;
}
-const realIO: ExportPipelineIO = { stat, writeFile, hasStream, videoColorInfo, h264Encoder, runFfmpeg };
+const realIO: ExportPipelineIO = { stat, writeFile, readFile, unlink, hasStream, videoColorInfo, h264Encoder, runFfmpeg };
/** Render `spec` into `job.outPath`. `mediaPathFor` maps a spec media file
* name to its staged path on disk (the engine reads the project folder; the
@@ -599,15 +616,14 @@ export async function runExport(
for (let j = 0; j < spec.clips.length; j++) {
const c = spec.clips[j];
if (!c.image || !c.file) continue;
- const dur = Math.max(0.1, (c.out - c.in) / clipRate(c));
+ const dur = spanLen(c);
imageClipInput.set(j, nInputs++);
inputs.push("-loop", "1", "-t", num(dur), "-framerate", String(fps), "-i", await resolveMedia(io.stat, mediaPathFor, c.file));
}
const imageOverlayInput = new Map<(typeof overlayVideos)[number], number>();
for (const oc of overlayVideos) {
if (!oc.image) continue;
- const ospeed = oc.speed && oc.speed > 0 ? oc.speed : 1;
- const olen = Math.max(0.1, (oc.out - oc.in) / ospeed);
+ const olen = spanLen(oc);
imageOverlayInput.set(oc, nInputs++);
inputs.push("-loop", "1", "-t", num(olen), "-framerate", String(fps), "-i", await resolveMedia(io.stat, mediaPathFor, oc.file));
}
@@ -644,7 +660,7 @@ export async function runExport(
for (let j = 0; j < spec.clips.length; j++) {
const c = spec.clips[j];
if (!c.mask) continue;
- const dur = c.file ? Math.max(0.1, (c.out - c.in) / clipRate(c)) : Math.max(0, c.out - c.in);
+ const dur = c.file ? spanLen(c) : Math.max(0, c.out - c.in);
const idx = await maskInput(c.mask, dur, `mask_clip_${j}`);
if (idx !== undefined) clipMaskInput.set(j, idx);
}
@@ -652,8 +668,7 @@ export async function runExport(
for (let k = 0; k < overlayVideos.length; k++) {
const oc = overlayVideos[k];
if (!oc.mask) continue;
- const ospeed = oc.speed && oc.speed > 0 ? oc.speed : 1;
- const olen = Math.max(0.1, (oc.out - oc.in) / ospeed);
+ const olen = spanLen(oc);
const idx = await maskInput(oc.mask, olen, `mask_ovl_${k}`);
if (idx !== undefined) overlayMaskInput.set(oc, idx);
}
@@ -663,7 +678,7 @@ export async function runExport(
for (let j = 0; j < spec.clips.length; j++) {
const c = spec.clips[j];
if (!c.shadow) continue;
- const dur = c.file ? Math.max(0.1, (c.out - c.in) / clipRate(c)) : Math.max(0, c.out - c.in);
+ const dur = c.file ? spanLen(c) : Math.max(0, c.out - c.in);
const idx = await maskInput(c.shadow, dur, `shadow_clip_${j}`);
if (idx !== undefined) clipShadowInput.set(j, idx);
}
@@ -671,10 +686,9 @@ export async function runExport(
for (let k = 0; k < overlayVideos.length; k++) {
const oc = overlayVideos[k];
if (!oc.shadow) continue;
- const ospeed = oc.speed && oc.speed > 0 ? oc.speed : 1;
// An overlay's shadow lands on the timeline beside the clip, so it plays
// for the clip's whole window rather than the segment's local clock.
- const olen = Math.max(0.1, (oc.out - oc.in) / ospeed);
+ const olen = spanLen(oc);
const idx = await maskInput(oc.shadow, olen, `shadow_ovl_${k}`);
if (idx !== undefined) overlayShadowInput.set(oc, idx);
}
@@ -688,14 +702,13 @@ export async function runExport(
for (let j = 0; j < spec.clips.length; j++) {
const c = spec.clips[j];
if (!c.border) continue;
- const dur = c.file ? Math.max(0.1, (c.out - c.in) / clipRate(c)) : Math.max(0, c.out - c.in);
+ const dur = c.file ? spanLen(c) : Math.max(0, c.out - c.in);
clipBorderInput.set(j, borderStill(c.border, dur));
}
const overlayBorderInput = new Map<(typeof overlayVideos)[number], number>();
for (const oc of overlayVideos) {
if (!oc.border) continue;
- const ospeed = oc.speed && oc.speed > 0 ? oc.speed : 1;
- const olen = Math.max(0.1, (oc.out - oc.in) / ospeed);
+ const olen = spanLen(oc);
overlayBorderInput.set(oc, borderStill(oc.border, olen));
}
@@ -785,7 +798,70 @@ export async function runExport(
// land everything after the gap later than the timeline shows, drifting
// burned-in captions off the picture.
const clipDur = (c: ExportSpec["clips"][number]) =>
- c.file ? Math.max(0.1, (c.out - c.in) / clipRate(c)) : Math.max(0, c.out - c.in);
+ c.file ? spanLen(c) : Math.max(0, c.out - c.in);
+
+ // A clip whose rate changes through its footage gets its sound baked ahead
+ // of the graph (retimeAudio.ts): the span plus the handles a crossing can
+ // reach into, laid in timeline seconds. The WAV joins the inputs and every
+ // audio read of that clip comes from it, needing no tempo.
+ type Sounding = {
+ file: string;
+ in: number;
+ out: number;
+ speed?: number;
+ speedCurve?: SpeedNode[];
+ soundBack?: number;
+ soundAhead?: number;
+ muted?: boolean;
+ hidden?: boolean;
+ image?: boolean;
+ };
+ const baked = new Map<Sounding, BakedAudio & { idx: number }>();
+ const bakeTargets: { c: Sounding; tag: string }[] = [
+ ...spec.clips.map((c, j) => ({ c, tag: `clip_${j}` })),
+ ...overlayVideos.map((oc, k) => ({ c: oc, tag: `ovl_${k}` })),
+ ...spec.audio.map((a, k) => ({ c: a, tag: `snd_${k}` })),
+ ];
+ for (const { c, tag } of bakeTargets) {
+ const rt = retimeOf(c);
+ if (rt.uniform || !c.file || c.image || c.muted || c.hidden || !audioPresence.get(c.file)) continue;
+ const file = path.join(job.tmpDir, `retime_${tag}.wav`);
+ const src = await resolveMedia(io.stat, mediaPathFor, c.file);
+ const bake = await bakeRetimedAudio(
+ {
+ ffmpeg: (args) => io.runFfmpeg(job, args),
+ readFile: (p) => io.readFile(p),
+ writeFile: (p, data) => io.writeFile(p, data),
+ unlink: (p) => io.unlink(p),
+ },
+ src,
+ rt,
+ c.soundBack ?? 0,
+ c.soundAhead ?? 0,
+ file
+ );
+ const idx = nInputs++;
+ inputs.push("-i", file);
+ baked.set(c, { ...bake, idx });
+ }
+ /**
+ * The head of an audio chain reading a clip's sound over the timeline-local
+ * window `[fromT, toT]` (seconds from the clip's head; negative reaches
+ * into the back handle): the input, its trim, and the tempo that lays it
+ * at the clip's rate. A baked clip is read from its WAV in timeline seconds
+ * and needs none.
+ */
+ const audioRead = (c: Sounding, fromT: number, toT: number): string => {
+ const b = baked.get(c);
+ if (b) {
+ return `[${b.idx}:a]atrim=${num(Math.max(0, b.back + fromT))}:${num(Math.max(0, b.back + toT))},asetpts=PTS-STARTPTS,`;
+ }
+ const rt = retimeOf(c);
+ const tempo = rt.rate !== 1 ? `${atempoChain(rt.rate)},` : "";
+ return `[${inputIndex.get(c.file)!}:a]atrim=${num(rt.srcAt(fromT))}:${num(rt.srcAt(toT))},asetpts=PTS-STARTPTS,${tempo}`;
+ };
+ /** Timeline seconds of source a clip has before its head, at its own pace. */
+ const headRoom = (c: Sounding) => Math.max(0, -retimeOf(c).tAt(0));
// Where each track-0 segment lands on the joined timeline: transitions
// pad their overlap, so the layout is the plain running sum of durations.
const clipStarts: number[] = [];
@@ -1092,7 +1168,7 @@ export async function runExport(
// Per-clip normalized video + audio segments for the join below.
spec.clips.forEach((c, j) => {
const idx = c.image ? imageClipInput.get(j)! : inputIndex.get(c.file)!;
- const speed = clipRate(c);
+ const rt = retimeOf(c);
const dur = clipDur(c);
const prevC = spec.clips[j - 1];
const nextC = spec.clips[j + 1];
@@ -1148,7 +1224,7 @@ export async function runExport(
} else {
timebase = c.image
? `[${idx}:v]setpts=PTS-STARTPTS`
- : `[${idx}:v]trim=${num(c.in)}:${num(c.out)},setpts=(PTS-STARTPTS)/${num(speed)}`;
+ : `[${idx}:v]trim=${num(c.in)}:${num(c.out)},setpts=${retimedPts(rt)}`;
}
if ((c.image || rmIn || videoPresence.get(c.file)) && !c.hidden) {
const region = regionPx(c.frame, W, H);
@@ -1319,7 +1395,6 @@ export async function runExport(
);
}
if (!c.muted && !c.hidden && audioPresence.get(c.file)) {
- const tempo = speed !== 1 ? `${atempoChain(speed)},` : "";
const vol = `${soundChain(c.sound)}${(c.volume ?? 1) !== 1 ? `volume=${num(c.volume ?? 1)},` : ""}`;
// The picture's fade edges carry the sound with them; zoom edges don't.
const afades =
@@ -1333,7 +1408,7 @@ export async function runExport(
crossExpr(dur, crossHalf[j + 1] ?? 0, false),
]);
filters.push(
- `[${idx}:a]atrim=${num(c.in)}:${num(c.out)},asetpts=PTS-STARTPTS,${tempo}` +
+ audioRead(c, 0, dur) +
`aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,${vol}` +
`apad=whole_dur=${num(dur)},atrim=0:${num(dur)}${afades}[a${j}]`
);
@@ -1426,18 +1501,16 @@ export async function runExport(
const handleStream = (
src: (typeof spec.clips)[number],
key: string,
- from: number,
- seconds: number,
+ fromT: number,
+ toT: number,
at: number,
expr: string
) => {
- if (seconds <= 0.01 || src.muted || src.hidden || src.image) return;
+ if (toT - fromT <= 0.01 || src.muted || src.hidden || src.image) return;
if (!audioPresence.get(src.file)) return;
- const rate = clipRate(src);
const lab = `xh${key}`;
filters.push(
- `[${inputIndex.get(src.file)!}:a]atrim=${num(from)}:${num(from + seconds * rate)},` +
- `asetpts=PTS-STARTPTS,${rate !== 1 ? `${atempoChain(rate)},` : ""}` +
+ audioRead(src, fromT, toT) +
`aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` +
`${soundChain(src.sound)}` +
`${(src.volume ?? 1) !== 1 ? `volume=${num(src.volume ?? 1)},` : ""}anull${crossFilters([expr])},` +
@@ -1453,11 +1526,12 @@ export async function runExport(
const ahead = Math.min(prevC.soundAhead ?? 0, half);
// Never reach past the head of the source: a handle is what the trim left
// behind, and there is none before zero.
- const back = Math.min(c.soundBack ?? 0, half, c.in / clipRate(c));
+ const back = Math.min(c.soundBack ?? 0, half, headRoom(c));
// The outgoing clip, still sounding past its out point.
- handleStream(prevC, `a${j}`, prevC.out, ahead, cut, crossExpr(0, half, false));
+ const prevDur = clipDur(prevC);
+ handleStream(prevC, `a${j}`, prevDur, prevDur + ahead, cut, crossExpr(0, half, false));
// The incoming clip, already sounding before its in point.
- handleStream(c, `b${j}`, c.in - back * clipRate(c), back, cut - back, crossExpr(back, half, true));
+ handleStream(c, `b${j}`, -back, 0, cut - back, crossExpr(back, half, true));
});
// Join the segments. Clips abut — a transition claims no layout — so every
@@ -1520,8 +1594,8 @@ export async function runExport(
const orIn = overlayRemovalInput.get(oc);
if (!oc.image && !orIn && !videoPresence.get(oc.file)) return onto;
const idx = oc.image ? imageOverlayInput.get(oc)! : inputIndex.get(oc.file)!;
- const ospeed = oc.speed && oc.speed > 0 ? oc.speed : 1;
- const olen = Math.max(0.1, (oc.out - oc.in) / ospeed);
+ const ort = retimeOf(oc);
+ const olen = spanLen(oc);
const end = Math.min(oc.start + olen, spec.duration);
const region = regionPx(oc.frame, W, H);
const cover = oc.fit === "fill" || (oc.fit == null && !region);
@@ -1588,7 +1662,7 @@ export async function runExport(
} else {
timebase = oc.image
? `[${idx}:v]setpts=PTS-STARTPTS`
- : `[${idx}:v]trim=${num(oc.in)}:${num(oc.out)},setpts=(PTS-STARTPTS)/${num(ospeed)}`;
+ : `[${idx}:v]trim=${num(oc.in)}:${num(oc.out)},setpts=${retimedPts(ort)}`;
}
let core = `${timebase},fps=${fps},${framing},setsar=1,${orIn ? "" : colorFix.get(oc.file) ?? ""}${gradeChain(orIn ? undefined : oc.grade)}format=${orIn ? "yuva420p" : lookFmt}`;
// Looks bake into footage overlays only: an image may carry alpha, which
@@ -1706,12 +1780,11 @@ export async function runExport(
filters.push(`[${onto}][${seg}]overlay=${pos}:${enable}:eof_action=pass[${next}]`);
}
if (!oc.muted && audioPresence.get(oc.file)) {
- const tempo = ospeed !== 1 ? `${atempoChain(ospeed)},` : "";
const vol = (oc.volume ?? 1) !== 1 ? `volume=${num(oc.volume ?? 1)},` : "";
// A cross dissolve reaches into the handle either side of its cut, so
// this stream starts before the clip's own head and runs past its tail;
// everything timed inside it shifts by that head reach.
- const back = Math.min(oc.soundBack ?? 0, hs, oc.in / ospeed);
+ const back = Math.min(oc.soundBack ?? 0, hs, headRoom(oc));
const ahead = Math.min(oc.soundAhead ?? 0, ts);
// The picture's fade edges carry the sound with them; zoom edges don't.
// A cross dissolve's ramps are equal-power and reach half level on the
@@ -1728,8 +1801,7 @@ export async function runExport(
const delayMs = Math.max(0, Math.round((oc.start - back) * 1000));
const lab = `ovs${k}`;
filters.push(
- `[${idx}:a]atrim=${num(oc.in - back * ospeed)}:${num(oc.out + ahead * ospeed)},` +
- `asetpts=PTS-STARTPTS,${tempo}` +
+ audioRead(oc, -back, olen + ahead) +
`aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` +
`${soundChain(oc.sound)}${vol}${afades}anull` +
`${crosses},adelay=${delayMs}:all=1[${lab}]`
@@ -1843,8 +1915,7 @@ export async function runExport(
const duckWindows = spec.audio
.filter((a) => a.duck !== undefined && a.duck < 1)
.map((a) => {
- const speed = a.speed && a.speed > 0 ? a.speed : 1;
- const len = Math.max(0.1, (a.out - a.in) / speed);
+ const len = spanLen(a);
return { from: a.start, to: a.start + len, gain: Math.max(0, a.duck!) };
});
// Flatten to non-overlapping segments at the lowest covering gain: chained
@@ -1880,20 +1951,17 @@ export async function runExport(
// Soundtrack clips: trim, gain, shift into place, mix with clip audio.
const soundLabels: string[] = [];
spec.audio.forEach((a, k) => {
- const idx = inputIndex.get(a.file)!;
if (!audioPresence.get(a.file)) return;
- const speed = a.speed && a.speed > 0 ? a.speed : 1;
const delayMs = Math.max(0, Math.round(a.start * 1000));
// Timeline length after any speed change; fade offsets are in these
- // (post-tempo) seconds, so atempo runs before the fades.
- const len = Math.max(0.1, (a.out - a.in) / speed);
- const tempo = speed !== 1 ? `${atempoChain(speed)},` : "";
+ // (post-tempo) seconds, so the tempo runs before the fades.
+ const len = spanLen(a);
const fades: string[] = [];
if (a.fadeIn && a.fadeIn > 0.01) fades.push(`afade=t=in:st=0:d=${num(a.fadeIn)}`);
if (a.fadeOut && a.fadeOut > 0.01)
fades.push(`afade=t=out:st=${num(Math.max(0, len - a.fadeOut))}:d=${num(a.fadeOut)}`);
filters.push(
- `[${idx}:a]atrim=${num(a.in)}:${num(a.out)},asetpts=PTS-STARTPTS,${tempo}` +
+ audioRead(a, 0, len) +
`aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,` +
`${soundChain(a.sound)}volume=${num(a.volume)},` +
(fades.length ? fades.join(",") + "," : "") +
diff --git a/site/src/cut/server/library.ts b/site/src/cut/server/library.ts
index 8c747be7..da33e519 100644
--- a/site/src/cut/server/library.ts
+++ b/site/src/cut/server/library.ts
@@ -83,6 +83,7 @@ export interface TemplateLayer {
opacity?: number;
muted: boolean;
speed?: number;
+ speedCurve?: [number, number][];
sound?: ClipSound;
track: number;
asClip?: boolean; // re-materializes as a track-0 timeline clip rather than an overlay
@@ -96,6 +97,7 @@ export interface TemplateAudio {
fadeIn?: number;
fadeOut?: number;
speed?: number;
+ speedCurve?: [number, number][];
sound?: ClipSound;
}
export interface LibraryTemplate {
diff --git a/site/src/cut/server/retimeAudio.ts b/site/src/cut/server/retimeAudio.ts
new file mode 100644
index 00000000..e602cdaf
--- /dev/null
+++ b/site/src/cut/server/retimeAudio.ts
@@ -0,0 +1,159 @@
+/**
+ * The sound of a clip whose rate changes through its footage, laid for
+ * ffmpeg.
+ *
+ * `atempo` takes one constant, and the bundled build (LGPL) carries nothing
+ * that takes a schedule. So a curved clip's audio is baked ahead of the graph:
+ * the trimmed span — widened by the handles a crossing reaches into — decodes
+ * to float PCM through ffmpeg, the same WSOLA the preview and the in-tab export
+ * run stretches it along the clip's own map, and the result lands as a WAV the
+ * graph reads like any other input, in timeline seconds and needing no tempo.
+ * One stretch implementation serves every renderer, which is the parity the
+ * export path promises.
+ */
+
+import type { Retime } from "@donkeycut/effects-kit";
+import { timeStretch } from "../lib/timeStretch";
+import { num } from "./util";
+
+/** The bake's sample rate: what the graphs resample every input to. */
+export const BAKE_RATE = 44100;
+const CHANNELS = 2;
+
+export interface BakedAudio {
+ /** The WAV on disk. */
+ file: string;
+ /** Timeline seconds the file holds before the clip's head (the back handle
+ * it was baked with, clamped to the source's start). */
+ back: number;
+ /** Timeline seconds the file runs. */
+ len: number;
+}
+
+export interface BakeIO {
+ /** Run ffmpeg with these args to completion. */
+ ffmpeg: (args: string[]) => Promise<void>;
+ readFile: (path: string) => Promise<Uint8Array>;
+ writeFile: (path: string, data: Uint8Array) => Promise<void>;
+ /** Drop the intermediate PCM once it is read. */
+ unlink: (path: string) => Promise<void>;
+}
+
+/**
+ * Bake `src`'s sound over the clip's span, plus `back` timeline seconds
+ * before its head and `ahead` after its tail, into `outFile`.
+ */
+export async function bakeRetimedAudio(
+ io: BakeIO,
+ src: string,
+ retime: Retime,
+ back: number,
+ ahead: number,
+ outFile: string
+): Promise<BakedAudio> {
+ const rt = retime;
+ // A handle cannot reach before the source's first sample; the head reach
+ // is whatever the source really had.
+ const from = Math.max(0, rt.srcAt(-Math.max(0, back)));
+ const to = Math.max(from, rt.srcAt(rt.len + Math.max(0, ahead)));
+ const pcm = `${outFile}.f32`;
+ await io.ffmpeg([
+ "-y",
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-ss",
+ num(from),
+ "-i",
+ src,
+ "-t",
+ num(Math.max(0.001, to - from)),
+ "-vn",
+ "-ac",
+ String(CHANNELS),
+ "-ar",
+ String(BAKE_RATE),
+ "-f",
+ "f32le",
+ pcm,
+ ]);
+ let raw: Uint8Array | null = await io.readFile(pcm);
+ await io.unlink(pcm).catch(() => {});
+ // A DataView reads the samples wherever the buffer starts, so the span is
+ // held once rather than copied to an aligned array first — this is minutes
+ // of audio on a long clip.
+ const view = new DataView(raw.buffer, raw.byteOffset, raw.byteLength);
+ const frames = Math.floor(raw.byteLength / 4 / CHANNELS);
+ const channels = Array.from({ length: CHANNELS }, () => new Float32Array(frames));
+ for (let i = 0; i < frames; i++)
+ for (let c = 0; c < CHANNELS; c++)
+ channels[c][i] = view.getFloat32((i * CHANNELS + c) * 4, true);
+ // The read buffer is dead once the channels hold the samples; letting go of
+ // it here keeps the stretch from running beside a second copy of the span.
+ raw = null;
+ // The file may run short of `to` — the source ended first — so the output
+ // length follows the samples actually read, through the same map.
+ const inSec = frames / BAKE_RATE;
+ const tFrom = rt.tAt(from);
+ const outLength = Math.max(1, Math.round((rt.tAt(from + inSec) - tFrom) * BAKE_RATE));
+ const out = timeStretch(channels, BAKE_RATE, {
+ factorAt: (sec) => 1 / rt.rateAtSrc(from + sec),
+ outLength,
+ });
+ await io.writeFile(outFile, wavFloat32(out, BAKE_RATE));
+ return { file: outFile, back: -tFrom, len: outLength / BAKE_RATE };
+}
+
+/** A WAV container around float samples: what ffmpeg reads back as pcm_f32le. */
+export function wavFloat32(channels: Float32Array[], sampleRate: number): Uint8Array {
+ const n = channels[0]?.length ?? 0;
+ const ch = Math.max(1, channels.length);
+ const dataBytes = n * ch * 4;
+ const buf = new ArrayBuffer(44 + dataBytes);
+ const view = new DataView(buf);
+ const ascii = (at: number, s: string) => {
+ for (let i = 0; i < s.length; i++) view.setUint8(at + i, s.charCodeAt(i));
+ };
+ ascii(0, "RIFF");
+ view.setUint32(4, 36 + dataBytes, true);
+ ascii(8, "WAVE");
+ ascii(12, "fmt ");
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 3, true); // IEEE float
+ view.setUint16(22, ch, true);
+ view.setUint32(24, sampleRate, true);
+ view.setUint32(28, sampleRate * ch * 4, true);
+ view.setUint16(32, ch * 4, true);
+ view.setUint16(34, 32, true);
+ ascii(36, "data");
+ view.setUint32(40, dataBytes, true);
+ const samples = new Float32Array(buf, 44, n * ch);
+ for (let i = 0; i < n; i++)
+ for (let c = 0; c < ch; c++) samples[i * ch + c] = channels[c]?.[i] ?? 0;
+ return new Uint8Array(buf);
+}
+
+/**
+ * The `setpts` expression that lays a clip's frames along its map: the
+ * reduced polyline of the retime as a sum of clipped ramps, so nothing nests
+ * however many knots the curve has. `T - STARTT` is the frame's seconds past
+ * the trim's first frame; the sum is its timeline second, over the timebase.
+ */
+export function setptsExpr(retime: Retime): string {
+ // Slopes carry six places: a rounded slope drifts by its error times the
+ // knot's width, and three places would put a long ramp a frame off.
+ const fnum = (n: number) => String(Math.round(n * 1e6) / 1e6);
+ const k = retime.knots;
+ const terms: string[] = [];
+ for (let i = 0; i + 1 < k.length; i++) {
+ const [s0, t0] = k[i];
+ const [s1, t1] = k[i + 1];
+ const w = s1 - s0;
+ if (w <= 0) continue;
+ const slope = (t1 - t0) / w;
+ const x = i === 0 ? "T-STARTT" : `T-STARTT-${fnum(s0)}`;
+ terms.push(`clip(${x},0,${fnum(w)})*${fnum(slope)}`);
+ }
+ if (terms.length === 0) return "PTS-STARTPTS";
+ return `(${terms.join("+")})/TB`;
+}
diff --git a/site/src/cut/server/transcribe.ts b/site/src/cut/server/transcribe.ts
index 0956ae6d..2e2f1bb8 100644
--- a/site/src/cut/server/transcribe.ts
+++ b/site/src/cut/server/transcribe.ts
@@ -1,5 +1,5 @@
import { spawn } from "node:child_process";
-import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, rm, stat, unlink, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { alignCues } from "../lib/cueAlign";
@@ -7,6 +7,8 @@ import { assertGraphSafe } from "./filterGraph";
import { assertLocalRuntime } from "./local-only";
import { createJobRegistry } from "./jobRegistry";
import { mediaPath, readProject } from "./projects";
+import { retimeOf, type SpeedNode } from "@donkeycut/effects-kit";
+import { bakeRetimedAudio, type BakedAudio } from "./retimeAudio";
import { atempoChain, findOnPath, hasStream, num, round } from "./util";
/** The audible slice of the cut, in timeline time (mirrors ExportSpec). */
@@ -20,10 +22,19 @@ export interface TranscribeSpec {
out: number;
muted: boolean;
speed?: number;
+ speedCurve?: SpeedNode[];
/** Cross-dissolve overlap into the next clip, timeline seconds. */
transition?: number;
}[];
- audio: { file: string; in: number; out: number; start: number; volume: number; speed?: number }[];
+ audio: {
+ file: string;
+ in: number;
+ out: number;
+ start: number;
+ volume: number;
+ speed?: number;
+ speedCurve?: SpeedNode[];
+ }[];
}
interface Word {
@@ -341,17 +352,47 @@ async function runTranscribe(job: TranscribeJob, spec: TranscribeSpec) {
// so the transcript's cue times line up with what the user sees. A gap
// spacer (no file) keeps its exact length — flooring it at 0.1s would
// land every cue after the gap late.
- const clipDur = (c: TranscribeSpec["clips"][number]) => {
- const speed = c.speed && c.speed > 0 ? c.speed : 1;
- return c.file ? Math.max(0.1, (c.out - c.in) / speed) : Math.max(0, c.out - c.in);
+ const clipDur = (c: TranscribeSpec["clips"][number]) =>
+ c.file ? Math.max(0.1, retimeOf(c).len) : Math.max(0, c.out - c.in);
+ // A clip whose rate changes through its footage is baked to a WAV in
+ // timeline seconds first (the same stretch the export and the preview
+ // use), so its words land where they play.
+ type Sounding = { file: string; in: number; out: number; speed?: number; speedCurve?: SpeedNode[] };
+ const baked = new Map<Sounding, BakedAudio & { idx: number }>();
+ const bakeIO = {
+ ffmpeg: async (args: string[]) => {
+ await run("ffmpeg", args);
+ },
+ readFile: (p: string) => readFile(p),
+ writeFile: (p: string, data: Uint8Array) => writeFile(p, data),
+ unlink: (p: string) => unlink(p),
+ };
+ const curved: { c: Sounding; tag: string }[] = [
+ ...spec.clips.filter((c) => !c.muted).map((c, j) => ({ c, tag: `clip_${j}` })),
+ ...spec.audio.map((a, k) => ({ c: a, tag: `snd_${k}` })),
+ ];
+ for (const { c, tag } of curved) {
+ const rt = retimeOf(c);
+ if (rt.uniform || !c.file || !audible.get(c.file)) continue;
+ const file = path.join(tmpDir, `retime_${tag}.wav`);
+ const bake = await bakeRetimedAudio(bakeIO, mediaPath(spec.projectId, c.file), rt, 0, 0, file);
+ const idx = inputs.length / 2;
+ inputs.push("-i", file);
+ baked.set(c, { ...bake, idx });
+ }
+ /** The head of a chain reading a clip's sound over its whole span. */
+ const audioRead = (c: Sounding, len: number) => {
+ const b = baked.get(c);
+ if (b) return `[${b.idx}:a]atrim=${num(b.back)}:${num(b.back + len)},asetpts=PTS-STARTPTS,`;
+ const rate = retimeOf(c).rate;
+ const tempo = rate !== 1 ? `${atempoChain(rate)},` : "";
+ return `[${inputIndex.get(c.file)}:a]atrim=${num(c.in)}:${num(c.out)},asetpts=PTS-STARTPTS,${tempo}`;
};
spec.clips.forEach((c, j) => {
- const speed = c.speed && c.speed > 0 ? c.speed : 1;
const dur = clipDur(c);
if (!c.muted && audible.get(c.file)) {
- const tempo = speed !== 1 ? `${atempoChain(speed)},` : "";
filters.push(
- `[${inputIndex.get(c.file)}:a]atrim=${num(c.in)}:${num(c.out)},asetpts=PTS-STARTPTS,${tempo}` +
+ audioRead(c, dur) +
`aresample=16000,aformat=sample_fmts=s16:channel_layouts=mono,` +
`apad=whole_dur=${num(dur)},atrim=0:${num(dur)}[a${j}]`
);
@@ -391,10 +432,8 @@ async function runTranscribe(job: TranscribeJob, spec: TranscribeSpec) {
const soundLabels: string[] = [];
spec.audio.forEach((a, k) => {
if (!audible.get(a.file)) return;
- const speed = a.speed && a.speed > 0 ? a.speed : 1;
- const tempo = speed !== 1 ? `${atempoChain(speed)},` : "";
filters.push(
- `[${inputIndex.get(a.file)}:a]atrim=${num(a.in)}:${num(a.out)},asetpts=PTS-STARTPTS,${tempo}` +
+ audioRead(a, Math.max(0.1, retimeOf(a).len)) +
`aresample=16000,aformat=sample_fmts=s16:channel_layouts=mono,` +
`volume=${num(a.volume)},adelay=${Math.max(0, Math.round(a.start * 1000))}:all=1[snd${k}]`
);