Skip to content

Open source CapCut alternative.

TypeScript

feat(site): smooth slow motion, interpolated by ffmpeg over a clip's slow runs [rebuild]

David Leddbd99618470
diff --git a/site/packages/effects-kit/src/retime.test.ts b/site/packages/effects-kit/src/retime.test.ts
index 3113cc63..0ec91304 100644
--- a/site/packages/effects-kit/src/retime.test.ts
+++ b/site/packages/effects-kit/src/retime.test.ts
@@ -3,8 +3,11 @@ import {
flatSpeedCurve,
mirrorRetimable,
retimeOf,
+ slowRuns,
+ smoothsAt,
speedCurveOf,
speedCurvePreset,
+ speedCurvePresetOf,
SPEED_CURVE_MAX,
SPEED_CURVE_MIN,
type SpeedNode,
@@ -77,6 +80,21 @@ describe("retimeOf", () => {
}
});
+ test("a flat curve is four even nodes at the clip's rate", () => {
+ expect(flatSpeedCurve({ in: 1, out: 7, speed: 2 })).toEqual([
+ [1, 2],
+ [3, 2],
+ [5, 2],
+ [7, 2],
+ ]);
+ });
+
+ test("a curve flat at 1× reads as the Flat preset whatever its node count", () => {
+ expect(speedCurvePresetOf([[0, 1], [4, 1]], 0, 4)).toBe("flat");
+ expect(speedCurvePresetOf(flatSpeedCurve({ in: 0, out: 4 }), 0, 4)).toBe("flat");
+ expect(speedCurvePresetOf([[0, 2], [4, 2]], 0, 4)).toBeUndefined();
+ });
+
test("a flat curve equals the uniform map", () => {
const u = retimeOf({ in: 1, out: 7, speed: 2 });
const c = retimeOf({ in: 1, out: 7, speedCurve: flatSpeedCurve({ in: 1, out: 7, speed: 2 }) });
@@ -224,3 +242,40 @@ describe("speedCurvePreset", () => {
expect(speedCurvePreset("nope", 0, 1)).toBeUndefined();
});
});
+
+describe("slowRuns", () => {
+ test("a uniform span is one run when slow and none when not", () => {
+ expect(slowRuns(retimeOf({ in: 0, out: 4, speed: 0.5 }), 1 / 30)).toEqual([[0, 8]]);
+ expect(slowRuns(retimeOf({ in: 0, out: 4, speed: 1 }), 1 / 30)).toEqual([]);
+ expect(slowRuns(retimeOf({ in: 0, out: 4, speed: 2 }), 1 / 30)).toEqual([]);
+ });
+
+ test("a curve that dips and races past 1× is slow in the dip alone", () => {
+ const rt = retimeOf({
+ in: 0,
+ out: 6,
+ speedCurve: [
+ [0, 1],
+ [2, 0.4],
+ [4, 3],
+ [6, 3],
+ ],
+ });
+ const runs = slowRuns(rt, 1 / 30);
+ expect(runs.length).toBe(1);
+ const [[from, to]] = runs;
+ expect(from).toBeGreaterThan(0);
+ expect(to).toBeLessThan(rt.len);
+ // Inside the run the rate is under 1×, and just outside it is not.
+ expect(rt.rateAt((from + to) / 2)).toBeLessThan(1);
+ expect(rt.rateAt(from - 0.1)).toBeGreaterThanOrEqual(1 - 1e-3);
+ expect(rt.rateAt(to + 0.1)).toBeGreaterThanOrEqual(1 - 1e-3);
+ });
+
+ test("smoothsAt is on only with the flag and under 1×", () => {
+ const rt = retimeOf({ in: 0, out: 4, speed: 0.5 });
+ expect(smoothsAt({ smoothSlow: true }, rt, 1)).toBe(true);
+ expect(smoothsAt({}, rt, 1)).toBe(false);
+ expect(smoothsAt({ smoothSlow: true }, retimeOf({ in: 0, out: 4, speed: 1 }), 1)).toBe(false);
+ });
+});
diff --git a/site/packages/effects-kit/src/retime.ts b/site/packages/effects-kit/src/retime.ts
index 6dcba4df..f49f433e 100644
--- a/site/packages/effects-kit/src/retime.ts
+++ b/site/packages/effects-kit/src/retime.ts
@@ -301,6 +301,41 @@ export function srcSpan(rt: Retime, tFrom: number, tTo: number): { lo: number; h
return a <= b ? { lo: a, hi: b } : { lo: b, hi: a };
}
+/** Rate below which a smoothed clip synthesizes frames. Exactly 1× shows
+ * every source frame once; only a slower stretch repeats them. */
+export const SLOW_RATE = 1 - 1e-3;
+
+/** Whether a clip's picture is synthesized at timeline offset `tLocal`: the
+ * clip smooths its slow motion, and the map runs slower than 1× there. */
+export const smoothsAt = (c: { smoothSlow?: boolean }, rt: Retime, tLocal: number) =>
+ !!c.smoothSlow && rt.rateAt(tLocal) < SLOW_RATE;
+
+/**
+ * The stretches of a span, in timeline seconds from its head, where the map
+ * runs slower than 1×: `[from, to]` pairs, ascending, sampled every `step`
+ * seconds (an output frame) with runs shorter than two samples dropped. A
+ * uniform slow span is one run over its whole length; a curve that dips and
+ * recovers gives one run per dip. A renderer that cannot ask the rate per
+ * frame (an ffmpeg graph) interpolates exactly these.
+ */
+export function slowRuns(rt: Retime, step: number): [number, number][] {
+ const runs: [number, number][] = [];
+ if (!(step > 0) || rt.len <= 0) return runs;
+ if (rt.uniform) return rt.rate < SLOW_RATE ? [[0, rt.len]] : runs;
+ let open: number | null = null;
+ const n = Math.ceil(rt.len / step);
+ for (let i = 0; i <= n; i++) {
+ const t = Math.min(rt.len, i * step);
+ const slow = t < rt.len && rt.rateAt(t) < SLOW_RATE;
+ if (slow && open === null) open = t;
+ else if (!slow && open !== null) {
+ if (t - open >= 2 * step - 1e-9) runs.push([open, t]);
+ open = null;
+ }
+ }
+ return runs;
+}
+
/**
* The same span over media that has been turned around: source second `s`
* of the original sits at `pivot − s` in the turned copy. A reversed clip
@@ -318,14 +353,14 @@ export function mirrorRetimable<T extends Retimable>(c: T, pivot: number): T {
};
}
-/** A flat curve at the clip's current rate, one node on each edge of the
- * span — what a clip starts from when it enters curve editing. */
+/** A flat curve at the clip's current rate: four nodes spaced evenly
+ * through the span, one on each edge and two between — what a clip starts
+ * from when it enters curve editing, with handles already where a ramp
+ * usually wants them. */
export function flatSpeedCurve(c: Retimable): SpeedNode[] {
const rate = clampRate(uniformRate(c));
- return [
- [c.in, rate],
- [c.out, rate],
- ];
+ const span = Math.max(0, c.out - c.in);
+ return [0, 1 / 3, 2 / 3, 1].map((x): SpeedNode => [c.in + x * span, rate]);
}
/* ------------------------------------------------------------------ */
@@ -353,6 +388,8 @@ export const SPEED_CURVE_PRESETS: SpeedCurvePreset[] = [
hint: "one rate across the clip",
shape: [
[0, 1],
+ [1 / 3, 1],
+ [2 / 3, 1],
[1, 1],
],
},
@@ -461,6 +498,10 @@ export function speedCurvePreset(id: string, inS: number, outS: number): SpeedNo
export function speedCurvePresetOf(nodes: SpeedNode[], inS: number, outS: number): string | undefined {
const span = Math.max(0, outS - inS);
if (span <= 0) return undefined;
+ // Flat is a rate, not a node count: a curve at 1× everywhere is the Flat
+ // preset however many nodes carry it, so one seeded before the preset grew
+ // its middle nodes still reads as Flat.
+ if (nodes.length >= 2 && nodes.every(([, rate]) => Math.abs(rate - 1) < 1e-3)) return "flat";
return SPEED_CURVE_PRESETS.find(
(p) =>
p.shape.length === nodes.length &&
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index c8954e34..aac8c5d5 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -805,6 +805,7 @@ export async function buildExportPayload(
speed: clipSpeed(sp.clip),
speedCurve: sp.clip.speedCurve,
reverse: sp.clip.reverse,
+ smoothSlow: sp.clip.smoothSlow,
transition: sp.transitionOut,
// A cross dissolve carries its own window: the picture cuts, so the
// server's blend must not see one. The handles are what the crossing
@@ -1010,6 +1011,7 @@ export async function buildExportPayload(
speed: c.speed,
speedCurve: c.speedCurve,
reverse: c.reverse,
+ smoothSlow: c.smoothSlow,
image: assetById.get(c.assetId)!.type === "image",
grade: normalizeGrade(c.grade),
look: c.look,
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index c3207b39..158d9309 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -529,6 +529,9 @@ export interface EditorState {
setClipSpeedCurve: (id: string, nodes: SpeedNode[] | undefined) => void;
/** Lay a preset ramp over the clip's trimmed span. */
setClipSpeedPreset: (id: string, preset: string) => void;
+ /** Smooth the clip's slow motion: synthesize the frames between source
+ * frames wherever its rate runs below 1×. */
+ setClipSmoothSlow: (id: string, on: boolean) => void;
/** Play the clip's footage backward (picture and sound); the footprint
* stays. */
setClipReverse: (id: string, reverse: boolean) => void;
@@ -2865,6 +2868,12 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
get().setClipSpeedCurve(id, nodes);
},
+ setClipSmoothSlow: (id, on) => {
+ const clip = get().clips.find((c) => c.id === id);
+ if (!clip || !!clip.smoothSlow === on) return;
+ get().updateClip(id, { smoothSlow: on || undefined });
+ },
+
setClipReverse: (id, reverse) => {
const clip = get().clips.find((c) => c.id === id);
if (!clip || !!clip.reverse === reverse) return;
@@ -3913,13 +3922,13 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
if (mi == null) continue;
// Track-0 clips re-materialize onto track 0 (asClip), so a template
// stands up its own video instead of an empty timeline.
- layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, speedCurve: sp.clip.speedCurve, reverse: sp.clip.reverse, sound: sp.clip.sound, track: 1, asClip: true });
+ layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, speedCurve: sp.clip.speedCurve, reverse: sp.clip.reverse, smoothSlow: sp.clip.smoothSlow, sound: sp.clip.sound, track: 1, asClip: true });
} else {
const c = s.clips.find((x) => x.id === sel.id);
if (!c) continue;
const mi = mediaFor(c.assetId);
if (mi == null) continue;
- layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, speedCurve: c.speedCurve, reverse: c.reverse, sound: c.sound, track: c.track + 1 });
+ layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, speedCurve: c.speedCurve, reverse: c.reverse, smoothSlow: c.smoothSlow, sound: c.sound, track: c.track + 1 });
}
} else if (sel.kind === "audio") {
const c = s.audioClips.find((x) => x.id === sel.id);
@@ -4058,6 +4067,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
...(l.speed ? { speed: l.speed } : {}),
...(l.speedCurve ? { speedCurve: l.speedCurve } : {}),
...(l.reverse ? { reverse: true } : {}),
+ ...(l.smoothSlow ? { smoothSlow: true } : {}),
...(l.sound ? { sound: l.sound } : {}),
}));
const topTrack = Math.max(0, ...overlayLayers(get().clips).map((c) => c.track));
@@ -4078,6 +4088,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
...(l.speed ? { speed: l.speed } : {}),
...(l.speedCurve ? { speedCurve: l.speedCurve } : {}),
...(l.reverse ? { reverse: true } : {}),
+ ...(l.smoothSlow ? { smoothSlow: true } : {}),
...(l.sound ? { sound: l.sound } : {}),
}));
const newAudio: AudioClip[] = template.audio
diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.ts
index 186c8527..5e107ee7 100644
--- a/site/src/cut/lib/types.ts
+++ b/site/src/cut/lib/types.ts
@@ -554,6 +554,12 @@ export interface VideoClip {
* tail shows `in`, picture and sound alike. The rate still applies, so the
* footprint is unchanged. */
reverse?: boolean;
+ /** Smooth slow motion: wherever the rate runs below 1×, the frames between
+ * two source frames are synthesized so the picture moves at the output
+ * rate. At 1× and faster the footage shows its own frames. The preview
+ * and the tab's export blend the two neighbours; ffmpeg exports estimate
+ * the motion between them. */
+ smoothSlow?: boolean;
/** Transition into the next clip on this clip's track, in timeline seconds
* (absent/0 = hard cut). Every style overlaps the two clips by this much,
* so the cut shortens. On upper tracks a transition blends the incoming
@@ -1044,6 +1050,7 @@ export interface TemplateLayer {
speed?: number;
speedCurve?: SpeedNode[];
reverse?: boolean;
+ smoothSlow?: boolean;
sound?: ClipSound;
track: number;
/** Came from video track 0 — re-materializes as a timeline clip, not an
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.ts
index 8a0f82ed..969d4488 100644
--- a/site/src/cut/server/exportPipeline.test.ts
+++ b/site/src/cut/server/exportPipeline.test.ts
@@ -355,6 +355,44 @@ describe("a speed curve in the filtergraph", () => {
});
});
+describe("smooth slow motion in the filtergraph", () => {
+ test("a slowed clip runs motion interpolation over its whole span", async () => {
+ const g = await graphFor({ clips: [clip("a.mp4", { speed: 0.5, smoothSlow: true }), clip("b.mp4")] });
+ const video = g.find((f) => f.startsWith("[0:v]trim="))!;
+ expect(video).toContain("minterpolate=fps=30:mi_mode=mci");
+ expect(video).not.toContain("split=");
+ // The plain clip beside it is untouched.
+ expect(g.find((f) => f.startsWith("[1:v]trim="))).not.toContain("minterpolate");
+ });
+
+ test("a curve that dips is interpolated in the dip alone", async () => {
+ const curved = clip("a.mp4", {
+ smoothSlow: true,
+ speedCurve: [
+ [0, 1],
+ [1.5, 0.4],
+ [3, 3],
+ [4, 3],
+ ],
+ });
+ const g = await graphFor({ clips: [curved, clip("b.mp4")] });
+ const head = g.find((f) => f.startsWith("[0:v]trim="))!;
+ expect(head).toContain("split=3[smic0_0][smic0_1][smic0_2]");
+ const pieces = g.filter((f) => f.startsWith("[smic0_"));
+ expect(pieces.length).toBe(3);
+ expect(pieces[0]).toContain("fps=30[smo");
+ expect(pieces[0]).not.toContain("minterpolate");
+ expect(pieces[1]).toContain("minterpolate=fps=30");
+ expect(pieces[2]).not.toContain("minterpolate");
+ expect(g.some((f) => f.includes("concat=n=3:v=1:a=0,fps=30[smoc0]"))).toBe(true);
+ });
+
+ test("a clip at 1× or faster keeps its plain chain with the flag on", async () => {
+ const g = await graphFor({ clips: [clip("a.mp4", { speed: 2, smoothSlow: true })] });
+ expect(g.join(";")).not.toContain("minterpolate");
+ });
+});
+
describe("clip sound in the filtergraph", () => {
const sound = {
eq: [2, 1, -2, -1, 1.5, 1, -1],
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index 05c65767..5e869fd0 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -7,7 +7,7 @@ import { assertGraphSafe, fexpr } from "./filterGraph";
import { bakeRetimedAudio, setptsExpr, type BakedAudio } from "./retimeAudio";
import { bakeTurnedMedia } from "./turnMedia";
import { CLIP_MAX_ZOOM, projectFadeSeconds, regionPx, TRANSITION_XFADE, TRANSITION_ZOOM, type ColorGrade, type TransitionStyle } from "../lib/types";
-import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, mirrorRetimable, retimeOf, shortestTurn, srcSpan, sortedKeys, soundFilters, type ClipSound, type OverlayKey, type Retime, type SpeedNode } from "@donkeycut/effects-kit";
+import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, mirrorRetimable, retimeOf, shortestTurn, slowRuns, srcSpan, sortedKeys, soundFilters, type ClipSound, type OverlayKey, type Retime, type SpeedNode } from "@donkeycut/effects-kit";
// The render pipeline itself: spec in, finished mp4 out. Shared by the local
// engine's job registry (jobs.ts) and the cloud render worker, which stage
@@ -101,6 +101,8 @@ export interface ExportSpec {
* (see retimeOf); present, `speed` is ignored. */
speedCurve?: SpeedNode[];
reverse?: boolean;
+ /** Synthesize the frames wherever the rate runs below 1× (see slowRuns). */
+ smoothSlow?: boolean;
/** Transition into the next clip, in timeline seconds (overlap). */
transition?: number;
/** Half the cross dissolve into the next clip, in timeline seconds: the
@@ -192,6 +194,8 @@ export interface ExportSpec {
* (see retimeOf); present, `speed` is ignored. */
speedCurve?: SpeedNode[];
reverse?: boolean;
+ /** Synthesize the frames wherever the rate runs below 1× (see slowRuns). */
+ smoothSlow?: boolean;
/** Transition ramps, timeline seconds from this overlay's head/tail. On
* an upper track a fade is an alpha fade (the tracks beneath show
* through); a cross transition arrives as the incoming clip's headFade
@@ -483,6 +487,60 @@ const retimedPts = (rt: Retime) =>
rt.uniform ? `(PTS-STARTPTS)/${num(rt.rate)}` : fexpr(setptsExpr(rt));
/** Timeline length of a media clip's span through its rate or curve. */
+/** ffmpeg's motion-compensated interpolation, laying `fps` frames over the
+ * sparse ones a slowed span has: bidirectional block motion estimation and
+ * overlapped-block compensation, with its scene-change guard on so a cut
+ * inside the footage is never blended across. */
+const MOTION_INTERPOLATE = "mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1:scd=fdiff";
+
+/**
+ * The head of a footage clip's picture chain: its retimed, framed source at
+ * `fps`. A clip smoothing its slow motion runs ffmpeg's motion interpolation
+ * over exactly the stretches its rate is under 1× — the span is cut into
+ * pieces at those boundaries, each piece re-stamped to `fps` on its own
+ * (interpolated or plain), and the pieces joined back — so a curve that
+ * dips and races past 1× is smoothed in the dip alone. The framing runs
+ * first, so the estimate works at the output size on the frames the slow
+ * stretch actually has.
+ */
+function framedTimebase(
+ timebase: string,
+ framing: string,
+ tag: string,
+ c: { smoothSlow?: boolean; image?: boolean },
+ rt: Retime,
+ fps: number,
+ filters: string[]
+): string {
+ const plain = `${timebase},fps=${fps},${framing},setsar=1`;
+ if (c.image || !c.smoothSlow) return plain;
+ const runs = slowRuns(rt, 1 / fps);
+ if (runs.length === 0) return plain;
+ const interp = `minterpolate=fps=${fps}:${MOTION_INTERPOLATE}`;
+ const framed = `${timebase},${framing},setsar=1`;
+ if (runs.length === 1 && runs[0][0] <= 1e-6 && runs[0][1] >= rt.len - 1e-6) {
+ return `${framed},${interp}`;
+ }
+ const pieces: { from: number; to: number; slow: boolean }[] = [];
+ let at = 0;
+ for (const [from, to] of runs) {
+ if (from > at + 1e-6) pieces.push({ from: at, to: from, slow: false });
+ pieces.push({ from, to, slow: true });
+ at = to;
+ }
+ if (rt.len > at + 1e-6) pieces.push({ from: at, to: rt.len, slow: false });
+ filters.push(`${framed},split=${pieces.length}${pieces.map((_, k) => `[smi${tag}_${k}]`).join("")}`);
+ pieces.forEach((pc, k) => {
+ filters.push(
+ `[smi${tag}_${k}]trim=${num(pc.from)}:${num(pc.to)},setpts=PTS-STARTPTS,${pc.slow ? interp : `fps=${fps}`}[smo${tag}_${k}]`
+ );
+ });
+ filters.push(
+ `${pieces.map((_, k) => `[smo${tag}_${k}]`).join("")}concat=n=${pieces.length}:v=1:a=0,fps=${fps}[smo${tag}]`
+ );
+ return `[smo${tag}]null`;
+}
+
const spanLen = (c: { in: number; out: number; speed?: number; speedCurve?: SpeedNode[]; reverse?: boolean }) =>
Math.max(0.1, retimeOf(c).len);
@@ -1448,7 +1506,7 @@ export async function runExport(
// a still just replays its looped input.
// The grade sits after the color conversion (so it acts on the same
// BT.709 values the preview shows) and before the terminal format.
- let core = `${timebase},fps=${fps},${frame},setsar=1,${rmIn ? "" : colorFix.get(c.file) ?? ""}${gradeChain(rmIn ? undefined : c.grade)}format=${segFmt}`;
+ let core = `${framedTimebase(timebase, frame, `c${j}`, rmIn ? {} : c, rt, fps, filters)},${rmIn ? "" : colorFix.get(c.file) ?? ""}${gradeChain(rmIn ? undefined : c.grade)}format=${segFmt}`;
// The look bakes in after grade + framing, before the edge effects, so
// animations move already-graded pixels (matching the preview order). A
// removal clip's look runs after the flatten below instead — the chain's
@@ -1851,7 +1909,7 @@ export async function runExport(
? `[${idx}:v]setpts=PTS-STARTPTS`
: `[${idx}:v]trim=${num(oc.in)}:${num(oc.out)},setpts=${retimedPts(ort)}`;
}
- let core = `${timebase},fps=${fps},${framing},setsar=1,${orIn ? "" : colorFix.get(oc.file) ?? ""}${gradeChain(orIn ? undefined : oc.grade)}format=${orIn ? "yuva420p" : lookFmt}`;
+ let core = `${framedTimebase(timebase, framing, `o${k}`, orIn ? {} : oc, ort, fps, filters)},${orIn ? "" : colorFix.get(oc.file) ?? ""}${gradeChain(orIn ? undefined : oc.grade)}format=${orIn ? "yuva420p" : lookFmt}`;
// Looks bake into footage overlays only: an image may carry alpha, which
// the look chain's internal filters would flatten onto black over the
// tracks beneath. The alpha fades stay safe: they apply after the look.