Open source CapCut alternative.
TypeScript
feat(site): smooth slow motion, interpolated by ffmpeg over a clip's slow runs [rebuild]
David Leddbd99618470
diff --git a/site/packages/effects-kit/src/retime.test.ts b/site/packages/effects-kit/src/retime.test.tsindex 3113cc63..0ec91304 100644--- a/site/packages/effects-kit/src/retime.test.ts+++ b/site/packages/effects-kit/src/retime.test.ts@@ -3,8 +3,11 @@ import {flatSpeedCurve,mirrorRetimable,retimeOf,+ slowRuns,+ smoothsAt,speedCurveOf,speedCurvePreset,+ speedCurvePresetOf,SPEED_CURVE_MAX,SPEED_CURVE_MIN,type SpeedNode,@@ -77,6 +80,21 @@ describe("retimeOf", () => {}});+ test("a flat curve is four even nodes at the clip's rate", () => {+ expect(flatSpeedCurve({ in: 1, out: 7, speed: 2 })).toEqual([+ [1, 2],+ [3, 2],+ [5, 2],+ [7, 2],+ ]);+ });++ test("a curve flat at 1× reads as the Flat preset whatever its node count", () => {+ expect(speedCurvePresetOf([[0, 1], [4, 1]], 0, 4)).toBe("flat");+ expect(speedCurvePresetOf(flatSpeedCurve({ in: 0, out: 4 }), 0, 4)).toBe("flat");+ expect(speedCurvePresetOf([[0, 2], [4, 2]], 0, 4)).toBeUndefined();+ });+test("a flat curve equals the uniform map", () => {const u = retimeOf({ in: 1, out: 7, speed: 2 });const c = retimeOf({ in: 1, out: 7, speedCurve: flatSpeedCurve({ in: 1, out: 7, speed: 2 }) });@@ -224,3 +242,40 @@ describe("speedCurvePreset", () => {expect(speedCurvePreset("nope", 0, 1)).toBeUndefined();});});++describe("slowRuns", () => {+ test("a uniform span is one run when slow and none when not", () => {+ expect(slowRuns(retimeOf({ in: 0, out: 4, speed: 0.5 }), 1 / 30)).toEqual([[0, 8]]);+ expect(slowRuns(retimeOf({ in: 0, out: 4, speed: 1 }), 1 / 30)).toEqual([]);+ expect(slowRuns(retimeOf({ in: 0, out: 4, speed: 2 }), 1 / 30)).toEqual([]);+ });++ test("a curve that dips and races past 1× is slow in the dip alone", () => {+ const rt = retimeOf({+ in: 0,+ out: 6,+ speedCurve: [+ [0, 1],+ [2, 0.4],+ [4, 3],+ [6, 3],+ ],+ });+ const runs = slowRuns(rt, 1 / 30);+ expect(runs.length).toBe(1);+ const [[from, to]] = runs;+ expect(from).toBeGreaterThan(0);+ expect(to).toBeLessThan(rt.len);+ // Inside the run the rate is under 1×, and just outside it is not.+ expect(rt.rateAt((from + to) / 2)).toBeLessThan(1);+ expect(rt.rateAt(from - 0.1)).toBeGreaterThanOrEqual(1 - 1e-3);+ expect(rt.rateAt(to + 0.1)).toBeGreaterThanOrEqual(1 - 1e-3);+ });++ test("smoothsAt is on only with the flag and under 1×", () => {+ const rt = retimeOf({ in: 0, out: 4, speed: 0.5 });+ expect(smoothsAt({ smoothSlow: true }, rt, 1)).toBe(true);+ expect(smoothsAt({}, rt, 1)).toBe(false);+ expect(smoothsAt({ smoothSlow: true }, retimeOf({ in: 0, out: 4, speed: 1 }), 1)).toBe(false);+ });+});diff --git a/site/packages/effects-kit/src/retime.ts b/site/packages/effects-kit/src/retime.tsindex 6dcba4df..f49f433e 100644--- a/site/packages/effects-kit/src/retime.ts+++ b/site/packages/effects-kit/src/retime.ts@@ -301,6 +301,41 @@ export function srcSpan(rt: Retime, tFrom: number, tTo: number): { lo: number; hreturn a <= b ? { lo: a, hi: b } : { lo: b, hi: a };}+/** Rate below which a smoothed clip synthesizes frames. Exactly 1× shows+ * every source frame once; only a slower stretch repeats them. */+export const SLOW_RATE = 1 - 1e-3;++/** Whether a clip's picture is synthesized at timeline offset `tLocal`: the+ * clip smooths its slow motion, and the map runs slower than 1× there. */+export const smoothsAt = (c: { smoothSlow?: boolean }, rt: Retime, tLocal: number) =>+ !!c.smoothSlow && rt.rateAt(tLocal) < SLOW_RATE;++/**+ * The stretches of a span, in timeline seconds from its head, where the map+ * runs slower than 1×: `[from, to]` pairs, ascending, sampled every `step`+ * seconds (an output frame) with runs shorter than two samples dropped. A+ * uniform slow span is one run over its whole length; a curve that dips and+ * recovers gives one run per dip. A renderer that cannot ask the rate per+ * frame (an ffmpeg graph) interpolates exactly these.+ */+export function slowRuns(rt: Retime, step: number): [number, number][] {+ const runs: [number, number][] = [];+ if (!(step > 0) || rt.len <= 0) return runs;+ if (rt.uniform) return rt.rate < SLOW_RATE ? [[0, rt.len]] : runs;+ let open: number | null = null;+ const n = Math.ceil(rt.len / step);+ for (let i = 0; i <= n; i++) {+ const t = Math.min(rt.len, i * step);+ const slow = t < rt.len && rt.rateAt(t) < SLOW_RATE;+ if (slow && open === null) open = t;+ else if (!slow && open !== null) {+ if (t - open >= 2 * step - 1e-9) runs.push([open, t]);+ open = null;+ }+ }+ return runs;+}+/*** The same span over media that has been turned around: source second `s`* of the original sits at `pivot − s` in the turned copy. A reversed clip@@ -318,14 +353,14 @@ export function mirrorRetimable<T extends Retimable>(c: T, pivot: number): T {};}-/** A flat curve at the clip's current rate, one node on each edge of the- * span — what a clip starts from when it enters curve editing. */+/** A flat curve at the clip's current rate: four nodes spaced evenly+ * through the span, one on each edge and two between — what a clip starts+ * from when it enters curve editing, with handles already where a ramp+ * usually wants them. */export function flatSpeedCurve(c: Retimable): SpeedNode[] {const rate = clampRate(uniformRate(c));- return [- [c.in, rate],- [c.out, rate],- ];+ const span = Math.max(0, c.out - c.in);+ return [0, 1 / 3, 2 / 3, 1].map((x): SpeedNode => [c.in + x * span, rate]);}/* ------------------------------------------------------------------ */@@ -353,6 +388,8 @@ export const SPEED_CURVE_PRESETS: SpeedCurvePreset[] = [hint: "one rate across the clip",shape: [[0, 1],+ [1 / 3, 1],+ [2 / 3, 1],[1, 1],],},@@ -461,6 +498,10 @@ export function speedCurvePreset(id: string, inS: number, outS: number): SpeedNoexport function speedCurvePresetOf(nodes: SpeedNode[], inS: number, outS: number): string | undefined {const span = Math.max(0, outS - inS);if (span <= 0) return undefined;+ // Flat is a rate, not a node count: a curve at 1× everywhere is the Flat+ // preset however many nodes carry it, so one seeded before the preset grew+ // its middle nodes still reads as Flat.+ if (nodes.length >= 2 && nodes.every(([, rate]) => Math.abs(rate - 1) < 1e-3)) return "flat";return SPEED_CURVE_PRESETS.find((p) =>p.shape.length === nodes.length &&diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.tsindex c8954e34..aac8c5d5 100644--- a/site/src/cut/lib/exportClient.ts+++ b/site/src/cut/lib/exportClient.ts@@ -805,6 +805,7 @@ export async function buildExportPayload(speed: clipSpeed(sp.clip),speedCurve: sp.clip.speedCurve,reverse: sp.clip.reverse,+ smoothSlow: sp.clip.smoothSlow,transition: sp.transitionOut,// A cross dissolve carries its own window: the picture cuts, so the// server's blend must not see one. The handles are what the crossing@@ -1010,6 +1011,7 @@ export async function buildExportPayload(speed: c.speed,speedCurve: c.speedCurve,reverse: c.reverse,+ smoothSlow: c.smoothSlow,image: assetById.get(c.assetId)!.type === "image",grade: normalizeGrade(c.grade),look: c.look,diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex c3207b39..158d9309 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -529,6 +529,9 @@ export interface EditorState {setClipSpeedCurve: (id: string, nodes: SpeedNode[] | undefined) => void;/** Lay a preset ramp over the clip's trimmed span. */setClipSpeedPreset: (id: string, preset: string) => void;+ /** Smooth the clip's slow motion: synthesize the frames between source+ * frames wherever its rate runs below 1×. */+ setClipSmoothSlow: (id: string, on: boolean) => void;/** Play the clip's footage backward (picture and sound); the footprint* stays. */setClipReverse: (id: string, reverse: boolean) => void;@@ -2865,6 +2868,12 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {get().setClipSpeedCurve(id, nodes);},+ setClipSmoothSlow: (id, on) => {+ const clip = get().clips.find((c) => c.id === id);+ if (!clip || !!clip.smoothSlow === on) return;+ get().updateClip(id, { smoothSlow: on || undefined });+ },+setClipReverse: (id, reverse) => {const clip = get().clips.find((c) => c.id === id);if (!clip || !!clip.reverse === reverse) return;@@ -3913,13 +3922,13 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {if (mi == null) continue;// Track-0 clips re-materialize onto track 0 (asClip), so a template// stands up its own video instead of an empty timeline.- layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, speedCurve: sp.clip.speedCurve, reverse: sp.clip.reverse, sound: sp.clip.sound, track: 1, asClip: true });+ layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, speedCurve: sp.clip.speedCurve, reverse: sp.clip.reverse, smoothSlow: sp.clip.smoothSlow, sound: sp.clip.sound, track: 1, asClip: true });} else {const c = s.clips.find((x) => x.id === sel.id);if (!c) continue;const mi = mediaFor(c.assetId);if (mi == null) continue;- layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, speedCurve: c.speedCurve, reverse: c.reverse, sound: c.sound, track: c.track + 1 });+ layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, speedCurve: c.speedCurve, reverse: c.reverse, smoothSlow: c.smoothSlow, sound: c.sound, track: c.track + 1 });}} else if (sel.kind === "audio") {const c = s.audioClips.find((x) => x.id === sel.id);@@ -4058,6 +4067,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {...(l.speed ? { speed: l.speed } : {}),...(l.speedCurve ? { speedCurve: l.speedCurve } : {}),...(l.reverse ? { reverse: true } : {}),+ ...(l.smoothSlow ? { smoothSlow: true } : {}),...(l.sound ? { sound: l.sound } : {}),}));const topTrack = Math.max(0, ...overlayLayers(get().clips).map((c) => c.track));@@ -4078,6 +4088,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {...(l.speed ? { speed: l.speed } : {}),...(l.speedCurve ? { speedCurve: l.speedCurve } : {}),...(l.reverse ? { reverse: true } : {}),+ ...(l.smoothSlow ? { smoothSlow: true } : {}),...(l.sound ? { sound: l.sound } : {}),}));const newAudio: AudioClip[] = template.audiodiff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.tsindex 186c8527..5e107ee7 100644--- a/site/src/cut/lib/types.ts+++ b/site/src/cut/lib/types.ts@@ -554,6 +554,12 @@ export interface VideoClip {* tail shows `in`, picture and sound alike. The rate still applies, so the* footprint is unchanged. */reverse?: boolean;+ /** Smooth slow motion: wherever the rate runs below 1×, the frames between+ * two source frames are synthesized so the picture moves at the output+ * rate. At 1× and faster the footage shows its own frames. The preview+ * and the tab's export blend the two neighbours; ffmpeg exports estimate+ * the motion between them. */+ smoothSlow?: boolean;/** Transition into the next clip on this clip's track, in timeline seconds* (absent/0 = hard cut). Every style overlaps the two clips by this much,* so the cut shortens. On upper tracks a transition blends the incoming@@ -1044,6 +1050,7 @@ export interface TemplateLayer {speed?: number;speedCurve?: SpeedNode[];reverse?: boolean;+ smoothSlow?: boolean;sound?: ClipSound;track: number;/** Came from video track 0 — re-materializes as a timeline clip, not andiff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.tsindex 8a0f82ed..969d4488 100644--- a/site/src/cut/server/exportPipeline.test.ts+++ b/site/src/cut/server/exportPipeline.test.ts@@ -355,6 +355,44 @@ describe("a speed curve in the filtergraph", () => {});});+describe("smooth slow motion in the filtergraph", () => {+ test("a slowed clip runs motion interpolation over its whole span", async () => {+ const g = await graphFor({ clips: [clip("a.mp4", { speed: 0.5, smoothSlow: true }), clip("b.mp4")] });+ const video = g.find((f) => f.startsWith("[0:v]trim="))!;+ expect(video).toContain("minterpolate=fps=30:mi_mode=mci");+ expect(video).not.toContain("split=");+ // The plain clip beside it is untouched.+ expect(g.find((f) => f.startsWith("[1:v]trim="))).not.toContain("minterpolate");+ });++ test("a curve that dips is interpolated in the dip alone", async () => {+ const curved = clip("a.mp4", {+ smoothSlow: true,+ speedCurve: [+ [0, 1],+ [1.5, 0.4],+ [3, 3],+ [4, 3],+ ],+ });+ const g = await graphFor({ clips: [curved, clip("b.mp4")] });+ const head = g.find((f) => f.startsWith("[0:v]trim="))!;+ expect(head).toContain("split=3[smic0_0][smic0_1][smic0_2]");+ const pieces = g.filter((f) => f.startsWith("[smic0_"));+ expect(pieces.length).toBe(3);+ expect(pieces[0]).toContain("fps=30[smo");+ expect(pieces[0]).not.toContain("minterpolate");+ expect(pieces[1]).toContain("minterpolate=fps=30");+ expect(pieces[2]).not.toContain("minterpolate");+ expect(g.some((f) => f.includes("concat=n=3:v=1:a=0,fps=30[smoc0]"))).toBe(true);+ });++ test("a clip at 1× or faster keeps its plain chain with the flag on", async () => {+ const g = await graphFor({ clips: [clip("a.mp4", { speed: 2, smoothSlow: true })] });+ expect(g.join(";")).not.toContain("minterpolate");+ });+});+describe("clip sound in the filtergraph", () => {const sound = {eq: [2, 1, -2, -1, 1.5, 1, -1],diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.tsindex 05c65767..5e869fd0 100644--- a/site/src/cut/server/exportPipeline.ts+++ b/site/src/cut/server/exportPipeline.ts@@ -7,7 +7,7 @@ import { assertGraphSafe, fexpr } from "./filterGraph";import { bakeRetimedAudio, setptsExpr, type BakedAudio } from "./retimeAudio";import { bakeTurnedMedia } from "./turnMedia";import { CLIP_MAX_ZOOM, projectFadeSeconds, regionPx, TRANSITION_XFADE, TRANSITION_ZOOM, type ColorGrade, type TransitionStyle } from "../lib/types";-import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, mirrorRetimable, retimeOf, shortestTurn, srcSpan, sortedKeys, soundFilters, type ClipSound, type OverlayKey, type Retime, type SpeedNode } from "@donkeycut/effects-kit";+import { audioFxFilters, buildGradeLut, effectFilterLines, gradeKey, gradeNeedsLut, gradeToFfmpegFilter, isAudioEffect, lookFilterLines, lutToCube, mirrorRetimable, retimeOf, shortestTurn, slowRuns, srcSpan, sortedKeys, soundFilters, type ClipSound, type OverlayKey, type Retime, type SpeedNode } from "@donkeycut/effects-kit";// The render pipeline itself: spec in, finished mp4 out. Shared by the local// engine's job registry (jobs.ts) and the cloud render worker, which stage@@ -101,6 +101,8 @@ export interface ExportSpec {* (see retimeOf); present, `speed` is ignored. */speedCurve?: SpeedNode[];reverse?: boolean;+ /** Synthesize the frames wherever the rate runs below 1× (see slowRuns). */+ smoothSlow?: boolean;/** Transition into the next clip, in timeline seconds (overlap). */transition?: number;/** Half the cross dissolve into the next clip, in timeline seconds: the@@ -192,6 +194,8 @@ export interface ExportSpec {* (see retimeOf); present, `speed` is ignored. */speedCurve?: SpeedNode[];reverse?: boolean;+ /** Synthesize the frames wherever the rate runs below 1× (see slowRuns). */+ smoothSlow?: boolean;/** Transition ramps, timeline seconds from this overlay's head/tail. On* an upper track a fade is an alpha fade (the tracks beneath show* through); a cross transition arrives as the incoming clip's headFade@@ -483,6 +487,60 @@ const retimedPts = (rt: Retime) =>rt.uniform ? `(PTS-STARTPTS)/${num(rt.rate)}` : fexpr(setptsExpr(rt));/** Timeline length of a media clip's span through its rate or curve. */+/** ffmpeg's motion-compensated interpolation, laying `fps` frames over the+ * sparse ones a slowed span has: bidirectional block motion estimation and+ * overlapped-block compensation, with its scene-change guard on so a cut+ * inside the footage is never blended across. */+const MOTION_INTERPOLATE = "mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1:scd=fdiff";++/**+ * The head of a footage clip's picture chain: its retimed, framed source at+ * `fps`. A clip smoothing its slow motion runs ffmpeg's motion interpolation+ * over exactly the stretches its rate is under 1× — the span is cut into+ * pieces at those boundaries, each piece re-stamped to `fps` on its own+ * (interpolated or plain), and the pieces joined back — so a curve that+ * dips and races past 1× is smoothed in the dip alone. The framing runs+ * first, so the estimate works at the output size on the frames the slow+ * stretch actually has.+ */+function framedTimebase(+ timebase: string,+ framing: string,+ tag: string,+ c: { smoothSlow?: boolean; image?: boolean },+ rt: Retime,+ fps: number,+ filters: string[]+): string {+ const plain = `${timebase},fps=${fps},${framing},setsar=1`;+ if (c.image || !c.smoothSlow) return plain;+ const runs = slowRuns(rt, 1 / fps);+ if (runs.length === 0) return plain;+ const interp = `minterpolate=fps=${fps}:${MOTION_INTERPOLATE}`;+ const framed = `${timebase},${framing},setsar=1`;+ if (runs.length === 1 && runs[0][0] <= 1e-6 && runs[0][1] >= rt.len - 1e-6) {+ return `${framed},${interp}`;+ }+ const pieces: { from: number; to: number; slow: boolean }[] = [];+ let at = 0;+ for (const [from, to] of runs) {+ if (from > at + 1e-6) pieces.push({ from: at, to: from, slow: false });+ pieces.push({ from, to, slow: true });+ at = to;+ }+ if (rt.len > at + 1e-6) pieces.push({ from: at, to: rt.len, slow: false });+ filters.push(`${framed},split=${pieces.length}${pieces.map((_, k) => `[smi${tag}_${k}]`).join("")}`);+ pieces.forEach((pc, k) => {+ filters.push(+ `[smi${tag}_${k}]trim=${num(pc.from)}:${num(pc.to)},setpts=PTS-STARTPTS,${pc.slow ? interp : `fps=${fps}`}[smo${tag}_${k}]`+ );+ });+ filters.push(+ `${pieces.map((_, k) => `[smo${tag}_${k}]`).join("")}concat=n=${pieces.length}:v=1:a=0,fps=${fps}[smo${tag}]`+ );+ return `[smo${tag}]null`;+}+const spanLen = (c: { in: number; out: number; speed?: number; speedCurve?: SpeedNode[]; reverse?: boolean }) =>Math.max(0.1, retimeOf(c).len);@@ -1448,7 +1506,7 @@ export async function runExport(// a still just replays its looped input.// The grade sits after the color conversion (so it acts on the same// BT.709 values the preview shows) and before the terminal format.- let core = `${timebase},fps=${fps},${frame},setsar=1,${rmIn ? "" : colorFix.get(c.file) ?? ""}${gradeChain(rmIn ? undefined : c.grade)}format=${segFmt}`;+ let core = `${framedTimebase(timebase, frame, `c${j}`, rmIn ? {} : c, rt, fps, filters)},${rmIn ? "" : colorFix.get(c.file) ?? ""}${gradeChain(rmIn ? undefined : c.grade)}format=${segFmt}`;// The look bakes in after grade + framing, before the edge effects, so// animations move already-graded pixels (matching the preview order). A// removal clip's look runs after the flatten below instead — the chain's@@ -1851,7 +1909,7 @@ export async function runExport(? `[${idx}:v]setpts=PTS-STARTPTS`: `[${idx}:v]trim=${num(oc.in)}:${num(oc.out)},setpts=${retimedPts(ort)}`;}- let core = `${timebase},fps=${fps},${framing},setsar=1,${orIn ? "" : colorFix.get(oc.file) ?? ""}${gradeChain(orIn ? undefined : oc.grade)}format=${orIn ? "yuva420p" : lookFmt}`;+ let core = `${framedTimebase(timebase, framing, `o${k}`, orIn ? {} : oc, ort, fps, filters)},${orIn ? "" : colorFix.get(oc.file) ?? ""}${gradeChain(orIn ? undefined : oc.grade)}format=${orIn ? "yuva420p" : lookFmt}`;// Looks bake into footage overlays only: an image may carry alpha, which// the look chain's internal filters would flatten onto black over the// tracks beneath. The alpha fades stay safe: they apply after the look.