Skip to content

Open source CapCut alternative.

TypeScript

feat(site): the editor plays, mixes and draws through the clip's map

David Leff46f8aad137
diff --git a/site/src/cut/components/Preview.tsx b/site/src/cut/components/Preview.tsx
index 41af7299..72f1b62b 100644
--- a/site/src/cut/components/Preview.tsx
+++ b/site/src/cut/components/Preview.tsx
@@ -11,7 +11,7 @@ import {
usePreviewSelector,
useSkim,
} from "@/cut/lib/playhead";
-import { projectDuration, useEditor } from "@/cut/lib/store";
+import { clipLen, projectDuration, useEditor } from "@/cut/lib/store";
import { copyStageFrame, primeStageFrame, stageHasFrame } from "@/cut/lib/stageFrame";
import {
capturePoster,
@@ -505,8 +505,7 @@ export function Preview() {
if (c.hidden) continue;
const r = rectOf(c);
if (isFullRect(r)) continue;
- const speed = c.speed && c.speed > 0 ? c.speed : 1;
- const len = Math.max(0.1, (c.out - c.in) / speed);
+ const len = clipLen(c);
if (t < c.start || t >= c.start + len) continue;
if (px < r.x || px > r.x + r.w || py < r.y || py > r.y + r.h) continue;
if (!best || c.track > best.track) best = { id: c.id, track: c.track };
@@ -858,8 +857,7 @@ function ClipTransformGizmo({ stage }: { stage: Stage }) {
const brushClipId = useBrushUi((s) => s.clipId);
const live = usePreviewSelector((t) => {
if (!selectedClip) return false;
- const speed = selectedClip.speed && selectedClip.speed > 0 ? selectedClip.speed : 1;
- const len = Math.max(0.1, (selectedClip.out - selectedClip.in) / speed);
+ const len = clipLen(selectedClip);
return t >= selectedClip.start && t < selectedClip.start + len;
});
// While hover-scrubbing the preview shows the skimmer's frame, where the
@@ -1238,8 +1236,7 @@ function ClipMaskGizmo({ stage }: { stage: { w: number; h: number } }) {
if (skimTime !== null) return null;
const clip = masked;
if (!armed || !clip?.mask) return null;
- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;
- const len = Math.max(0.1, (clip.out - clip.in) / speed);
+ const len = clipLen(clip);
if (tLocal < 0 || tLocal >= len) return null;
const rect = rectOf(clip);
const pose = clipKeyed(clip) ? clipPoseAt(clip, tLocal) : null;
diff --git a/site/src/cut/components/RemovalBrush.tsx b/site/src/cut/components/RemovalBrush.tsx
index 9082a47a..e6f534c3 100644
--- a/site/src/cut/components/RemovalBrush.tsx
+++ b/site/src/cut/components/RemovalBrush.tsx
@@ -1,5 +1,7 @@
"use client";
+import { retimeOf } from "@donkeycut/effects-kit";
+
import { useEffect, useRef, useState } from "react";
import { createPortal } from "react-dom";
import { useBrushUi } from "@/cut/lib/removal/brushUi";
@@ -56,8 +58,7 @@ export function RemovalBrush({ stage }: { stage: { w: number; h: number } }) {
// pointer rides the timeline and the overlay tracks the frame on screen.
const tLocal = usePreviewSelector((t) => (armed && clip ? t - clip.start : -1));
if (!armed || !clip) return null;
- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;
- const len = Math.max(0.1, (clip.out - clip.in) / speed);
+ const len = Math.max(0.1, retimeOf(clip).len);
if (tLocal < 0 || tLocal >= len) return null;
const asset = assets.find((a) => a.id === clip.assetId);
if (!asset?.width || !asset?.height) return null;
@@ -132,7 +133,7 @@ function BrushSurface({
if (ctx) s.paintOverlay(ctx, c.width, c.height);
};
- const srcT = () => clip.in + Math.max(0, tLocal) * (clip.speed && clip.speed > 0 ? clip.speed : 1);
+ const srcT = () => retimeOf(clip).srcAt(Math.max(0, tLocal));
// The overlay shows the strokes recorded on the frame the preview shows:
// the mask rebuilds when the seeds change from outside (undo, redo, the
@@ -142,8 +143,7 @@ function BrushSurface({
// stored paint, which has no place inside a frame tick.
const seeds = clip.removal?.seeds;
const playing = useEditor((s) => s.playing);
- const speedNow = clip.speed && clip.speed > 0 ? clip.speed : 1;
- const srcNow = clip.in + Math.max(0, tLocal) * speedNow;
+ const srcNow = retimeOf(clip).srcAt(Math.max(0, tLocal));
const activeT = playing
? null
: ((seeds?.paint ?? []).find((p) => Math.abs(p.t - srcNow) < 0.05)?.t ?? null);
diff --git a/site/src/cut/components/RemovalPanel.tsx b/site/src/cut/components/RemovalPanel.tsx
index e240e089..d450c644 100644
--- a/site/src/cut/components/RemovalPanel.tsx
+++ b/site/src/cut/components/RemovalPanel.tsx
@@ -27,6 +27,7 @@ import {
STROKE_FEATHER_MAX,
STROKE_OFFSET_MAX,
STROKE_WIDTH_MAX,
+ retimeOf,
} from "@donkeycut/effects-kit";
import {
removalActive,
@@ -368,13 +369,13 @@ function CustomControls({ clip }: { clip: VideoClip }) {
.sort((a, b) => a - b);
const showStrokes = () => {
if (!strokeTimes.length) return;
- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;
- const srcNow = clip.in + Math.max(0, playheadAt() - clip.start) * speed;
+ const rt = retimeOf(clip);
+ const srcNow = rt.srcAt(Math.max(0, playheadAt() - clip.start));
const next = strokeTimes.find((t) => t > srcNow + 0.05) ?? strokeTimes[0];
- const end = clip.start + (clip.out - clip.in) / speed;
+ const end = clip.start + rt.len;
useEditor
.getState()
- .seek(Math.min(Math.max(clip.start, clip.start + (next - clip.in) / speed), end - 0.001));
+ .seek(Math.min(Math.max(clip.start, clip.start + rt.tAt(next)), end - 0.001));
};
return (
<>
diff --git a/site/src/cut/components/TemplateCard.tsx b/site/src/cut/components/TemplateCard.tsx
index 93d3e9ce..da0cb82a 100644
--- a/site/src/cut/components/TemplateCard.tsx
+++ b/site/src/cut/components/TemplateCard.tsx
@@ -33,7 +33,7 @@ import { refFromTemplate, useAssetDrop, type AssetRef } from "@/cut/lib/assetRef
import { useRefCopy } from "@/cut/lib/refCopy";
import { cardIconButton } from "@/cut/components/iconButton";
import { formatTime } from "@/cut/lib/time";
-import { EFFECT_LABELS } from "@donkeycut/effects-kit";
+import { EFFECT_LABELS, retimeOf, type SpeedNode } from "@donkeycut/effects-kit";
import { SHAPE_LABELS } from "@/cut/lib/types";
import type { LibraryTemplate, TemplateMedia } from "@/cut/lib/types";
import { cn } from "@/lib/utils";
@@ -105,8 +105,8 @@ export function TemplateCard({
// The contents, one row per item, in save order: clips and stills, sounds,
// titles, then a caption count.
- const speedLen = (x: { in: number; out: number; speed?: number }) =>
- (x.out - x.in) / (x.speed && x.speed > 0 ? x.speed : 1);
+ const speedLen = (x: { in: number; out: number; speed?: number; speedCurve?: SpeedNode[] }) =>
+ retimeOf(x).len;
const parts: {
icon: typeof Film;
label: string;
diff --git a/site/src/cut/components/TransitionsPanel.tsx b/site/src/cut/components/TransitionsPanel.tsx
index 716ec11d..59ef2102 100644
--- a/site/src/cut/components/TransitionsPanel.tsx
+++ b/site/src/cut/components/TransitionsPanel.tsx
@@ -138,7 +138,7 @@ function useCutFrames(): { a: string | null; b: string | null } {
if (dist >= bestDist) continue;
bestDist = dist;
best = {
- a: { asset: out.asset, srcT: out.clip.in + out.len * (out.clip.speed ?? 1) },
+ a: { asset: out.asset, srcT: out.clip.out },
b: { asset: inc.asset, srcT: inc.clip.in },
};
}
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.ts
index 69f0aa02..42d29dd5 100644
--- a/site/src/cut/hooks/usePlayback.ts
+++ b/site/src/cut/hooks/usePlayback.ts
@@ -1,14 +1,14 @@
"use client";
+
import { useEffect, type RefObject } from "react";
import {
- clipSpeed,
getClipSpans,
overlayLayers,
projectDuration,
useEditor,
} from "@/cut/lib/store";
-import { matteLumaToAlpha } from "@donkeycut/effects-kit";
+import { matteLumaToAlpha, retimeOf } from "@donkeycut/effects-kit";
import { playheadAt, previewAt, setPlayhead, subscribePlayhead } from "@/cut/lib/playhead";
import { assetIsSilent, clipCovers, projectFadeSeconds, rectOf } from "@/cut/lib/types";
import type { ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";
@@ -78,7 +78,7 @@ const DECODER_BUDGET = 12;
/** Source time of a clip at timeline time `t`. */
const sourceTimeOf = (clip: VideoClip, t: number) =>
- clip.in + Math.max(0, t - clip.start) * clipSpeed(clip);
+ retimeOf(clip).srcAt(Math.max(0, t - clip.start));
/**
* When the preview gives up resolution to keep the picture with the sound.
@@ -110,7 +110,7 @@ const PLAY_SETTLE_MS = 6_000;
/** The decode identity of a clip — see `mappingKey`. */
const keyOf = (clip: VideoClip, asset: MediaAsset) =>
- mappingKey(asset.id, clipSpeed(clip), clip.in, clip.start);
+ mappingKey(asset.id, retimeOf(clip), clip.in, clip.start);
let engineSerial = 0;
@@ -135,13 +135,18 @@ export function engineLog(msg: string): void {
* schedules one stable voice and the frame-by-frame gain is all that moves.
*/
const voiceSpan = (sp: ClipSpan) => {
- const speed = clipSpeed(sp.clip);
+ // The handles a crossing reaches into play at the pace of the footage
+ // beside them, so the voice's span widens through the clip's own map and
+ // its retime is rebuilt over the wider span.
+ const rt = retimeOf(sp.clip);
+ const wIn = rt.srcAt(-sp.soundBack);
+ const wOut = rt.srcAt(rt.len + sp.soundAhead);
return {
url: sp.asset.url,
start: sp.start - sp.soundBack,
- in: sp.clip.in - sp.soundBack * speed,
- out: sp.clip.out + sp.soundAhead * speed,
- speed,
+ in: wIn,
+ out: wOut,
+ retime: retimeOf({ in: wIn, out: wOut, speed: sp.clip.speed, speedCurve: sp.clip.speedCurve }),
sound: sp.clip.sound,
};
};
@@ -376,12 +381,12 @@ class Engine {
// alpha read below is a per-frame pixel pass; capping the matte's decode
// keeps that pass off the frame budget.
const src = this.pool.get(
- mappingKey(asset.id, clipSpeed(clip), m.in, clip.start),
+ mappingKey(asset.id, retimeOf(clip), m.in, clip.start),
asset,
Math.min(this.decodeHeight(), 480)
);
this.used.add(src);
- src.want(mt, this.renderPlaying);
+ src.want(mt, this.renderPlaying, false, retimeOf(clip).rateAt(at - clip.start));
const frame = src.frameAt(mt, 0, dur);
if (!frame) return null;
const held = this.matteAlpha.get(clip.id);
@@ -423,7 +428,7 @@ class Engine {
const src = this.pool.get(keyOf(span.clip, span.asset), span.asset, this.decodeHeight());
this.used.add(src);
const st = sourceTimeOf(span.clip, t);
- src.want(st, playing, this.readBackward);
+ src.want(st, playing, this.readBackward, retimeOf(span.clip).rateAt(t - span.start));
// A failed source that already decoded frames keeps showing the nearest
// one it holds — a transient blip (a network drop, a signed URL mid
// re-mint) reads as a held frame, and only a source with nothing at all
@@ -581,8 +586,8 @@ class Engine {
for (const a of s.audioClips) {
const asset = s.assets.find((x) => x.id === a.assetId);
if (!asset || a.hidden || assetIsSilent(asset)) continue;
- const speed = a.speed && a.speed > 0 ? a.speed : 1;
- const len = Math.max(0.1, (a.out - a.in) / speed);
+ const rt = retimeOf(a);
+ const len = Math.max(0.1, rt.len);
if (t < a.start || t >= a.start + len) continue;
// Fade envelope: linear ramps at either end of the clip.
const rel = t - a.start;
@@ -599,7 +604,7 @@ class Engine {
start: a.start,
in: a.in,
out: a.out,
- speed,
+ retime: rt,
gain: a.volume * g * dg,
sound: a.sound,
});
diff --git a/site/src/cut/lib/assetRef.ts b/site/src/cut/lib/assetRef.ts
index af77837d..44445396 100644
--- a/site/src/cut/lib/assetRef.ts
+++ b/site/src/cut/lib/assetRef.ts
@@ -818,7 +818,7 @@ export function selectionRefTokens(s: EntitySources & {
audioClips: AudioClip[];
selection: Selection;
multiSelection: Selection[];
- selectedKey: { kind: "overlay" | "clip"; id: string; t: number; track: "pose" | "mask" } | null;
+ selectedKey: { kind: "overlay" | "clip"; id: string; t: number; track: "pose" | "mask" | "speed" } | null;
}): string | null {
const entities = entityRefs(s);
diff --git a/site/src/cut/lib/audioMix.ts b/site/src/cut/lib/audioMix.ts
index 21659b34..95d68a5e 100644
--- a/site/src/cut/lib/audioMix.ts
+++ b/site/src/cut/lib/audioMix.ts
@@ -21,8 +21,10 @@ import {
AUDIO_FX_RAMP,
audioFxRecipe,
buildAudioFx,
+ retimeOf,
soundRecipe,
type ClipSound,
+ type SpeedNode,
} from "@donkeycut/effects-kit";
import { decodeAudioSpan } from "./mediaRead";
import { timeStretch } from "./timeStretch";
@@ -35,6 +37,7 @@ export interface MixClip {
out: number;
muted: boolean;
speed?: number;
+ speedCurve?: SpeedNode[];
/** Gain on the clip's own audio; absent = 1. */
volume?: number;
/** The clip's own treatment, run on its sound before the fades. */
@@ -62,6 +65,7 @@ export interface MixItem {
start: number;
volume: number;
speed?: number;
+ speedCurve?: SpeedNode[];
/** Half the cross dissolve at this item's head and tail, with the handle
* each side reaches into — an upper-track cut crosses the same way track
* 0's does. */
@@ -105,15 +109,14 @@ export interface MixOptions {
resolve: (file: string) => string;
}
-const speedOf = (s?: number) => (s && s > 0 ? s : 1);
-
-/** Timeline length of one clip: source span over speed. A gap spacer keeps its
- * exact length — flooring it would land everything after it late. */
+/** Timeline length of one clip: its source span through its rate. A gap
+ * spacer keeps its exact length — flooring it would land everything after it
+ * late. */
const clipDur = (c: MixClip) =>
- c.file ? Math.max(0.1, (c.out - c.in) / speedOf(c.speed)) : Math.max(0, c.out - c.in);
+ c.file ? Math.max(0.1, retimeOf(c).len) : Math.max(0, c.out - c.in);
/** Timeline footprint of a placed item. */
-const itemDur = (a: MixItem) => Math.max(0, (a.out - a.in) / speedOf(a.speed));
+const itemDur = (a: MixItem) => Math.max(0, retimeOf(a).len);
/**
* Where the sequential fold puts each track-0 clip, and how long its
@@ -157,20 +160,20 @@ export function foldClips(clips: MixClip[]) {
* by the ramp that crossing actually got. */
const handles = (g: { clip: MixClip; crossIn: number; crossOut: number }) => ({
// Never past the head of the source: a handle is what the trim left behind.
- back: Math.min(g.clip.soundBack ?? 0, g.crossIn, g.clip.in / speedOf(g.clip.speed)),
+ back: Math.min(g.clip.soundBack ?? 0, g.crossIn, Math.max(0, -retimeOf(g.clip).tAt(0))),
ahead: Math.min(g.clip.soundAhead ?? 0, g.crossOut),
});
/** The source span an entry decodes: its own trim, widened by the handle each
- * crossing at its edges reaches into. Timeline seconds scale by speed on the
- * way into the source. */
+ * crossing at its edges reaches into. Timeline seconds go through the entry's
+ * own map on the way into the source. */
const crossSpan = (
- c: { file: string; in: number; out: number; speed?: number },
+ c: { file: string; in: number; out: number; speed?: number; speedCurve?: SpeedNode[] },
back = 0,
ahead = 0
) => {
- const speed = speedOf(c.speed);
- return { file: c.file, in: c.in - back * speed, out: c.out + ahead * speed };
+ const rt = retimeOf(c);
+ return { file: c.file, in: rt.srcAt(-back), out: rt.srcAt(rt.len + ahead) };
};
/**
@@ -364,16 +367,33 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB
* length it needs first, and then played at its own rate.
*/
const stretched = new Map<string, AudioBuffer>();
- const fitted = (key: string, buf: AudioBuffer, speed: number): AudioBuffer => {
- if (speed === 1) return buf;
- // A span used by more than one clip at the same speed is stretched once.
- const memo = `${key}@${speed}`;
+ const fitted = (
+ key: string,
+ buf: AudioBuffer,
+ entry: { in: number; out: number; speed?: number; speedCurve?: SpeedNode[] },
+ from: number
+ ): AudioBuffer => {
+ const rt = retimeOf(entry);
+ if (rt.uniform && rt.rate === 1) return buf;
+ // A span used by more than one clip at the same rate is stretched once.
+ const memo = `${key}@${rt.key}`;
const hit = stretched.get(memo);
if (hit) return hit;
const channels = Array.from({ length: buf.numberOfChannels }, (_, c) =>
buf.getChannelData(c)
);
- const fit = timeStretch(channels, buf.sampleRate, 1 / speed);
+ // `from` is the source second the buffer starts at; the schedule follows
+ // the clip's own map from there and lands exactly on its length.
+ const fit = timeStretch(
+ channels,
+ buf.sampleRate,
+ rt.uniform
+ ? 1 / rt.rate
+ : {
+ factorAt: (sec) => 1 / rt.rateAtSrc(from + sec),
+ outLength: Math.round((rt.tAt(from + buf.duration) - rt.tAt(from)) * buf.sampleRate),
+ }
+ );
const out = new AudioBuffer({
length: Math.max(1, fit[0].length),
numberOfChannels: buf.numberOfChannels,
@@ -420,7 +440,7 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB
if (!buf) continue; // muted, silent, or a gap spacer: only shapes time
const from = g.at - g.back;
const end = g.at + g.dur;
- play(fitted(key, buf, speedOf(c.speed)), from, g.dur + g.back + g.ahead, ducked, c.sound, (gain) => {
+ play(fitted(key, buf, c, Math.max(0, g.span.in)), from, g.dur + g.back + g.ahead, ducked, c.sound, (gain) => {
const level = Math.max(0, c.volume ?? 1);
gain.gain.value = level;
// A crossing and a fade never share an edge — the cut carries one
@@ -451,7 +471,7 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB
// A voiceover that ducks everything else is not itself ducked; it still
// lands on the bus, so the effect elements treat it with the rest.
const into = a.duck !== undefined && a.duck < 1 ? bus : ducked;
- play(fitted(key, buf, speedOf(a.speed)), a.start - back, dur + back + ahead, into, a.sound, (gain) => {
+ play(fitted(key, buf, a, Math.max(0, crossSpan(a, back, ahead).in)), a.start - back, dur + back + ahead, into, a.sound, (gain) => {
const level = Math.max(0, a.volume);
gain.gain.value = level;
const fIn = Math.min(a.fadeIn ?? 0, dur);
diff --git a/site/src/cut/lib/cloudTranscribe.ts b/site/src/cut/lib/cloudTranscribe.ts
index 033eaed8..39570a34 100644
--- a/site/src/cut/lib/cloudTranscribe.ts
+++ b/site/src/cut/lib/cloudTranscribe.ts
@@ -25,10 +25,11 @@ export interface CloudTranscribeSpec {
out: number;
muted: boolean;
speed?: number;
+ speedCurve?: [number, number][];
/** Cross-dissolve overlap into the next clip, timeline seconds. */
transition?: number;
}[];
- audio: { file: string; in: number; out: number; start: number; volume: number; speed?: number }[];
+ audio: { file: string; in: number; out: number; start: number; volume: number; speed?: number; speedCurve?: [number, number][] }[];
}
const RATE = 16000; // the wire format the hosted route expects
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index aa441529..aafc4896 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -16,7 +16,7 @@ import { renderRemovalPieces } from "./removalVideo";
import { createRasterCanvas, rasterCanvasToPng } from "./raster";
import { clipSpeed, getClipSpans, overlayLayers, projectDuration, spanSequence, useEditor } from "./store";
import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
-import { isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, paintMaskLuma, paintStrokeInk } from "@donkeycut/effects-kit";
+import { isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, paintMaskLuma, paintStrokeInk, retimeOf, type SpeedNode } from "@donkeycut/effects-kit";
import { renderElementFrames, renderElementPng } from "./textRender";
import { clipCovers, clipKeyed, clipPosed, clipPoseAt, clipZoom, contentRect, frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, regionPx, removalActive, shadowInk, subjectMasked } from "./types";
import { liveReader } from "./liveReader";
@@ -300,7 +300,7 @@ function removalShadowSilhouette(
if (r!.backdrop && r!.backdrop.kind !== "none") return null;
const asset = assets.find((a) => a.id === clip.assetId);
if (!asset) return null;
- const speed = clipSpeed(clip);
+ const rt = retimeOf(clip);
const still = asset.type === "image";
const canvas = createRasterCanvas(2, 2);
const ctx = canvas.getContext("2d", {
@@ -349,7 +349,7 @@ function removalShadowSilhouette(
const mdur = Math.max(0.1, matte.duration || 0.1);
return {
at: async (tLocal) => {
- const srcT = still ? 0 : clip.in + tLocal * speed;
+ const srcT = still ? 0 : rt.srcAt(tLocal);
const frame = await reader.frameAt(
Math.min(Math.max(0, srcT - r!.matte!.in), mdur - 0.001)
);
@@ -601,6 +601,7 @@ export async function buildExportPayload(
panY: sp.clip.panY ?? 0,
frame: sp.clip.frame,
speed: clipSpeed(sp.clip),
+ speedCurve: sp.clip.speedCurve,
transition: sp.transitionOut,
// A cross dissolve carries its own window: the picture cuts, so the
// server's blend must not see one. The handles are what the crossing
@@ -633,7 +634,7 @@ export async function buildExportPayload(
// opacity, since opacity ships as coverage luma.
for (let i = 0; i < spans.length; i++) {
const c = posed(spans[i].clip);
- const dur = Math.max(0.1, (c.out - c.in) / clipSpeed(c));
+ const dur = Math.max(0.1, retimeOf(c).len);
const pictures = await renderClipMaskPictures(
c,
{ x: 0, y: 0, w: settings.width, h: settings.height },
@@ -804,6 +805,7 @@ export async function buildExportPayload(
volume: c.volume,
sound: c.sound,
speed: c.speed,
+ speedCurve: c.speedCurve,
image: assetById.get(c.assetId)!.type === "image",
grade: normalizeGrade(c.grade),
look: c.look,
@@ -830,8 +832,7 @@ export async function buildExportPayload(
const box = region
? { x: region.rx, y: region.ry, w: region.rw, h: region.rh }
: { x: 0, y: 0, w: settings.width, h: settings.height };
- const ospeed = c.speed && c.speed > 0 ? c.speed : 1;
- const olen = Math.max(0.1, (c.out - c.in) / ospeed);
+ const olen = Math.max(0.1, retimeOf(c).len);
const pictures = await renderClipMaskPictures(
c,
box,
@@ -901,6 +902,7 @@ export async function buildExportPayload(
fadeIn: a.fadeIn ?? 0,
fadeOut: a.fadeOut ?? 0,
speed: a.speed,
+ speedCurve: a.speedCurve,
sound: a.sound,
duck: a.duck,
}));
@@ -1411,14 +1413,16 @@ const CARD_SECONDS = 5;
* for the share card, which is a five-second window onto the opening rather
* than a render of the whole project. */
export function docFirstSeconds(doc: ExportDoc, seconds: number): ExportDoc {
- // A clip's timeline footprint is (out - in) / speed, so the trim that ends
- // it at `seconds` scales back through the speed.
- const clampOut = <T extends { start: number; in: number; out: number; speed?: number }>(
+ // The trim that ends a clip at `seconds` goes back through the clip's own
+ // map, so a curved clip cuts at the source second really playing there.
+ const clampOut = <
+ T extends { start: number; in: number; out: number; speed?: number; speedCurve?: SpeedNode[] },
+ >(
clip: T
): T => {
- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;
- const room = (seconds - clip.start) * speed;
- return clip.out - clip.in <= room ? clip : { ...clip, out: clip.in + room };
+ const rt = retimeOf(clip);
+ const room = seconds - clip.start;
+ return rt.len <= room ? clip : { ...clip, out: rt.srcAt(room) };
};
const starts = <T extends { start: number }>(item: T) => item.start < seconds;
return {
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index 5cbc16c7..af510a2f 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -35,7 +35,7 @@ import { frameSink, openMedia, videoTrackOf } from "./mediaRead";
import { allowance, canvasBytes, holdMemory } from "./memoryBudget";
import { getClipSpans, overlayLayers, projectDuration, spanSequence } from "./store";
import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
-import { applyEffectToCanvas, evalOverlayFrame, grainTile, isAudioEffect, isMaskAnimated, isOverlayAnimated, maskFrameAt, MATTE_FPS, matteLumaToAlpha, planAnimatedLayers, type LottieHandle, type OverlayAnim, type PaintPhase } from "@donkeycut/effects-kit";
+import { applyEffectToCanvas, evalOverlayFrame, retimeOf, grainTile, isAudioEffect, isMaskAnimated, isOverlayAnimated, maskFrameAt, MATTE_FPS, matteLumaToAlpha, planAnimatedLayers, type LottieHandle, type OverlayAnim, type PaintPhase } from "@donkeycut/effects-kit";
import { backdropStill, loadBackdropStill } from "./backdropStills";
import { hasSubjectOverlays, SubjectMaskCompositor } from "./behindPass";
import { createRasterCanvas, type RasterSurface } from "./raster";
@@ -141,8 +141,7 @@ const READER_GRACE = 2;
/** A clip's source time at timeline time `t`. */
function sourceTimeAt(span: ClipSpan, t: number): number {
- const speed = span.clip.speed && span.clip.speed > 0 ? span.clip.speed : 1;
- return span.clip.in + Math.max(0, t - span.start) * speed;
+ return retimeOf(span.clip).srcAt(Math.max(0, t - span.start));
}
/**
@@ -536,6 +535,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
start: sp.start,
volume: sp.clip.volume ?? 1,
speed: sp.clip.speed,
+ speedCurve: sp.clip.speedCurve,
sound: sp.clip.sound,
muted: sp.clip.muted || assetIsSilent(sp.asset),
fadeIn: ramps[i].head,
@@ -557,6 +557,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
start: a.start,
volume: a.volume,
speed: a.speed,
+ speedCurve: a.speedCurve,
fadeIn: a.fadeIn,
fadeOut: a.fadeOut,
sound: a.sound,
@@ -576,6 +577,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strin
out: sp.clip.out,
muted: sp.clip.muted || !!sp.clip.hidden || assetIsSilent(sp.asset),
speed: sp.clip.speed,
+ speedCurve: sp.clip.speedCurve,
volume: sp.clip.volume,
sound: sp.clip.sound,
transition: sp.transitionOut,
diff --git a/site/src/cut/lib/filmstrip.test.ts b/site/src/cut/lib/filmstrip.test.ts
index 4b8611c6..241a32eb 100644
--- a/site/src/cut/lib/filmstrip.test.ts
+++ b/site/src/cut/lib/filmstrip.test.ts
@@ -1,4 +1,5 @@
import { describe, expect, test } from "bun:test";
+import { retimeOf } from "@donkeycut/effects-kit";
import {
FILM_TILE_CAP,
MIN_SUB_TILE_PX,
@@ -296,3 +297,57 @@ describe("pickThumbTimes", () => {
for (let i = 1; i < times.length; i++) expect(times[i]).toBeGreaterThan(times[i - 1]);
});
});
+
+describe("a curved clip's strips agree", () => {
+ // The speed curve editor draws the same clip as the timeline bar, at its own
+ // width. Both plan through the clip's map, so a point the same fraction
+ // along each of them pictures the same moment of the footage — which is what
+ // makes a node in the graph sit over the frame it retimes.
+ const clip = { in: 0, out: DURATION, speedCurve: [[0, 2], [DURATION, 0.25]] as [number, number][] };
+ const retime = retimeOf(clip);
+ const stripFor = (w: number) =>
+ planFilmstrip({
+ thumbs,
+ thumbStep: THUMB_STEP,
+ duration: DURATION,
+ aspect: 16 / 9,
+ filmIn: clip.in,
+ w,
+ pps: w / retime.len,
+ speed: retime.rate,
+ retime,
+ tileH: 60,
+ minTileW: 26,
+ });
+
+ test("the graph's picture is the bar's picture", () => {
+ const bar = stripFor(1400);
+ const graph = stripFor(700);
+ expect(bar.length).toBeGreaterThan(2);
+ expect(graph.length).toBeGreaterThan(2);
+ const at = (tiles: typeof bar, w: number, f: number) => {
+ const x = f * w;
+ return tiles.find((t) => x >= t.left && x < t.left + t.width) ?? tiles[tiles.length - 1];
+ };
+ for (const f of [0.1, 0.3, 0.5, 0.7, 0.9]) {
+ const a = at(bar, 1400, f);
+ const b = at(graph, 700, f);
+ // Within a tile of each other: the two widths cut the same map into
+ // different numbers of tiles, so the moments agree, not the boundaries.
+ const tileSpan = retime.srcAt(retime.len * (b.width / 700 + f)) - retime.srcAt(retime.len * f);
+ expect(Math.abs(a.srcT - b.srcT)).toBeLessThanOrEqual(Math.abs(tileSpan) + 0.5);
+ }
+ });
+
+ test("both walk the footage forward across the whole box", () => {
+ for (const w of [700, 1400]) {
+ const tiles = stripFor(w);
+ for (let i = 1; i < tiles.length; i++) expect(tiles[i].srcT).toBeGreaterThan(tiles[i - 1].srcT);
+ // The clip opens at 2x, so its first tile already covers seconds of
+ // footage; what matters is that the strip starts inside the opening
+ // stretch and ends on the tail.
+ expect(tiles[0].srcT).toBeLessThan(retime.srcAt(retime.len * 0.25));
+ expect(tiles[tiles.length - 1].srcT).toBeGreaterThan(DURATION - 2);
+ }
+ });
+});
diff --git a/site/src/cut/lib/filmstrip.ts b/site/src/cut/lib/filmstrip.ts
index eec8c2fa..137467f7 100644
--- a/site/src/cut/lib/filmstrip.ts
+++ b/site/src/cut/lib/filmstrip.ts
@@ -36,6 +36,8 @@
* grid: a scroll keeps a tile's capture where it was, a zoom re-aims it.
*/
+import type { Retime } from "@donkeycut/effects-kit";
+
export type FilmTile = {
/** The tile's place on the source grid at this zoom, stable across
* scrolls and trims; a zoom re-aims the same id at a new moment. */
@@ -109,6 +111,42 @@ export function pickThumbTimes(
});
}
+/** Tiles on the timeline grid for a clip whose rate changes: `imgW` pixels
+ * each, the last one ending at the clip's edge, each showing the source
+ * second playing at its middle. */
+function planCurvedStrip(
+ p: Omit<Parameters<typeof planFilmstrip>[0], "retime"> & { retime: Retime },
+ imgW: number
+): FilmTile[] {
+ const rt = p.retime;
+ const tiles: FilmTile[] = [];
+ const n = Math.max(1, Math.ceil(p.w / imgW));
+ const first = p.view ? Math.max(0, Math.floor(p.view.lo / imgW)) : 0;
+ const last = p.view ? Math.min(n - 1, Math.ceil(p.view.hi / imgW) - 1) : n - 1;
+ for (let i = first; i <= last; i++) {
+ const left = i * imgW;
+ const width = Math.min(imgW, p.w - left);
+ if (width <= 0) break;
+ const tMid = (left + width / 2) / p.pps;
+ const mid = Math.max(0, Math.min(p.duration, rt.srcAt(tMid)));
+ const a = Math.max(0, rt.srcAt(left / p.pps));
+ const b = Math.min(p.duration, rt.srcAt((left + width) / p.pps));
+ const idx = Math.min(p.thumbs.length - 1, Math.max(0, Math.floor(mid / p.thumbStep)));
+ let src = p.thumbs[idx];
+ let exact = false;
+ const snapped = (Math.floor(mid / WANT_GRID_S) + 0.5) * WANT_GRID_S;
+ let wantT = snapped > a && snapped < b ? snapped : mid;
+ wantT = Math.max(0, Math.min(p.duration - 0.05, wantT));
+ const hit = p.exactFrame?.(wantT) ?? null;
+ if (hit) {
+ src = hit;
+ exact = true;
+ }
+ tiles.push({ id: String(i), src, left, width, srcT: mid, wantT, exact });
+ }
+ return tiles;
+}
+
export function planFilmstrip(p: {
thumbs: string[];
thumbStep: number;
@@ -121,7 +159,13 @@ export function planFilmstrip(p: {
/** Drawn width of the clip box, px. */
w: number;
pps: number;
+ /** Source seconds per timeline second across the strip (the clip's rate,
+ * or its average rate under a curve). */
speed: number;
+ /** The clip's map, when its rate changes through the footage. The strip
+ * then lays tiles on the timeline grid — a held moment takes the width it
+ * plays for — with each tile picturing the source second at its middle. */
+ retime?: Retime;
tileH: number;
minTileW: number;
/** Scene changes in the source, seconds, ascending. While refining they
@@ -140,6 +184,7 @@ export function planFilmstrip(p: {
const span = p.view ? Math.min(p.w, p.view.hi - p.view.lo) : p.w;
const cells = Math.max(1, Math.ceil(span / natural));
const imgW = natural * Math.ceil(cells / FILM_TILE_CAP);
+ if (p.retime && !p.retime.uniform) return planCurvedStrip({ ...p, retime: p.retime }, imgW);
const stepT = (imgW / p.pps) * p.speed;
const filmOut = p.filmIn + (p.w / p.pps) * p.speed;
// The window, in source time, clamped to the clip.
diff --git a/site/src/cut/lib/framePlan.ts b/site/src/cut/lib/framePlan.ts
index 3dc8d171..77a231ac 100644
--- a/site/src/cut/lib/framePlan.ts
+++ b/site/src/cut/lib/framePlan.ts
@@ -1,5 +1,7 @@
"use client";
+import { retimeOf } from "@donkeycut/effects-kit";
+
/**
* What the cut looks like at one instant, worked out from the document alone.
*
@@ -194,8 +196,7 @@ export function duckGainAt(audioClips: AudioClip[], t: number): number {
let g = 1;
for (const a of audioClips) {
if (a.hidden || a.duck === undefined || a.duck >= 1) continue;
- const speed = a.speed && a.speed > 0 ? a.speed : 1;
- const len = Math.max(0.1, (a.out - a.in) / speed);
+ const len = Math.max(0.1, retimeOf(a).len);
if (t >= a.start && t < a.start + len) g = Math.min(g, Math.max(0, a.duck));
}
return g;
diff --git a/site/src/cut/lib/frameSource.test.ts b/site/src/cut/lib/frameSource.test.ts
index e0e264ad..4ebb5c8c 100644
--- a/site/src/cut/lib/frameSource.test.ts
+++ b/site/src/cut/lib/frameSource.test.ts
@@ -1,5 +1,6 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
-import type { Timed } from "./frameSource";
+import { retimeOf } from "@donkeycut/effects-kit";
+import { DECODE_AHEAD_S, type Timed } from "./frameSource";
import type { MediaAsset } from "./types";
// ── the files a source reads, stood in for ──────────────────────────────────
@@ -238,26 +239,36 @@ describe("FrameRing", () => {
});
describe("mappingKey", () => {
+ const uniform = (speed: number, inPoint: number, out = 100) =>
+ retimeOf({ in: inPoint, out, speed });
test("clips showing identical pictures share a decoder", () => {
// A plain split: same file, same speed, and the same source time at
// timeline zero. One decoder reads straight across the cut.
- const left = mappingKey("a", 1, 0, 0);
- const right = mappingKey("a", 1, 4, 4);
+ const left = mappingKey("a", uniform(1, 0), 0, 0);
+ const right = mappingKey("a", uniform(1, 4), 4, 4);
expect(left).toBe(right);
});
test("different trims of one file keep their own decoders", () => {
// Both live at timeline 4, showing different moments — a same-source
// dissolve has to blend two distinct frames.
- expect(mappingKey("a", 1, 0, 4)).not.toBe(mappingKey("a", 1, 8, 4));
+ expect(mappingKey("a", uniform(1, 0), 0, 4)).not.toBe(mappingKey("a", uniform(1, 8), 8, 4));
});
test("a speed change is a different mapping", () => {
- expect(mappingKey("a", 1, 0, 0)).not.toBe(mappingKey("a", 2, 0, 0));
+ expect(mappingKey("a", uniform(1, 0), 0, 0)).not.toBe(mappingKey("a", uniform(2, 0), 0, 0));
+ });
+
+ test("a speed curve is its own mapping, keyed on the curve and the start", () => {
+ const curved = (start: number) =>
+ mappingKey("a", retimeOf({ in: 0, out: 10, speedCurve: [[0, 1], [10, 4]] }), 0, start);
+ expect(curved(0)).toBe(curved(0));
+ expect(curved(0)).not.toBe(curved(2));
+ expect(curved(0)).not.toBe(mappingKey("a", uniform(1, 0), 0, 0));
});
test("different files never share", () => {
- expect(mappingKey("a", 1, 0, 0)).not.toBe(mappingKey("b", 1, 0, 0));
+ expect(mappingKey("a", uniform(1, 0), 0, 0)).not.toBe(mappingKey("b", uniform(1, 0), 0, 0));
});
});
@@ -265,7 +276,7 @@ describe("walkClaim", () => {
/** A walk that has landed frames from `from` to `tail`, asked for `t`. */
const landed = (t: number, from: number, tail: number, covered = false, heldBefore = true) =>
walkClaim({
- t, walkFor: from, from, tail, landed: true, covered, heldBefore, comingS: 5, playing: true,
+ t, ahead: DECODE_AHEAD_S, walkFor: from, from, tail, landed: true, covered, heldBefore, comingS: 5, playing: true,
});
test("playback inside the walk keeps it", () => {
@@ -309,7 +320,7 @@ describe("walkClaim", () => {
// reader is behind the anchor by construction, and that is not the walk
// going past it — it is the walk about to be right.
const lead = {
- t: 4, walkFor: 4, from: 4.5, tail: 4.6, landed: true, covered: false, heldBefore: true, playing: true,
+ t: 4, ahead: DECODE_AHEAD_S, walkFor: 4, from: 4.5, tail: 4.6, landed: true, covered: false, heldBefore: true, playing: true,
};
expect(walkClaim({ ...lead, comingS: 5 })).toBe("hold");
expect(walkClaim({ ...lead, t: 4.4, comingS: 5 })).toBe("hold");
diff --git a/site/src/cut/lib/frameSource.ts b/site/src/cut/lib/frameSource.ts
index 14e499de..e44e15f4 100644
--- a/site/src/cut/lib/frameSource.ts
+++ b/site/src/cut/lib/frameSource.ts
@@ -1,5 +1,7 @@
"use client";
+import type { Retime } from "@donkeycut/effects-kit";
+
/**
* Frames for the preview, decoded straight off the file.
*
@@ -148,6 +150,9 @@ export type WalkClaim = "hold" | "hop" | "restart";
export function walkClaim(w: {
/** The moment being asked for. */
t: number;
+ /** Source seconds the walk is kept ahead of the reader, for the clip's
+ * rate; the 1× lookahead when absent. */
+ ahead?: number;
/** Where the reader stood when this walk was sent — at or behind `from`. */
walkFor: number;
/** Where the walk was anchored. */
@@ -183,7 +188,7 @@ export function walkClaim(w: {
// the lookahead's slack so a walk merely holding its lead is left alone.
const behind =
w.playing &&
- (w.t < w.walkFor - SAME || (!w.heldBefore && w.t < w.tail - DECODE_AHEAD_S));
+ (w.t < w.walkFor - SAME || (!w.heldBefore && w.t < w.tail - (w.ahead ?? DECODE_AHEAD_S)));
if (!w.landed) {
// Nothing landed means the walk is still doing the only work that can
// produce a picture — finding the keyframe before its anchor, fetching the
@@ -532,6 +537,8 @@ export class ClipFrameSource {
* has since left. */
private aim = 0;
private closed = false;
+ /** Source seconds the walk keeps ahead of the reader, for the clip's rate. */
+ private ahead = DECODE_AHEAD_S;
/** Set when the file turns out to hold no picture this browser can read. */
private unreadable = false;
/** Failed opens so far, and the timer that will clear `unreadable` for the
@@ -849,8 +856,12 @@ export class ClipFrameSource {
* pointer backing into a clip arrives at its last frame, which is above
* everything that source was ever asked for.
*/
- want(t: number, playing: boolean, backward = false): void {
+ want(t: number, playing: boolean, backward = false, rate = 1): void {
if (this.closed || this.unreadable) return;
+ // The lookahead is source seconds tuned for a clip playing at 1×. A clip
+ // running faster eats the ring that much sooner, so the walk stays the
+ // same timeline distance ahead; the ring's own bound still caps it.
+ this.ahead = DECODE_AHEAD_S * Math.max(1, rate);
// Being read is what wakes a stood-down source; the walk below is what
// gives it a decoder again.
this.asleep = false;
@@ -1169,6 +1180,7 @@ export class ClipFrameSource {
if (this.starting || this.stream) {
const claim = walkClaim({
t,
+ ahead: this.ahead,
walkFor: this.walkFor,
from: this.streamFrom,
tail: this.streamTail,
@@ -1486,7 +1498,7 @@ export class ClipFrameSource {
// or past `t` fill the ring — pushing another would drop one that is
// still wanted. Only the walk's frames count: leftovers from other
// gestures merely age out of the cache as the walk pushes.
- if (this.streamTail >= t + DECODE_AHEAD_S) break;
+ if (this.streamTail >= t + this.ahead) break;
if (this.ring.between(Math.max(t, this.streamFrom), this.streamTail) >= RING - 1) break;
const pullAt = performance.now();
this.nextStartedAt = pullAt;
@@ -1879,14 +1891,19 @@ export class FrameSourcePool {
/**
* The decode identity of a clip: the frames it shows are a function of its
- * file, its speed, and where its source time stands at timeline zero. Two clips
- * agreeing on all three show the same picture at every instant.
+ * file, its rate, and where its source time stands at timeline zero. Two clips
+ * agreeing on all three show the same picture at every instant. A clip whose
+ * rate changes through the footage is its own map: its curve and its start.
*/
export function mappingKey(
assetId: string,
- speed: number,
+ retime: Retime,
inPoint: number,
start: number
): string {
- return `${assetId}|${speed}|${(inPoint - start * speed).toFixed(3)}`;
+ if (retime.uniform) {
+ const speed = retime.rate;
+ return `${assetId}|${speed}|${(inPoint - start * speed).toFixed(3)}`;
+ }
+ return `${assetId}|${retime.key}|${inPoint.toFixed(3)}|${start.toFixed(3)}`;
}
diff --git a/site/src/cut/lib/genvideo/docWriter.ts b/site/src/cut/lib/genvideo/docWriter.ts
index 03492791..0465e0e7 100644
--- a/site/src/cut/lib/genvideo/docWriter.ts
+++ b/site/src/cut/lib/genvideo/docWriter.ts
@@ -1,5 +1,7 @@
"use client";
+import { retimeOf } from "@donkeycut/effects-kit";
+
/**
* Background writes to a project that is NOT open in the editor. A scene run
* outlives the open project: when the user switches away, its placements,
@@ -175,10 +177,7 @@ export function docPlaceGenClip(
: 0;
const { out, speed } = fillSlot(asset.type, Math.max(MIN_LEN, asset.duration - srcIn), slot, SPEED_MIN);
const row = doc.clips.filter((c) => c.track === 0);
- const spans = row.map((c) => {
- const sp = c.speed && c.speed > 0 ? c.speed : 1;
- return { id: c.id, start: c.start, end: c.start + (c.out - c.in) / sp };
- });
+ const spans = row.map((c) => ({ id: c.id, start: c.start, end: c.start + retimeOf(c).len }));
const anchorIds = new Set(opts?.anchorAfterIds ?? []);
const prevEnd = spans.filter((s) => anchorIds.has(s.id)).reduce((m, s) => Math.max(m, s.end), -1);
const len = speed !== undefined && speed > 0 ? out / speed : out;
diff --git a/site/src/cut/lib/genvideo/editorBridge.ts b/site/src/cut/lib/genvideo/editorBridge.ts
index d3b5cd6b..fba0c590 100644
--- a/site/src/cut/lib/genvideo/editorBridge.ts
+++ b/site/src/cut/lib/genvideo/editorBridge.ts
@@ -1,5 +1,7 @@
"use client";
+import { retimeOf, type SpeedNode } from "@donkeycut/effects-kit";
+
/**
* The real EditorBridge: the orchestrator's typed hands on the live Cut
* timeline. It is the single frames↔seconds conversion boundary — the plan
@@ -35,10 +37,9 @@ export const GEN_FPS = 30;
const toSec = (frames: number) => frames / GEN_FPS;
-/** Effective timeline footprint of a clip (seconds), honoring speed. */
-function footprintSec(c: { in: number; out: number; speed?: number }): number {
- const src = c.out - c.in;
- return c.speed && c.speed > 0 ? src / c.speed : src;
+/** Effective timeline footprint of a clip (seconds), through its rate. */
+function footprintSec(c: { in: number; out: number; speed?: number; speedCurve?: SpeedNode[] }): number {
+ return retimeOf(c).len;
}
export class StoreEditorBridge implements EditorBridge {
@@ -131,16 +132,14 @@ export class StoreEditorBridge implements EditorBridge {
if ((await this.mode()) === "store") {
const clip = useEditor.getState().clips.find((c) => c.id === clipId);
if (!clip) return;
- // Trim the tail so the footprint matches, at whatever speed the clip carries.
- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;
- useEditor.getState().setClipTrim(clipId, clip.in, clip.in + toSec(durationFrames) * speed);
+ // Trim the tail so the footprint matches, at whatever rate the clip carries.
+ useEditor.getState().setClipTrim(clipId, clip.in, retimeOf(clip).srcAt(toSec(durationFrames)));
return;
}
await withProjectDoc(this.projectId, (doc) => {
doc.clips = doc.clips.map((c) => {
if (c.id !== clipId) return c;
- const speed = c.speed && c.speed > 0 ? c.speed : 1;
- return { ...c, out: c.in + toSec(durationFrames) * speed };
+ return { ...c, out: retimeOf(c).srcAt(toSec(durationFrames)) };
});
});
}
diff --git a/site/src/cut/lib/laneTracks.ts b/site/src/cut/lib/laneTracks.ts
index f0135fc1..c5e38c27 100644
--- a/site/src/cut/lib/laneTracks.ts
+++ b/site/src/cut/lib/laneTracks.ts
@@ -1,5 +1,7 @@
"use client";
+import { retimeOf } from "@donkeycut/effects-kit";
+
/**
* The lane-track coordinator: the one place for how items on the timeline's
* free-positioned tracks behave. Audio, titles, upper video layers, and
@@ -121,8 +123,6 @@ interface LaneAdapter<T> {
afterMove?(raw: T, delta: number): void;
}
-const speedOf = (c: { speed?: number }) => (c.speed && c.speed > 0 ? c.speed : 1);
-
/** The asset's beats inside a media item's trimmed range, in timeline
* seconds. Beats live on the asset in source seconds, so every clip showing
* the source maps its own window of them. */
@@ -132,9 +132,9 @@ function mediaBeatTimes(
): number[] {
const beats = s.assets.find((a) => a.id === c.assetId)?.beats?.beats;
if (!beats?.length) return [];
- const speed = speedOf(c);
+ const rt = retimeOf(c);
const times: number[] = [];
- for (const b of beats) if (b >= c.in && b <= c.out) times.push(c.start + (b - c.in) / speed);
+ for (const b of beats) if (b >= c.in && b <= c.out) times.push(c.start + rt.tAt(b));
return times;
}
@@ -142,7 +142,7 @@ function videoMaxLen(s: S, c: VideoClip): number {
const a = s.assets.find((x) => x.id === c.assetId);
// A still has no source length, so its clip can stretch to any duration.
if (a?.type === "image") return Infinity;
- return ((a?.duration ?? c.out) - c.in) / speedOf(c);
+ return retimeOf({ ...c, out: a?.duration ?? c.out }).len;
}
const clipAdapter: LaneAdapter<VideoClip> = {
@@ -156,17 +156,17 @@ const clipAdapter: LaneAdapter<VideoClip> = {
movePatch: (c, start) => ({ id: c.id, patch: { start } }),
trimLeftPatch: (c, newStart) => ({
id: c.id,
- patch: { start: newStart, in: c.in + (newStart - c.start) * speedOf(c) },
+ patch: { start: newStart, in: retimeOf(c).srcAt(newStart - c.start) },
}),
trimRightPatch: (c, newEnd) => ({
id: c.id,
- patch: { out: c.in + (newEnd - c.start) * speedOf(c) },
+ patch: { out: retimeOf(c).srcAt(newEnd - c.start) },
}),
revealLeftPatch: (c, start, reveal) => ({
id: c.id,
- patch: { start, in: c.in + (reveal - c.start) * speedOf(c) },
+ patch: { start, in: retimeOf(c).srcAt(reveal - c.start) },
}),
- leftFloor: (c) => Math.max(0, c.start - c.in / speedOf(c)),
+ leftFloor: (c) => Math.max(0, c.start + retimeOf(c).tAt(0)),
maxLen: videoMaxLen,
closesGap: true,
assetOf: (s, c) => s.assets.find((x) => x.id === c.assetId),
@@ -183,19 +183,19 @@ const audioAdapter: LaneAdapter<AudioClip> = {
movePatch: (a, start) => ({ id: a.id, patch: { start } }),
trimLeftPatch: (a, newStart) => ({
id: a.id,
- patch: { start: newStart, in: a.in + (newStart - a.start) * speedOf(a) },
+ patch: { start: newStart, in: retimeOf(a).srcAt(newStart - a.start) },
}),
trimRightPatch: (a, newEnd) => ({
id: a.id,
- patch: { out: a.in + (newEnd - a.start) * speedOf(a) },
+ patch: { out: retimeOf(a).srcAt(newEnd - a.start) },
}),
revealLeftPatch: (a, start, reveal) => ({
id: a.id,
- patch: { start, in: a.in + (reveal - a.start) * speedOf(a) },
+ patch: { start, in: retimeOf(a).srcAt(reveal - a.start) },
}),
- leftFloor: (a) => Math.max(0, a.start - a.in / speedOf(a)),
+ leftFloor: (a) => Math.max(0, a.start + retimeOf(a).tAt(0)),
maxLen: (s, a) =>
- ((s.assets.find((x) => x.id === a.assetId)?.duration ?? a.out) - a.in) / speedOf(a),
+ retimeOf({ ...a, out: s.assets.find((x) => x.id === a.assetId)?.duration ?? a.out }).len,
lanePatch: (a, lane) => ({ id: a.id, patch: { lane: lane > 0 ? lane : undefined } }),
assetOf: (s, a) => s.assets.find((x) => x.id === a.assetId),
beatTimes: mediaBeatTimes,
@@ -227,17 +227,17 @@ const overlayClipAdapter: LaneAdapter<VideoClip> = {
movePatch: (c, start) => ({ id: c.id, patch: { start } }),
trimLeftPatch: (c, newStart) => ({
id: c.id,
- patch: { start: newStart, in: c.in + (newStart - c.start) * speedOf(c) },
+ patch: { start: newStart, in: retimeOf(c).srcAt(newStart - c.start) },
}),
trimRightPatch: (c, newEnd) => ({
id: c.id,
- patch: { out: c.in + (newEnd - c.start) * speedOf(c) },
+ patch: { out: retimeOf(c).srcAt(newEnd - c.start) },
}),
revealLeftPatch: (c, start, reveal) => ({
id: c.id,
- patch: { start, in: c.in + (reveal - c.start) * speedOf(c) },
+ patch: { start, in: retimeOf(c).srcAt(reveal - c.start) },
}),
- leftFloor: (c) => Math.max(0, c.start - c.in / speedOf(c)),
+ leftFloor: (c) => Math.max(0, c.start + retimeOf(c).tAt(0)),
maxLen: videoMaxLen,
closesGap: true,
assetOf: (s, c) => s.assets.find((x) => x.id === c.assetId),
diff --git a/site/src/cut/lib/maskVideo.ts b/site/src/cut/lib/maskVideo.ts
index 631432f8..e236cf4d 100644
--- a/site/src/cut/lib/maskVideo.ts
+++ b/site/src/cut/lib/maskVideo.ts
@@ -1,5 +1,7 @@
"use client";
+import { retimeOf } from "@donkeycut/effects-kit";
+
/**
* The person-matte video for the ffmpeg export path. The page — the only
* place that can run segmentation — composes the video layers over the
@@ -37,10 +39,8 @@ export const MASK_FPS = 15;
const MASK_SHORT = 480;
/** Timeline second inside a span mapped to its source second. */
-const sourceTimeAt = (sp: ClipSpan, t: number) => {
- const speed = sp.clip.speed && sp.clip.speed > 0 ? sp.clip.speed : 1;
- return Math.min(sp.clip.out, sp.clip.in + Math.max(0, t - sp.start) * speed);
-};
+const sourceTimeAt = (sp: ClipSpan, t: number) =>
+ Math.min(sp.clip.out, retimeOf(sp.clip).srcAt(Math.max(0, t - sp.start)));
/** A clip whose picture trims by the person matte (either direction). */
const subjectClip = (c: { mask?: { kind: string } }) => c.mask?.kind === "subject";
diff --git a/site/src/cut/lib/previewMixer.test.ts b/site/src/cut/lib/previewMixer.test.ts
index 04faa7ee..31f4599b 100644
--- a/site/src/cut/lib/previewMixer.test.ts
+++ b/site/src/cut/lib/previewMixer.test.ts
@@ -1,4 +1,5 @@
import { describe, expect, mock, test } from "bun:test";
+import { retimeOf } from "@donkeycut/effects-kit";
import { AudioBuffer } from "node-web-audio-api";
/**
@@ -254,7 +255,7 @@ const voice = {
start: 0,
in: 0,
out: CLIP_S,
- speed: 1,
+ retime: retimeOf({ in: 0, out: CLIP_S, speed: 1 }),
gain: 1,
};
diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.ts
index 306f2fc7..1cf4759f 100644
--- a/site/src/cut/lib/previewMixer.ts
+++ b/site/src/cut/lib/previewMixer.ts
@@ -41,12 +41,13 @@ import {
soundRecipe,
type AudioFxNodes,
type ClipSound,
+ type Retime,
} from "@donkeycut/effects-kit";
import type { WrappedAudioBuffer } from "mediabunny";
import { assembleAudio, decodeAudioSpan, openAudioWalk, type AudioWalk } from "./mediaRead";
import { allowance, holdMemory } from "./memoryBudget";
import { meterAudioLate, meterClockJump } from "./perfTrace";
-import { timeStretch } from "./timeStretch";
+import { timeStretch, type StretchSchedule } from "./timeStretch";
/** How much of a voice is pulled off its walk and scheduled at once, in source
* seconds. Long enough that a stretch of sound is one scheduling decision;
@@ -111,7 +112,8 @@ export interface Voice {
start: number;
in: number;
out: number;
- speed: number;
+ /** The map from this voice's timeline span onto its source span. */
+ retime: Retime;
/** Everything the frame plan says about how loud it is right now. */
gain: number;
/** The clip's own treatment, run on this voice alone before it joins the
@@ -127,7 +129,7 @@ interface Scheduled {
interface LiveVoice {
url: string;
- speed: number;
+ retime: Retime;
in: number;
out: number;
start: number;
@@ -452,7 +454,7 @@ export class PreviewMixer {
// read aims the walk and pulls the bytes it sits in — the primer a
// parked playhead uses — so the release finds an open file already
// read where the first window starts.
- const source = live.in + (from - live.start) * live.speed;
+ const source = live.retime.srcAt(from - live.start);
if (!live.filling && source < live.out - 1e-3) void this.prime(live, source);
}
}
@@ -580,7 +582,7 @@ export class PreviewMixer {
(prev.start !== v.start ||
prev.in !== v.in ||
prev.out !== v.out ||
- prev.speed !== v.speed)
+ prev.retime.key !== v.retime.key)
) {
this.release(prev);
this.voices.delete(v.id);
@@ -660,7 +662,7 @@ export class PreviewMixer {
prev.start !== v.start ||
prev.in !== v.in ||
prev.out !== v.out ||
- prev.speed !== v.speed)
+ prev.retime.key !== v.retime.key)
) {
this.release(prev);
this.voices.delete(v.id);
@@ -671,7 +673,7 @@ export class PreviewMixer {
this.treat(live, v.sound);
const from = Math.max(v.start, t);
if (live.scheduled !== from) this.reaim(live, from);
- const source = live.in + (from - live.start) * live.speed;
+ const source = live.retime.srcAt(from - live.start);
// Already reading, at the end of its source, or standing close enough
// that the play's first read is the one it was going to make anyway.
if (live.filling || source >= live.out - 1e-3) continue;
@@ -746,7 +748,7 @@ export class PreviewMixer {
head.connect(gain);
const live: LiveVoice = {
url: v.url,
- speed: v.speed,
+ retime: v.retime,
in: v.in,
out: v.out,
start: v.start,
@@ -843,7 +845,7 @@ export class PreviewMixer {
const current = () => this.voices.get(id) === live && live.gen === gen;
try {
const from = live.scheduled;
- const sourceFrom = live.in + (from - live.start) * live.speed;
+ const sourceFrom = live.retime.srcAt(from - live.start);
if (sourceFrom >= live.out - 1e-3) {
live.ended = true;
return;
@@ -855,7 +857,7 @@ export class PreviewMixer {
// waiting. The first window is short so the sound joins it now.
const group = live.windows.length ? GROUP_S : FIRST_GROUP_S;
buffer =
- Math.abs(live.speed - 1) > 1e-3
+ !live.retime.uniform || Math.abs(live.retime.rate - 1) > 1e-3
? await this.stretchedWindow(live, sourceFrom)
: await this.walkGroup(live, sourceFrom, current, group);
} catch {
@@ -972,13 +974,25 @@ export class PreviewMixer {
* everything older is a full decode away anyway.
*/
private async stretchedWindow(live: LiveVoice, sourceFrom: number): Promise<AudioBuffer | null> {
- const sourceTo = Math.min(live.out, sourceFrom + WINDOW_STRETCHED_S * live.speed);
- const key = `${live.url}|${sourceFrom.toFixed(3)}|${sourceTo.toFixed(3)}|${live.speed}`;
+ const rt = live.retime;
+ const tFrom = rt.tAt(sourceFrom);
+ const sourceTo = Math.min(live.out, rt.srcAt(tFrom + WINDOW_STRETCHED_S));
+ const key = `${live.url}|${sourceFrom.toFixed(3)}|${sourceTo.toFixed(3)}|${rt.key}`;
const held = this.decoded.get(key);
if (held) return held;
const raw = await decodeAudioSpan(live.url, sourceFrom, sourceTo);
if (!raw) return null;
- const buffer = this.stretch(raw, 1 / live.speed);
+ // A curve's window is laid by the clip's own map from where the window
+ // starts, so its length is exactly the timeline the picture spends on it.
+ const buffer = this.stretch(
+ raw,
+ rt.uniform
+ ? 1 / rt.rate
+ : {
+ factorAt: (sec) => 1 / rt.rateAtSrc(sourceFrom + sec),
+ outLength: Math.round((rt.tAt(sourceFrom + raw.duration) - tFrom) * raw.sampleRate),
+ }
+ );
// Oldest first until the cache is back inside its share. Emptying the
// whole cache would throw away the window the playhead is standing on along
// with the ones it has finished, and the next frame would pay a full decode
@@ -1011,7 +1025,7 @@ export class PreviewMixer {
live.filling = false;
live.ended = false;
live.scheduled = timeline;
- live.walk?.seek(live.in + (timeline - live.start) * live.speed);
+ live.walk?.seek(live.retime.srcAt(timeline - live.start));
}
/** A read that failed, came back empty, or stopped answering. The voice
@@ -1038,7 +1052,7 @@ export class PreviewMixer {
}
/** Re-lay a buffer at a different length, keeping its pitch. */
- private stretch(buffer: AudioBuffer, factor: number): AudioBuffer {
+ private stretch(buffer: AudioBuffer, factor: number | StretchSchedule): AudioBuffer {
const channels: Float32Array[] = [];
for (let c = 0; c < buffer.numberOfChannels; c++) channels.push(buffer.getChannelData(c));
const out = timeStretch(channels, buffer.sampleRate, factor);
diff --git a/site/src/cut/lib/removalVideo.ts b/site/src/cut/lib/removalVideo.ts
index 3ad32ce6..640335fa 100644
--- a/site/src/cut/lib/removalVideo.ts
+++ b/site/src/cut/lib/removalVideo.ts
@@ -10,11 +10,10 @@
* channel, which is why the layer travels as a pair.
*/
-import { matteLumaToAlpha, removalActive } from "@donkeycut/effects-kit";
+import { matteLumaToAlpha, removalActive, retimeOf } from "@donkeycut/effects-kit";
import { openCanvasVideo, scaledEvenSize } from "./canvasVideo";
import { FrameCompositor, type Frame } from "./composite";
import { createRasterCanvas, decodeRasterImageUrl, type RasterSurface } from "./raster";
-import { clipSpeed } from "./store";
import type { MediaAsset, VideoClip } from "./types";
import { liveReader } from "./liveReader";
@@ -72,9 +71,9 @@ export async function renderRemovalPieces(
const matteAsset = r.matte ? assets.find((a) => a.id === r.matte!.assetId) : undefined;
if (!matteAsset) return null;
- const speed = clipSpeed(clip);
+ const rt = retimeOf(clip);
const still = asset.type === "image";
- const dur = Math.max(0.1, (clip.out - clip.in) / speed);
+ const dur = Math.max(0.1, rt.len);
const frames = Math.max(1, Math.ceil(dur * opts.fps));
const baked = clip.removal!.matte;
@@ -143,7 +142,7 @@ export async function renderRemovalPieces(
for (let i = 0; i < frames; i++) {
const s = Math.min(dur - 1 / (opts.fps * 2), i / opts.fps);
- const srcT = still ? 0 : clip.in + s * speed;
+ const srcT = still ? 0 : rt.srcAt(s);
const frame = await reader.frameAt(srcT);
if (frame.kind !== "ready") {
rgbCtx.fillStyle = "#000000";
diff --git a/site/src/cut/lib/store.test.ts b/site/src/cut/lib/store.test.ts
index e569a22e..bcac4d13 100644
--- a/site/src/cut/lib/store.test.ts
+++ b/site/src/cut/lib/store.test.ts
@@ -1874,6 +1874,19 @@ describe("transcribe spec", () => {
expect(spec.clips.find((c) => c.file === video.fileName)?.muted).toBe(false);
});
+ test("a curved clip ships its curve, so the mix lays it at the length it plays", () => {
+ const video = asset(6);
+ const clip = vclip({ assetId: video.id, start: 0, in: 0, out: 4, speedCurve: [[0, 0.5], [4, 0.5]] });
+ const spec = cutTranscribeSpec({
+ clips: [clip],
+ assets: [video],
+ audioClips: [],
+ overlays: [],
+ })!;
+ expect(spec.clips[0].speedCurve).toEqual([[0, 0.5], [4, 0.5]]);
+ expect(spec.duration).toBeCloseTo(8, 3);
+ });
+
test("leaves a still on an upper track out of the audio items", () => {
const video = asset(6);
const still = { ...asset(0, "image"), fileName: "still.png" };
@@ -1923,3 +1936,84 @@ describe("assetIdsInUse", () => {
expect(assetIdsInUse({ clips: [], audioClips: [], overlays: [] }).size).toBe(0);
});
});
+
+describe("speed curves", () => {
+ test("a curve that lengthens a track-0 clip pushes the run behind it", () => {
+ const a = asset(4);
+ const c1 = vclip({ assetId: a.id, start: 0, in: 0, out: 2 });
+ const c2 = vclip({ assetId: a.id, start: 2, in: 0, out: 2 });
+ useEditor.setState({ assets: [a], clips: [c1, c2] });
+ useEditor.getState().setClipSpeedCurve(c1.id, [
+ [0, 0.5],
+ [2, 0.5],
+ ]);
+ const s = useEditor.getState();
+ expect(clipLen(s.clips[0])).toBeCloseTo(4, 3);
+ expect(s.clips[1].start).toBeCloseTo(4, 3);
+ // Leaving curve mode keeps the length as a uniform rate.
+ useEditor.getState().setClipSpeedCurve(c1.id, undefined);
+ const t = useEditor.getState();
+ expect(t.clips[0].speedCurve).toBeUndefined();
+ expect(t.clips[0].speed).toBeCloseTo(0.5, 3);
+ expect(t.clips[1].start).toBeCloseTo(4, 3);
+ });
+
+ test("a crossing's handles read the rate at their own edge", () => {
+ const a = asset(10);
+ // Trimmed the same distance from both ends, so an edge picked by the
+ // amount of source left would read the wrong one.
+ const c1 = vclip({
+ assetId: a.id,
+ start: 0,
+ in: 2,
+ out: 8,
+ speedCurve: [[2, 1], [8, 4]],
+ });
+ // The next clip abuts, so there is a cut for the crossing to play on.
+ const c2 = vclip({ assetId: a.id, start: clipLen(c1), in: 2, out: 8 });
+ useEditor.setState({ assets: [a], clips: [c1, c2] });
+ useEditor.getState().setClipTransition(c1.id, 1.4, "audiocross");
+ const s = useEditor.getState();
+ const [sp] = getClipSpans(s.clips, s.assets);
+ // Two source seconds past the out point, played at the tail's 4x: half a
+ // timeline second, short of the 0.7 the crossing wanted.
+ expect(sp.soundAhead).toBeCloseTo(0.5, 3);
+ });
+
+ test("a picked speed node that is gone leaves the clip's pose keys alone", () => {
+ const a = asset(4);
+ const c1 = vclip({
+ assetId: a.id,
+ start: 0,
+ in: 0,
+ out: 2,
+ kf: [{ t: 1, x: 0.5, y: 0.5, scale: 1.2, rotation: 0, opacity: 1 }],
+ });
+ useEditor.setState({
+ assets: [a],
+ clips: [c1],
+ selection: { kind: "clip", id: c1.id },
+ // The strip left this behind; the clip carries no curve any more.
+ selectedKey: { kind: "clip", id: c1.id, t: 1, track: "speed" },
+ });
+ useEditor.getState().deleteSelection();
+ const s = useEditor.getState();
+ expect(s.clips).toHaveLength(1);
+ expect(s.clips[0].kf).toHaveLength(1);
+ });
+
+ test("Delete takes a picked node, never the clip", () => {
+ const a = asset(4);
+ const c1 = vclip({ assetId: a.id, start: 0, in: 0, out: 2, speedCurve: [[0, 1], [1, 2], [2, 1]] });
+ useEditor.setState({
+ assets: [a],
+ clips: [c1],
+ selection: { kind: "clip", id: c1.id },
+ selectedKey: { kind: "clip", id: c1.id, t: 1, track: "speed" },
+ });
+ useEditor.getState().deleteSelection();
+ const s = useEditor.getState();
+ expect(s.clips).toHaveLength(1);
+ expect(s.clips[0].speedCurve?.map((n) => n[0])).toEqual([0, 2]);
+ });
+});
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index 997280ac..1f5826f6 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -8,10 +8,14 @@ import {
lineLikeShape,
maskKeyAt,
removeKeyAt,
+ retimeOf,
+ speedCurveOf,
+ speedCurvePreset,
upsertKey,
type EffectId,
type MaskKey,
type OverlayKey,
+ type SpeedNode,
} from "@donkeycut/effects-kit";
import { create } from "zustand";
import type {
@@ -299,8 +303,9 @@ export interface EditorState {
selectedKey: {
kind: "overlay" | "clip";
id: string;
+ /** The key's time; for a speed node, the source second it sits on. */
t: number;
- track: "pose" | "mask";
+ track: "pose" | "mask" | "speed";
} | null;
playing: boolean;
/** Set while a play is held at a standstill because the cut has nothing to
@@ -439,6 +444,12 @@ export interface EditorState {
/** Set a clip's playback rate (0.25–4). The footprint change carries the
* timeline behind it the way a trim does (see resizeClipFootprint). */
setClipSpeed: (id: string, speed: number) => void;
+ /** Replace a clip's speed curve (undefined clears it back to a uniform rate
+ * at the curve's average). Nodes are [source second, rate]; the footprint
+ * change carries the timeline like setClipSpeed does. */
+ setClipSpeedCurve: (id: string, nodes: SpeedNode[] | undefined) => void;
+ /** Lay a preset ramp over the clip's trimmed span. */
+ setClipSpeedPreset: (id: string, preset: string) => void;
/** Set a clip's source trim points. While track 0 is the only video track,
* a track-0 resize ripples: everything past the clip's tail rides the moved
* edge both directions, gaps keeping their width. Otherwise the run rules
@@ -815,7 +826,7 @@ const touches = (patch: object, keys: readonly string[]) => keys.some((k) => k i
*/
function settleClipFootprint(id: string, patch: Partial<VideoClip>) {
const moved = touches(patch, ["start", "track"]);
- if (!moved && !touches(patch, ["in", "out", "speed"])) return;
+ if (!moved && !touches(patch, ["in", "out", "speed", "speedCurve"])) return;
const st = useEditor.getState();
const clip = st.clips.find((c) => c.id === id);
if (!clip) return;
@@ -1300,6 +1311,7 @@ export function cutTranscribeSpec(
start: a.start,
volume: a.volume,
speed: a.speed,
+ speedCurve: a.speedCurve,
}))
.concat(
overlayLayers(s.clips)
@@ -1318,6 +1330,7 @@ export function cutTranscribeSpec(
start: c.start,
volume: 1,
speed: c.speed,
+ speedCurve: c.speedCurve,
}))
);
if (spans.length === 0 && audio.length === 0) return null;
@@ -1339,6 +1352,7 @@ export function cutTranscribeSpec(
out: sp.clip.out,
muted: sp.clip.muted || assetIsSilent(sp.asset),
speed: clipSpeed(sp.clip),
+ speedCurve: sp.clip.speedCurve,
// The clamped cross-dissolve overlap, so the mix overlaps clip
// audio the same way the timeline does and cues stay in sync.
transition: sp.transitionOut,
@@ -2567,7 +2581,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
if (
get().transitions.length &&
!touches(patch, ["start", "track"]) &&
- touches(patch, ["in", "out", "speed"])
+ touches(patch, ["in", "out", "speed", "speedCurve"])
)
set((s) => ({ transitions: reanchorTransitions(before, s.clips, s.transitions) }));
},
@@ -2576,15 +2590,49 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
const clip = get().clips.find((c) => c.id === id);
if (!clip) return;
const clamped = Math.max(SPEED_FLOOR, speed);
- if (Math.abs(clamped - clipSpeed(clip)) < 1e-4) return;
- resizeClipFootprint(clip, { speed: clamped }, Math.max(MIN_LEN, (clip.out - clip.in) / clamped));
+ // A uniform rate replaces any curve the clip carried.
+ if (!clip.speedCurve && Math.abs(clamped - clipSpeed(clip)) < 1e-4) return;
+ resizeClipFootprint(
+ clip,
+ { speed: clamped, speedCurve: undefined },
+ Math.max(MIN_LEN, (clip.out - clip.in) / clamped)
+ );
+ },
+
+ setClipSpeedCurve: (id, nodes) => {
+ const clip = get().clips.find((c) => c.id === id);
+ if (!clip) return;
+ const cleaned = nodes ? speedCurveOf({ speedCurve: nodes }) : undefined;
+ if (!cleaned) {
+ if (!clip.speedCurve) return;
+ // Leaving curve mode keeps the clip's length: the uniform rate is the
+ // curve's average.
+ const avg = Math.max(SPEED_FLOOR, retimeOf(clip).rate);
+ resizeClipFootprint(clip, { speed: avg, speedCurve: undefined }, Math.max(MIN_LEN, retimeOf({ ...clip, speed: avg, speedCurve: undefined }).len));
+ return;
+ }
+ const next = { ...clip, speedCurve: cleaned };
+ if (retimeOf(next).key === retimeOf(clip).key) return;
+ resizeClipFootprint(clip, { speedCurve: cleaned }, Math.max(MIN_LEN, retimeOf(next).len));
+ },
+
+ setClipSpeedPreset: (id, preset) => {
+ const clip = get().clips.find((c) => c.id === id);
+ if (!clip) return;
+ const nodes = speedCurvePreset(preset, clip.in, clip.out);
+ if (!nodes) return;
+ get().setClipSpeedCurve(id, nodes);
},
setClipTrim: (id, nextIn, nextOut) => {
const clip = get().clips.find((c) => c.id === id);
if (!clip) return;
if (Math.abs(nextIn - clip.in) < 1e-6 && Math.abs(nextOut - clip.out) < 1e-6) return;
- resizeClipFootprint(clip, { in: nextIn, out: nextOut }, (nextOut - nextIn) / clipSpeed(clip));
+ resizeClipFootprint(
+ clip,
+ { in: nextIn, out: nextOut },
+ retimeOf({ ...clip, in: nextIn, out: nextOut }).len
+ );
},
setClipTransition: (id, seconds, style) => {
@@ -2951,7 +2999,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
setClipMaskKey: (id, tLocal, patch, opts) => {
const c = get().clips.find((x) => x.id === id);
if (!c?.mask) return;
- const len = Math.max(0.1, (c.out - c.in) / clipSpeed(c));
+ const len = clipLen(c);
const t = Math.max(0, Math.min(tLocal, len));
const existing = c.mask.kf?.find((k) => Math.abs(k.t - t) <= KEY_EPSILON);
const next = { ...(existing ?? maskKeyAt(c.mask, t)), ...patch, t };
@@ -3289,6 +3337,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
// Carry the clip's rate so the detached track keeps the muted picture's
// length and stays in sync (an AudioClip plays at its own `speed`).
...(clipSpeed(clip) !== 1 ? { speed: clipSpeed(clip) } : {}),
+ ...(clip.speedCurve ? { speedCurve: clip.speedCurve } : {}),
};
set((s) => ({
audioClips: [...s.audioClips, audio],
@@ -3325,11 +3374,11 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
if (selection?.kind === "clip") {
const c = get().clips.find((x) => x.id === selection.id);
if (c && c.track !== 0) {
- const sp = c.speed && c.speed > 0 ? c.speed : 1;
- const eff = (c.out - c.in) / sp;
+ const rt = retimeOf(c);
+ const eff = rt.len;
if (t > c.start + 0.05 && t < c.start + eff - 0.05) {
push();
- const cutIn = c.in + (t - c.start) * sp;
+ const cutIn = rt.srcAt(t - c.start);
// The left half hard-cuts into the right; the right keeps the
// original dissolve into whatever came after (same as track 0).
// The cut lands mid-footage, so the edges it creates stay plain:
@@ -3403,8 +3452,8 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
);
if (!span) return;
push();
- // Source time advances `speed`× faster than timeline time.
- const cutAt = span.clip.in + (t - span.start) * clipSpeed(span.clip);
+ // The clip's own map says which source second plays at `t`.
+ const cutAt = retimeOf(span.clip).srcAt(t - span.start);
// The left half hard-cuts into the right; the right keeps the original
// dissolve into whatever came after. Both halves stay in place. The cut
// lands mid-footage, so the edges it creates stay plain: the exit
@@ -3425,6 +3474,18 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
// A picked keyframe is the smaller thing under the cursor: Delete takes
// the key and leaves the item it belongs to alone.
const key = st.selectedKey;
+ if (key?.track === "speed") {
+ // A speed node is the whole of this branch: its times are source
+ // seconds, so falling through to the pose track would delete a
+ // keyframe that merely shares the number.
+ set({ selectedKey: null });
+ const clip = st.clips.find((x) => x.id === key.id);
+ const nodes = clip ? speedCurveOf(clip) : undefined;
+ if (nodes && nodes.length > 1 && nodes.some((nd) => Math.abs(nd[0] - key.t) <= KEY_EPSILON)) {
+ st.setClipSpeedCurve(key.id, nodes.filter((nd) => Math.abs(nd[0] - key.t) > KEY_EPSILON));
+ }
+ return;
+ }
if (key) {
set({ selectedKey: null });
const item =
@@ -3596,10 +3657,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
// A layer clip carries no span (spans are track 0); use its
// own footprint.
const c = s.clips.find((x) => x.id === sel.id);
- if (c) {
- const speed = c.speed && c.speed > 0 ? c.speed : 1;
- add(c.start, c.start + Math.max(0.1, (c.out - c.in) / speed));
- }
+ if (c) add(c.start, c.start + clipLen(c));
}
} else if (sel.kind === "audio") {
const c = s.audioClips.find((x) => x.id === sel.id);
@@ -3650,20 +3708,20 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
if (mi == null) continue;
// Track-0 clips re-materialize onto track 0 (asClip), so a template
// stands up its own video instead of an empty timeline.
- layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, sound: sp.clip.sound, track: 1, asClip: true });
+ layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, speedCurve: sp.clip.speedCurve, sound: sp.clip.sound, track: 1, asClip: true });
} else {
const c = s.clips.find((x) => x.id === sel.id);
if (!c) continue;
const mi = mediaFor(c.assetId);
if (mi == null) continue;
- layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, sound: c.sound, track: c.track + 1 });
+ layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, speedCurve: c.speedCurve, sound: c.sound, track: c.track + 1 });
}
} else if (sel.kind === "audio") {
const c = s.audioClips.find((x) => x.id === sel.id);
if (!c) continue;
const mi = mediaFor(c.assetId);
if (mi == null) continue;
- audio.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, volume: c.volume, fadeIn: c.fadeIn, fadeOut: c.fadeOut, speed: c.speed, sound: c.sound, duck: c.duck, lane: c.lane });
+ audio.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, volume: c.volume, fadeIn: c.fadeIn, fadeOut: c.fadeOut, speed: c.speed, speedCurve: c.speedCurve, sound: c.sound, duck: c.duck, lane: c.lane });
} else if (sel.kind === "overlay") {
const o = s.overlays.find((x) => x.id === sel.id);
// Asset-backed stickers stay out: a template copies only the media
@@ -3793,6 +3851,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
muted: l.muted,
...templateFraming(l),
...(l.speed ? { speed: l.speed } : {}),
+ ...(l.speedCurve ? { speedCurve: l.speedCurve } : {}),
...(l.sound ? { sound: l.sound } : {}),
}));
const topTrack = Math.max(0, ...overlayLayers(get().clips).map((c) => c.track));
@@ -3811,6 +3870,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
muted: l.muted,
...templateFraming(l),
...(l.speed ? { speed: l.speed } : {}),
+ ...(l.speedCurve ? { speedCurve: l.speedCurve } : {}),
...(l.sound ? { sound: l.sound } : {}),
}));
const newAudio: AudioClip[] = template.audio
@@ -3825,6 +3885,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
...(a.fadeIn ? { fadeIn: a.fadeIn } : {}),
...(a.fadeOut ? { fadeOut: a.fadeOut } : {}),
...(a.speed ? { speed: a.speed } : {}),
+ ...(a.speedCurve ? { speedCurve: a.speedCurve } : {}),
...(a.sound ? { sound: a.sound } : {}),
...(a.duck !== undefined && a.duck < 1 ? { duck: a.duck } : {}),
...(a.lane ? { lane: a.lane } : {}),
@@ -4036,6 +4097,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
out: sp.clip.out,
muted: false,
speed: clipSpeed(sp.clip),
+ speedCurve: sp.clip.speedCurve,
transition: 0,
},
],
@@ -5035,18 +5097,17 @@ export function serializeDoc(s: {
};
}
-/** Effective playback rate of a video clip (>0, default 1). */
+/** A clip's uniform playback rate (>0, default 1). A clip carrying a speed
+ * curve has no single rate; read `retimeOf(c)` for its map. */
export function clipSpeed(c: VideoClip) {
const s = c.speed ?? 1;
return s > 0 ? s : 1;
}
+/** Timeline footprint of a clip: its source span through its rate, or through
+ * its speed curve when it carries one (see retimeOf). */
export function clipLen(c: VideoClip | AudioClip) {
- const src = c.out - c.in;
- // Video clips play their source at `speed`, so the timeline footprint is
- // shorter/longer than the source. Audio clips have no speed.
- const eff = "speed" in c && c.speed && c.speed > 0 ? src / c.speed : src;
- return Math.max(MIN_LEN, eff);
+ return Math.max(MIN_LEN, retimeOf(c).len);
}
/** The longest fade a clip takes: half its footprint, so a fade in and a
@@ -5504,8 +5565,8 @@ function buildClipSpans(clips: VideoClip[], assets: MediaAsset[], track: number)
len,
transitionOut: onSound ? 0 : overlap,
soundOut,
- soundAhead: handleSeconds(soundOut, asset.duration - clip.out, clip),
- soundBack: handleSeconds(prevSound, clip.in, clip),
+ soundAhead: handleSeconds(soundOut, asset.duration - clip.out, clip, "tail"),
+ soundBack: handleSeconds(prevSound, clip.in, clip, "head"),
});
}
return spans;
@@ -5513,9 +5574,18 @@ function buildClipSpans(clips: VideoClip[], assets: MediaAsset[], track: number)
/** How far a cross dissolve reaches into a clip's handle: the ramp half it
* wants, capped by the source seconds actually there, in timeline seconds. */
-function handleSeconds(want: number, sourceLeft: number, clip: VideoClip): number {
+function handleSeconds(
+ want: number,
+ sourceLeft: number,
+ clip: VideoClip,
+ edge: "head" | "tail"
+): number {
if (want <= 0.01) return 0;
- return Math.max(0, Math.min(want, Math.max(0, sourceLeft) / clipSpeed(clip)));
+ // The handle plays at the pace of the footage beside the edge: a source
+ // second past `out` lasts as long as the last source second inside it.
+ const rt = retimeOf(clip);
+ const edgeRate = rt.rateAtSrc(edge === "head" ? clip.in : clip.out);
+ return Math.max(0, Math.min(want, Math.max(0, sourceLeft) / edgeRate));
}
/** One row of the timeline, whichever kind of thing sits on it. Video rows are
diff --git a/site/src/cut/lib/visualFrames.ts b/site/src/cut/lib/visualFrames.ts
index 8ed968cb..6b9c0b4a 100644
--- a/site/src/cut/lib/visualFrames.ts
+++ b/site/src/cut/lib/visualFrames.ts
@@ -1,5 +1,7 @@
"use client";
+import { retimeOf } from "@donkeycut/effects-kit";
+
import { framesAt } from "./mediaRead";
import { rasterCanvasToDataUrl } from "./raster";
import type { ClipSpan } from "./types";
@@ -76,8 +78,7 @@ export async function captureTimelineFrames(spans: ClipSpan[]): Promise<Captured
const at = ((i + 0.5) * total) / count;
const span = visible.find((sp) => at >= sp.start && at < sp.start + sp.len);
if (!span) continue;
- const speed = span.clip.speed && span.clip.speed > 0 ? span.clip.speed : 1;
- const srcTime = Math.max(0, span.clip.in + (at - span.start) * speed);
+ const srcTime = Math.max(0, retimeOf(span.clip).srcAt(at - span.start));
const group = bySource.get(span.asset.url) ?? { url: span.asset.url, moments: [] };
group.moments.push({ at, srcTime });
bySource.set(span.asset.url, group);