Open source CapCut alternative.
TypeScript
feat(site): the editor plays, mixes and draws through the clip's map
David Leff46f8aad137
diff --git a/site/src/cut/components/Preview.tsx b/site/src/cut/components/Preview.tsxindex 41af7299..72f1b62b 100644--- a/site/src/cut/components/Preview.tsx+++ b/site/src/cut/components/Preview.tsx@@ -11,7 +11,7 @@ import {usePreviewSelector,useSkim,} from "@/cut/lib/playhead";-import { projectDuration, useEditor } from "@/cut/lib/store";+import { clipLen, projectDuration, useEditor } from "@/cut/lib/store";import { copyStageFrame, primeStageFrame, stageHasFrame } from "@/cut/lib/stageFrame";import {capturePoster,@@ -505,8 +505,7 @@ export function Preview() {if (c.hidden) continue;const r = rectOf(c);if (isFullRect(r)) continue;- const speed = c.speed && c.speed > 0 ? c.speed : 1;- const len = Math.max(0.1, (c.out - c.in) / speed);+ const len = clipLen(c);if (t < c.start || t >= c.start + len) continue;if (px < r.x || px > r.x + r.w || py < r.y || py > r.y + r.h) continue;if (!best || c.track > best.track) best = { id: c.id, track: c.track };@@ -858,8 +857,7 @@ function ClipTransformGizmo({ stage }: { stage: Stage }) {const brushClipId = useBrushUi((s) => s.clipId);const live = usePreviewSelector((t) => {if (!selectedClip) return false;- const speed = selectedClip.speed && selectedClip.speed > 0 ? selectedClip.speed : 1;- const len = Math.max(0.1, (selectedClip.out - selectedClip.in) / speed);+ const len = clipLen(selectedClip);return t >= selectedClip.start && t < selectedClip.start + len;});// While hover-scrubbing the preview shows the skimmer's frame, where the@@ -1238,8 +1236,7 @@ function ClipMaskGizmo({ stage }: { stage: { w: number; h: number } }) {if (skimTime !== null) return null;const clip = masked;if (!armed || !clip?.mask) return null;- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;- const len = Math.max(0.1, (clip.out - clip.in) / speed);+ const len = clipLen(clip);if (tLocal < 0 || tLocal >= len) return null;const rect = rectOf(clip);const pose = clipKeyed(clip) ? clipPoseAt(clip, tLocal) : null;diff --git a/site/src/cut/components/RemovalBrush.tsx b/site/src/cut/components/RemovalBrush.tsxindex 9082a47a..e6f534c3 100644--- a/site/src/cut/components/RemovalBrush.tsx+++ b/site/src/cut/components/RemovalBrush.tsx@@ -1,5 +1,7 @@"use client";+import { retimeOf } from "@donkeycut/effects-kit";+import { useEffect, useRef, useState } from "react";import { createPortal } from "react-dom";import { useBrushUi } from "@/cut/lib/removal/brushUi";@@ -56,8 +58,7 @@ export function RemovalBrush({ stage }: { stage: { w: number; h: number } }) {// pointer rides the timeline and the overlay tracks the frame on screen.const tLocal = usePreviewSelector((t) => (armed && clip ? t - clip.start : -1));if (!armed || !clip) return null;- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;- const len = Math.max(0.1, (clip.out - clip.in) / speed);+ const len = Math.max(0.1, retimeOf(clip).len);if (tLocal < 0 || tLocal >= len) return null;const asset = assets.find((a) => a.id === clip.assetId);if (!asset?.width || !asset?.height) return null;@@ -132,7 +133,7 @@ function BrushSurface({if (ctx) s.paintOverlay(ctx, c.width, c.height);};- const srcT = () => clip.in + Math.max(0, tLocal) * (clip.speed && clip.speed > 0 ? clip.speed : 1);+ const srcT = () => retimeOf(clip).srcAt(Math.max(0, tLocal));// The overlay shows the strokes recorded on the frame the preview shows:// the mask rebuilds when the seeds change from outside (undo, redo, the@@ -142,8 +143,7 @@ function BrushSurface({// stored paint, which has no place inside a frame tick.const seeds = clip.removal?.seeds;const playing = useEditor((s) => s.playing);- const speedNow = clip.speed && clip.speed > 0 ? clip.speed : 1;- const srcNow = clip.in + Math.max(0, tLocal) * speedNow;+ const srcNow = retimeOf(clip).srcAt(Math.max(0, tLocal));const activeT = playing? null: ((seeds?.paint ?? []).find((p) => Math.abs(p.t - srcNow) < 0.05)?.t ?? null);diff --git a/site/src/cut/components/RemovalPanel.tsx b/site/src/cut/components/RemovalPanel.tsxindex e240e089..d450c644 100644--- a/site/src/cut/components/RemovalPanel.tsx+++ b/site/src/cut/components/RemovalPanel.tsx@@ -27,6 +27,7 @@ import {STROKE_FEATHER_MAX,STROKE_OFFSET_MAX,STROKE_WIDTH_MAX,+ retimeOf,} from "@donkeycut/effects-kit";import {removalActive,@@ -368,13 +369,13 @@ function CustomControls({ clip }: { clip: VideoClip }) {.sort((a, b) => a - b);const showStrokes = () => {if (!strokeTimes.length) return;- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;- const srcNow = clip.in + Math.max(0, playheadAt() - clip.start) * speed;+ const rt = retimeOf(clip);+ const srcNow = rt.srcAt(Math.max(0, playheadAt() - clip.start));const next = strokeTimes.find((t) => t > srcNow + 0.05) ?? strokeTimes[0];- const end = clip.start + (clip.out - clip.in) / speed;+ const end = clip.start + rt.len;useEditor.getState()- .seek(Math.min(Math.max(clip.start, clip.start + (next - clip.in) / speed), end - 0.001));+ .seek(Math.min(Math.max(clip.start, clip.start + rt.tAt(next)), end - 0.001));};return (<>diff --git a/site/src/cut/components/TemplateCard.tsx b/site/src/cut/components/TemplateCard.tsxindex 93d3e9ce..da0cb82a 100644--- a/site/src/cut/components/TemplateCard.tsx+++ b/site/src/cut/components/TemplateCard.tsx@@ -33,7 +33,7 @@ import { refFromTemplate, useAssetDrop, type AssetRef } from "@/cut/lib/assetRefimport { useRefCopy } from "@/cut/lib/refCopy";import { cardIconButton } from "@/cut/components/iconButton";import { formatTime } from "@/cut/lib/time";-import { EFFECT_LABELS } from "@donkeycut/effects-kit";+import { EFFECT_LABELS, retimeOf, type SpeedNode } from "@donkeycut/effects-kit";import { SHAPE_LABELS } from "@/cut/lib/types";import type { LibraryTemplate, TemplateMedia } from "@/cut/lib/types";import { cn } from "@/lib/utils";@@ -105,8 +105,8 @@ export function TemplateCard({// The contents, one row per item, in save order: clips and stills, sounds,// titles, then a caption count.- const speedLen = (x: { in: number; out: number; speed?: number }) =>- (x.out - x.in) / (x.speed && x.speed > 0 ? x.speed : 1);+ const speedLen = (x: { in: number; out: number; speed?: number; speedCurve?: SpeedNode[] }) =>+ retimeOf(x).len;const parts: {icon: typeof Film;label: string;diff --git a/site/src/cut/components/TransitionsPanel.tsx b/site/src/cut/components/TransitionsPanel.tsxindex 716ec11d..59ef2102 100644--- a/site/src/cut/components/TransitionsPanel.tsx+++ b/site/src/cut/components/TransitionsPanel.tsx@@ -138,7 +138,7 @@ function useCutFrames(): { a: string | null; b: string | null } {if (dist >= bestDist) continue;bestDist = dist;best = {- a: { asset: out.asset, srcT: out.clip.in + out.len * (out.clip.speed ?? 1) },+ a: { asset: out.asset, srcT: out.clip.out },b: { asset: inc.asset, srcT: inc.clip.in },};}diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.tsindex 69f0aa02..42d29dd5 100644--- a/site/src/cut/hooks/usePlayback.ts+++ b/site/src/cut/hooks/usePlayback.ts@@ -1,14 +1,14 @@"use client";+import { useEffect, type RefObject } from "react";import {- clipSpeed,getClipSpans,overlayLayers,projectDuration,useEditor,} from "@/cut/lib/store";-import { matteLumaToAlpha } from "@donkeycut/effects-kit";+import { matteLumaToAlpha, retimeOf } from "@donkeycut/effects-kit";import { playheadAt, previewAt, setPlayhead, subscribePlayhead } from "@/cut/lib/playhead";import { assetIsSilent, clipCovers, projectFadeSeconds, rectOf } from "@/cut/lib/types";import type { ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";@@ -78,7 +78,7 @@ const DECODER_BUDGET = 12;/** Source time of a clip at timeline time `t`. */const sourceTimeOf = (clip: VideoClip, t: number) =>- clip.in + Math.max(0, t - clip.start) * clipSpeed(clip);+ retimeOf(clip).srcAt(Math.max(0, t - clip.start));/*** When the preview gives up resolution to keep the picture with the sound.@@ -110,7 +110,7 @@ const PLAY_SETTLE_MS = 6_000;/** The decode identity of a clip — see `mappingKey`. */const keyOf = (clip: VideoClip, asset: MediaAsset) =>- mappingKey(asset.id, clipSpeed(clip), clip.in, clip.start);+ mappingKey(asset.id, retimeOf(clip), clip.in, clip.start);let engineSerial = 0;@@ -135,13 +135,18 @@ export function engineLog(msg: string): void {* schedules one stable voice and the frame-by-frame gain is all that moves.*/const voiceSpan = (sp: ClipSpan) => {- const speed = clipSpeed(sp.clip);+ // The handles a crossing reaches into play at the pace of the footage+ // beside them, so the voice's span widens through the clip's own map and+ // its retime is rebuilt over the wider span.+ const rt = retimeOf(sp.clip);+ const wIn = rt.srcAt(-sp.soundBack);+ const wOut = rt.srcAt(rt.len + sp.soundAhead);return {url: sp.asset.url,start: sp.start - sp.soundBack,- in: sp.clip.in - sp.soundBack * speed,- out: sp.clip.out + sp.soundAhead * speed,- speed,+ in: wIn,+ out: wOut,+ retime: retimeOf({ in: wIn, out: wOut, speed: sp.clip.speed, speedCurve: sp.clip.speedCurve }),sound: sp.clip.sound,};};@@ -376,12 +381,12 @@ class Engine {// alpha read below is a per-frame pixel pass; capping the matte's decode// keeps that pass off the frame budget.const src = this.pool.get(- mappingKey(asset.id, clipSpeed(clip), m.in, clip.start),+ mappingKey(asset.id, retimeOf(clip), m.in, clip.start),asset,Math.min(this.decodeHeight(), 480));this.used.add(src);- src.want(mt, this.renderPlaying);+ src.want(mt, this.renderPlaying, false, retimeOf(clip).rateAt(at - clip.start));const frame = src.frameAt(mt, 0, dur);if (!frame) return null;const held = this.matteAlpha.get(clip.id);@@ -423,7 +428,7 @@ class Engine {const src = this.pool.get(keyOf(span.clip, span.asset), span.asset, this.decodeHeight());this.used.add(src);const st = sourceTimeOf(span.clip, t);- src.want(st, playing, this.readBackward);+ src.want(st, playing, this.readBackward, retimeOf(span.clip).rateAt(t - span.start));// A failed source that already decoded frames keeps showing the nearest// one it holds — a transient blip (a network drop, a signed URL mid// re-mint) reads as a held frame, and only a source with nothing at all@@ -581,8 +586,8 @@ class Engine {for (const a of s.audioClips) {const asset = s.assets.find((x) => x.id === a.assetId);if (!asset || a.hidden || assetIsSilent(asset)) continue;- const speed = a.speed && a.speed > 0 ? a.speed : 1;- const len = Math.max(0.1, (a.out - a.in) / speed);+ const rt = retimeOf(a);+ const len = Math.max(0.1, rt.len);if (t < a.start || t >= a.start + len) continue;// Fade envelope: linear ramps at either end of the clip.const rel = t - a.start;@@ -599,7 +604,7 @@ class Engine {start: a.start,in: a.in,out: a.out,- speed,+ retime: rt,gain: a.volume * g * dg,sound: a.sound,});diff --git a/site/src/cut/lib/assetRef.ts b/site/src/cut/lib/assetRef.tsindex af77837d..44445396 100644--- a/site/src/cut/lib/assetRef.ts+++ b/site/src/cut/lib/assetRef.ts@@ -818,7 +818,7 @@ export function selectionRefTokens(s: EntitySources & {audioClips: AudioClip[];selection: Selection;multiSelection: Selection[];- selectedKey: { kind: "overlay" | "clip"; id: string; t: number; track: "pose" | "mask" } | null;+ selectedKey: { kind: "overlay" | "clip"; id: string; t: number; track: "pose" | "mask" | "speed" } | null;}): string | null {const entities = entityRefs(s);diff --git a/site/src/cut/lib/audioMix.ts b/site/src/cut/lib/audioMix.tsindex 21659b34..95d68a5e 100644--- a/site/src/cut/lib/audioMix.ts+++ b/site/src/cut/lib/audioMix.ts@@ -21,8 +21,10 @@ import {AUDIO_FX_RAMP,audioFxRecipe,buildAudioFx,+ retimeOf,soundRecipe,type ClipSound,+ type SpeedNode,} from "@donkeycut/effects-kit";import { decodeAudioSpan } from "./mediaRead";import { timeStretch } from "./timeStretch";@@ -35,6 +37,7 @@ export interface MixClip {out: number;muted: boolean;speed?: number;+ speedCurve?: SpeedNode[];/** Gain on the clip's own audio; absent = 1. */volume?: number;/** The clip's own treatment, run on its sound before the fades. */@@ -62,6 +65,7 @@ export interface MixItem {start: number;volume: number;speed?: number;+ speedCurve?: SpeedNode[];/** Half the cross dissolve at this item's head and tail, with the handle* each side reaches into — an upper-track cut crosses the same way track* 0's does. */@@ -105,15 +109,14 @@ export interface MixOptions {resolve: (file: string) => string;}-const speedOf = (s?: number) => (s && s > 0 ? s : 1);--/** Timeline length of one clip: source span over speed. A gap spacer keeps its- * exact length — flooring it would land everything after it late. */+/** Timeline length of one clip: its source span through its rate. A gap+ * spacer keeps its exact length — flooring it would land everything after it+ * late. */const clipDur = (c: MixClip) =>- c.file ? Math.max(0.1, (c.out - c.in) / speedOf(c.speed)) : Math.max(0, c.out - c.in);+ c.file ? Math.max(0.1, retimeOf(c).len) : Math.max(0, c.out - c.in);/** Timeline footprint of a placed item. */-const itemDur = (a: MixItem) => Math.max(0, (a.out - a.in) / speedOf(a.speed));+const itemDur = (a: MixItem) => Math.max(0, retimeOf(a).len);/*** Where the sequential fold puts each track-0 clip, and how long its@@ -157,20 +160,20 @@ export function foldClips(clips: MixClip[]) {* by the ramp that crossing actually got. */const handles = (g: { clip: MixClip; crossIn: number; crossOut: number }) => ({// Never past the head of the source: a handle is what the trim left behind.- back: Math.min(g.clip.soundBack ?? 0, g.crossIn, g.clip.in / speedOf(g.clip.speed)),+ back: Math.min(g.clip.soundBack ?? 0, g.crossIn, Math.max(0, -retimeOf(g.clip).tAt(0))),ahead: Math.min(g.clip.soundAhead ?? 0, g.crossOut),});/** The source span an entry decodes: its own trim, widened by the handle each- * crossing at its edges reaches into. Timeline seconds scale by speed on the- * way into the source. */+ * crossing at its edges reaches into. Timeline seconds go through the entry's+ * own map on the way into the source. */const crossSpan = (- c: { file: string; in: number; out: number; speed?: number },+ c: { file: string; in: number; out: number; speed?: number; speedCurve?: SpeedNode[] },back = 0,ahead = 0) => {- const speed = speedOf(c.speed);- return { file: c.file, in: c.in - back * speed, out: c.out + ahead * speed };+ const rt = retimeOf(c);+ return { file: c.file, in: rt.srcAt(-back), out: rt.srcAt(rt.len + ahead) };};/**@@ -364,16 +367,33 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB* length it needs first, and then played at its own rate.*/const stretched = new Map<string, AudioBuffer>();- const fitted = (key: string, buf: AudioBuffer, speed: number): AudioBuffer => {- if (speed === 1) return buf;- // A span used by more than one clip at the same speed is stretched once.- const memo = `${key}@${speed}`;+ const fitted = (+ key: string,+ buf: AudioBuffer,+ entry: { in: number; out: number; speed?: number; speedCurve?: SpeedNode[] },+ from: number+ ): AudioBuffer => {+ const rt = retimeOf(entry);+ if (rt.uniform && rt.rate === 1) return buf;+ // A span used by more than one clip at the same rate is stretched once.+ const memo = `${key}@${rt.key}`;const hit = stretched.get(memo);if (hit) return hit;const channels = Array.from({ length: buf.numberOfChannels }, (_, c) =>buf.getChannelData(c));- const fit = timeStretch(channels, buf.sampleRate, 1 / speed);+ // `from` is the source second the buffer starts at; the schedule follows+ // the clip's own map from there and lands exactly on its length.+ const fit = timeStretch(+ channels,+ buf.sampleRate,+ rt.uniform+ ? 1 / rt.rate+ : {+ factorAt: (sec) => 1 / rt.rateAtSrc(from + sec),+ outLength: Math.round((rt.tAt(from + buf.duration) - rt.tAt(from)) * buf.sampleRate),+ }+ );const out = new AudioBuffer({length: Math.max(1, fit[0].length),numberOfChannels: buf.numberOfChannels,@@ -420,7 +440,7 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioBif (!buf) continue; // muted, silent, or a gap spacer: only shapes timeconst from = g.at - g.back;const end = g.at + g.dur;- play(fitted(key, buf, speedOf(c.speed)), from, g.dur + g.back + g.ahead, ducked, c.sound, (gain) => {+ play(fitted(key, buf, c, Math.max(0, g.span.in)), from, g.dur + g.back + g.ahead, ducked, c.sound, (gain) => {const level = Math.max(0, c.volume ?? 1);gain.gain.value = level;// A crossing and a fade never share an edge — the cut carries one@@ -451,7 +471,7 @@ export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioB// A voiceover that ducks everything else is not itself ducked; it still// lands on the bus, so the effect elements treat it with the rest.const into = a.duck !== undefined && a.duck < 1 ? bus : ducked;- play(fitted(key, buf, speedOf(a.speed)), a.start - back, dur + back + ahead, into, a.sound, (gain) => {+ play(fitted(key, buf, a, Math.max(0, crossSpan(a, back, ahead).in)), a.start - back, dur + back + ahead, into, a.sound, (gain) => {const level = Math.max(0, a.volume);gain.gain.value = level;const fIn = Math.min(a.fadeIn ?? 0, dur);diff --git a/site/src/cut/lib/cloudTranscribe.ts b/site/src/cut/lib/cloudTranscribe.tsindex 033eaed8..39570a34 100644--- a/site/src/cut/lib/cloudTranscribe.ts+++ b/site/src/cut/lib/cloudTranscribe.ts@@ -25,10 +25,11 @@ export interface CloudTranscribeSpec {out: number;muted: boolean;speed?: number;+ speedCurve?: [number, number][];/** Cross-dissolve overlap into the next clip, timeline seconds. */transition?: number;}[];- audio: { file: string; in: number; out: number; start: number; volume: number; speed?: number }[];+ audio: { file: string; in: number; out: number; start: number; volume: number; speed?: number; speedCurve?: [number, number][] }[];}const RATE = 16000; // the wire format the hosted route expectsdiff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.tsindex aa441529..aafc4896 100644--- a/site/src/cut/lib/exportClient.ts+++ b/site/src/cut/lib/exportClient.ts@@ -16,7 +16,7 @@ import { renderRemovalPieces } from "./removalVideo";import { createRasterCanvas, rasterCanvasToPng } from "./raster";import { clipSpeed, getClipSpans, overlayLayers, projectDuration, spanSequence, useEditor } from "./store";import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";-import { isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, paintMaskLuma, paintStrokeInk } from "@donkeycut/effects-kit";+import { isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, paintMaskLuma, paintStrokeInk, retimeOf, type SpeedNode } from "@donkeycut/effects-kit";import { renderElementFrames, renderElementPng } from "./textRender";import { clipCovers, clipKeyed, clipPosed, clipPoseAt, clipZoom, contentRect, frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, regionPx, removalActive, shadowInk, subjectMasked } from "./types";import { liveReader } from "./liveReader";@@ -300,7 +300,7 @@ function removalShadowSilhouette(if (r!.backdrop && r!.backdrop.kind !== "none") return null;const asset = assets.find((a) => a.id === clip.assetId);if (!asset) return null;- const speed = clipSpeed(clip);+ const rt = retimeOf(clip);const still = asset.type === "image";const canvas = createRasterCanvas(2, 2);const ctx = canvas.getContext("2d", {@@ -349,7 +349,7 @@ function removalShadowSilhouette(const mdur = Math.max(0.1, matte.duration || 0.1);return {at: async (tLocal) => {- const srcT = still ? 0 : clip.in + tLocal * speed;+ const srcT = still ? 0 : rt.srcAt(tLocal);const frame = await reader.frameAt(Math.min(Math.max(0, srcT - r!.matte!.in), mdur - 0.001));@@ -601,6 +601,7 @@ export async function buildExportPayload(panY: sp.clip.panY ?? 0,frame: sp.clip.frame,speed: clipSpeed(sp.clip),+ speedCurve: sp.clip.speedCurve,transition: sp.transitionOut,// A cross dissolve carries its own window: the picture cuts, so the// server's blend must not see one. The handles are what the crossing@@ -633,7 +634,7 @@ export async function buildExportPayload(// opacity, since opacity ships as coverage luma.for (let i = 0; i < spans.length; i++) {const c = posed(spans[i].clip);- const dur = Math.max(0.1, (c.out - c.in) / clipSpeed(c));+ const dur = Math.max(0.1, retimeOf(c).len);const pictures = await renderClipMaskPictures(c,{ x: 0, y: 0, w: settings.width, h: settings.height },@@ -804,6 +805,7 @@ export async function buildExportPayload(volume: c.volume,sound: c.sound,speed: c.speed,+ speedCurve: c.speedCurve,image: assetById.get(c.assetId)!.type === "image",grade: normalizeGrade(c.grade),look: c.look,@@ -830,8 +832,7 @@ export async function buildExportPayload(const box = region? { x: region.rx, y: region.ry, w: region.rw, h: region.rh }: { x: 0, y: 0, w: settings.width, h: settings.height };- const ospeed = c.speed && c.speed > 0 ? c.speed : 1;- const olen = Math.max(0.1, (c.out - c.in) / ospeed);+ const olen = Math.max(0.1, retimeOf(c).len);const pictures = await renderClipMaskPictures(c,box,@@ -901,6 +902,7 @@ export async function buildExportPayload(fadeIn: a.fadeIn ?? 0,fadeOut: a.fadeOut ?? 0,speed: a.speed,+ speedCurve: a.speedCurve,sound: a.sound,duck: a.duck,}));@@ -1411,14 +1413,16 @@ const CARD_SECONDS = 5;* for the share card, which is a five-second window onto the opening rather* than a render of the whole project. */export function docFirstSeconds(doc: ExportDoc, seconds: number): ExportDoc {- // A clip's timeline footprint is (out - in) / speed, so the trim that ends- // it at `seconds` scales back through the speed.- const clampOut = <T extends { start: number; in: number; out: number; speed?: number }>(+ // The trim that ends a clip at `seconds` goes back through the clip's own+ // map, so a curved clip cuts at the source second really playing there.+ const clampOut = <+ T extends { start: number; in: number; out: number; speed?: number; speedCurve?: SpeedNode[] },+ >(clip: T): T => {- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;- const room = (seconds - clip.start) * speed;- return clip.out - clip.in <= room ? clip : { ...clip, out: clip.in + room };+ const rt = retimeOf(clip);+ const room = seconds - clip.start;+ return rt.len <= room ? clip : { ...clip, out: rt.srcAt(room) };};const starts = <T extends { start: number }>(item: T) => item.start < seconds;return {diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.tsindex 5cbc16c7..af510a2f 100644--- a/site/src/cut/lib/exportRender.ts+++ b/site/src/cut/lib/exportRender.ts@@ -35,7 +35,7 @@ import { frameSink, openMedia, videoTrackOf } from "./mediaRead";import { allowance, canvasBytes, holdMemory } from "./memoryBudget";import { getClipSpans, overlayLayers, projectDuration, spanSequence } from "./store";import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";-import { applyEffectToCanvas, evalOverlayFrame, grainTile, isAudioEffect, isMaskAnimated, isOverlayAnimated, maskFrameAt, MATTE_FPS, matteLumaToAlpha, planAnimatedLayers, type LottieHandle, type OverlayAnim, type PaintPhase } from "@donkeycut/effects-kit";+import { applyEffectToCanvas, evalOverlayFrame, retimeOf, grainTile, isAudioEffect, isMaskAnimated, isOverlayAnimated, maskFrameAt, MATTE_FPS, matteLumaToAlpha, planAnimatedLayers, type LottieHandle, type OverlayAnim, type PaintPhase } from "@donkeycut/effects-kit";import { backdropStill, loadBackdropStill } from "./backdropStills";import { hasSubjectOverlays, SubjectMaskCompositor } from "./behindPass";import { createRasterCanvas, type RasterSurface } from "./raster";@@ -141,8 +141,7 @@ const READER_GRACE = 2;/** A clip's source time at timeline time `t`. */function sourceTimeAt(span: ClipSpan, t: number): number {- const speed = span.clip.speed && span.clip.speed > 0 ? span.clip.speed : 1;- return span.clip.in + Math.max(0, t - span.start) * speed;+ return retimeOf(span.clip).srcAt(Math.max(0, t - span.start));}/**@@ -536,6 +535,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strinstart: sp.start,volume: sp.clip.volume ?? 1,speed: sp.clip.speed,+ speedCurve: sp.clip.speedCurve,sound: sp.clip.sound,muted: sp.clip.muted || assetIsSilent(sp.asset),fadeIn: ramps[i].head,@@ -557,6 +557,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strinstart: a.start,volume: a.volume,speed: a.speed,+ speedCurve: a.speedCurve,fadeIn: a.fadeIn,fadeOut: a.fadeOut,sound: a.sound,@@ -576,6 +577,7 @@ export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => strinout: sp.clip.out,muted: sp.clip.muted || !!sp.clip.hidden || assetIsSilent(sp.asset),speed: sp.clip.speed,+ speedCurve: sp.clip.speedCurve,volume: sp.clip.volume,sound: sp.clip.sound,transition: sp.transitionOut,diff --git a/site/src/cut/lib/filmstrip.test.ts b/site/src/cut/lib/filmstrip.test.tsindex 4b8611c6..241a32eb 100644--- a/site/src/cut/lib/filmstrip.test.ts+++ b/site/src/cut/lib/filmstrip.test.ts@@ -1,4 +1,5 @@import { describe, expect, test } from "bun:test";+import { retimeOf } from "@donkeycut/effects-kit";import {FILM_TILE_CAP,MIN_SUB_TILE_PX,@@ -296,3 +297,57 @@ describe("pickThumbTimes", () => {for (let i = 1; i < times.length; i++) expect(times[i]).toBeGreaterThan(times[i - 1]);});});++describe("a curved clip's strips agree", () => {+ // The speed curve editor draws the same clip as the timeline bar, at its own+ // width. Both plan through the clip's map, so a point the same fraction+ // along each of them pictures the same moment of the footage — which is what+ // makes a node in the graph sit over the frame it retimes.+ const clip = { in: 0, out: DURATION, speedCurve: [[0, 2], [DURATION, 0.25]] as [number, number][] };+ const retime = retimeOf(clip);+ const stripFor = (w: number) =>+ planFilmstrip({+ thumbs,+ thumbStep: THUMB_STEP,+ duration: DURATION,+ aspect: 16 / 9,+ filmIn: clip.in,+ w,+ pps: w / retime.len,+ speed: retime.rate,+ retime,+ tileH: 60,+ minTileW: 26,+ });++ test("the graph's picture is the bar's picture", () => {+ const bar = stripFor(1400);+ const graph = stripFor(700);+ expect(bar.length).toBeGreaterThan(2);+ expect(graph.length).toBeGreaterThan(2);+ const at = (tiles: typeof bar, w: number, f: number) => {+ const x = f * w;+ return tiles.find((t) => x >= t.left && x < t.left + t.width) ?? tiles[tiles.length - 1];+ };+ for (const f of [0.1, 0.3, 0.5, 0.7, 0.9]) {+ const a = at(bar, 1400, f);+ const b = at(graph, 700, f);+ // Within a tile of each other: the two widths cut the same map into+ // different numbers of tiles, so the moments agree, not the boundaries.+ const tileSpan = retime.srcAt(retime.len * (b.width / 700 + f)) - retime.srcAt(retime.len * f);+ expect(Math.abs(a.srcT - b.srcT)).toBeLessThanOrEqual(Math.abs(tileSpan) + 0.5);+ }+ });++ test("both walk the footage forward across the whole box", () => {+ for (const w of [700, 1400]) {+ const tiles = stripFor(w);+ for (let i = 1; i < tiles.length; i++) expect(tiles[i].srcT).toBeGreaterThan(tiles[i - 1].srcT);+ // The clip opens at 2x, so its first tile already covers seconds of+ // footage; what matters is that the strip starts inside the opening+ // stretch and ends on the tail.+ expect(tiles[0].srcT).toBeLessThan(retime.srcAt(retime.len * 0.25));+ expect(tiles[tiles.length - 1].srcT).toBeGreaterThan(DURATION - 2);+ }+ });+});diff --git a/site/src/cut/lib/filmstrip.ts b/site/src/cut/lib/filmstrip.tsindex eec8c2fa..137467f7 100644--- a/site/src/cut/lib/filmstrip.ts+++ b/site/src/cut/lib/filmstrip.ts@@ -36,6 +36,8 @@* grid: a scroll keeps a tile's capture where it was, a zoom re-aims it.*/+import type { Retime } from "@donkeycut/effects-kit";+export type FilmTile = {/** The tile's place on the source grid at this zoom, stable across* scrolls and trims; a zoom re-aims the same id at a new moment. */@@ -109,6 +111,42 @@ export function pickThumbTimes(});}+/** Tiles on the timeline grid for a clip whose rate changes: `imgW` pixels+ * each, the last one ending at the clip's edge, each showing the source+ * second playing at its middle. */+function planCurvedStrip(+ p: Omit<Parameters<typeof planFilmstrip>[0], "retime"> & { retime: Retime },+ imgW: number+): FilmTile[] {+ const rt = p.retime;+ const tiles: FilmTile[] = [];+ const n = Math.max(1, Math.ceil(p.w / imgW));+ const first = p.view ? Math.max(0, Math.floor(p.view.lo / imgW)) : 0;+ const last = p.view ? Math.min(n - 1, Math.ceil(p.view.hi / imgW) - 1) : n - 1;+ for (let i = first; i <= last; i++) {+ const left = i * imgW;+ const width = Math.min(imgW, p.w - left);+ if (width <= 0) break;+ const tMid = (left + width / 2) / p.pps;+ const mid = Math.max(0, Math.min(p.duration, rt.srcAt(tMid)));+ const a = Math.max(0, rt.srcAt(left / p.pps));+ const b = Math.min(p.duration, rt.srcAt((left + width) / p.pps));+ const idx = Math.min(p.thumbs.length - 1, Math.max(0, Math.floor(mid / p.thumbStep)));+ let src = p.thumbs[idx];+ let exact = false;+ const snapped = (Math.floor(mid / WANT_GRID_S) + 0.5) * WANT_GRID_S;+ let wantT = snapped > a && snapped < b ? snapped : mid;+ wantT = Math.max(0, Math.min(p.duration - 0.05, wantT));+ const hit = p.exactFrame?.(wantT) ?? null;+ if (hit) {+ src = hit;+ exact = true;+ }+ tiles.push({ id: String(i), src, left, width, srcT: mid, wantT, exact });+ }+ return tiles;+}+export function planFilmstrip(p: {thumbs: string[];thumbStep: number;@@ -121,7 +159,13 @@ export function planFilmstrip(p: {/** Drawn width of the clip box, px. */w: number;pps: number;+ /** Source seconds per timeline second across the strip (the clip's rate,+ * or its average rate under a curve). */speed: number;+ /** The clip's map, when its rate changes through the footage. The strip+ * then lays tiles on the timeline grid — a held moment takes the width it+ * plays for — with each tile picturing the source second at its middle. */+ retime?: Retime;tileH: number;minTileW: number;/** Scene changes in the source, seconds, ascending. While refining they@@ -140,6 +184,7 @@ export function planFilmstrip(p: {const span = p.view ? Math.min(p.w, p.view.hi - p.view.lo) : p.w;const cells = Math.max(1, Math.ceil(span / natural));const imgW = natural * Math.ceil(cells / FILM_TILE_CAP);+ if (p.retime && !p.retime.uniform) return planCurvedStrip({ ...p, retime: p.retime }, imgW);const stepT = (imgW / p.pps) * p.speed;const filmOut = p.filmIn + (p.w / p.pps) * p.speed;// The window, in source time, clamped to the clip.diff --git a/site/src/cut/lib/framePlan.ts b/site/src/cut/lib/framePlan.tsindex 3dc8d171..77a231ac 100644--- a/site/src/cut/lib/framePlan.ts+++ b/site/src/cut/lib/framePlan.ts@@ -1,5 +1,7 @@"use client";+import { retimeOf } from "@donkeycut/effects-kit";+/*** What the cut looks like at one instant, worked out from the document alone.*@@ -194,8 +196,7 @@ export function duckGainAt(audioClips: AudioClip[], t: number): number {let g = 1;for (const a of audioClips) {if (a.hidden || a.duck === undefined || a.duck >= 1) continue;- const speed = a.speed && a.speed > 0 ? a.speed : 1;- const len = Math.max(0.1, (a.out - a.in) / speed);+ const len = Math.max(0.1, retimeOf(a).len);if (t >= a.start && t < a.start + len) g = Math.min(g, Math.max(0, a.duck));}return g;diff --git a/site/src/cut/lib/frameSource.test.ts b/site/src/cut/lib/frameSource.test.tsindex e0e264ad..4ebb5c8c 100644--- a/site/src/cut/lib/frameSource.test.ts+++ b/site/src/cut/lib/frameSource.test.ts@@ -1,5 +1,6 @@import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";-import type { Timed } from "./frameSource";+import { retimeOf } from "@donkeycut/effects-kit";+import { DECODE_AHEAD_S, type Timed } from "./frameSource";import type { MediaAsset } from "./types";// ── the files a source reads, stood in for ──────────────────────────────────@@ -238,26 +239,36 @@ describe("FrameRing", () => {});describe("mappingKey", () => {+ const uniform = (speed: number, inPoint: number, out = 100) =>+ retimeOf({ in: inPoint, out, speed });test("clips showing identical pictures share a decoder", () => {// A plain split: same file, same speed, and the same source time at// timeline zero. One decoder reads straight across the cut.- const left = mappingKey("a", 1, 0, 0);- const right = mappingKey("a", 1, 4, 4);+ const left = mappingKey("a", uniform(1, 0), 0, 0);+ const right = mappingKey("a", uniform(1, 4), 4, 4);expect(left).toBe(right);});test("different trims of one file keep their own decoders", () => {// Both live at timeline 4, showing different moments — a same-source// dissolve has to blend two distinct frames.- expect(mappingKey("a", 1, 0, 4)).not.toBe(mappingKey("a", 1, 8, 4));+ expect(mappingKey("a", uniform(1, 0), 0, 4)).not.toBe(mappingKey("a", uniform(1, 8), 8, 4));});test("a speed change is a different mapping", () => {- expect(mappingKey("a", 1, 0, 0)).not.toBe(mappingKey("a", 2, 0, 0));+ expect(mappingKey("a", uniform(1, 0), 0, 0)).not.toBe(mappingKey("a", uniform(2, 0), 0, 0));+ });++ test("a speed curve is its own mapping, keyed on the curve and the start", () => {+ const curved = (start: number) =>+ mappingKey("a", retimeOf({ in: 0, out: 10, speedCurve: [[0, 1], [10, 4]] }), 0, start);+ expect(curved(0)).toBe(curved(0));+ expect(curved(0)).not.toBe(curved(2));+ expect(curved(0)).not.toBe(mappingKey("a", uniform(1, 0), 0, 0));});test("different files never share", () => {- expect(mappingKey("a", 1, 0, 0)).not.toBe(mappingKey("b", 1, 0, 0));+ expect(mappingKey("a", uniform(1, 0), 0, 0)).not.toBe(mappingKey("b", uniform(1, 0), 0, 0));});});@@ -265,7 +276,7 @@ describe("walkClaim", () => {/** A walk that has landed frames from `from` to `tail`, asked for `t`. */const landed = (t: number, from: number, tail: number, covered = false, heldBefore = true) =>walkClaim({- t, walkFor: from, from, tail, landed: true, covered, heldBefore, comingS: 5, playing: true,+ t, ahead: DECODE_AHEAD_S, walkFor: from, from, tail, landed: true, covered, heldBefore, comingS: 5, playing: true,});test("playback inside the walk keeps it", () => {@@ -309,7 +320,7 @@ describe("walkClaim", () => {// reader is behind the anchor by construction, and that is not the walk// going past it — it is the walk about to be right.const lead = {- t: 4, walkFor: 4, from: 4.5, tail: 4.6, landed: true, covered: false, heldBefore: true, playing: true,+ t: 4, ahead: DECODE_AHEAD_S, walkFor: 4, from: 4.5, tail: 4.6, landed: true, covered: false, heldBefore: true, playing: true,};expect(walkClaim({ ...lead, comingS: 5 })).toBe("hold");expect(walkClaim({ ...lead, t: 4.4, comingS: 5 })).toBe("hold");diff --git a/site/src/cut/lib/frameSource.ts b/site/src/cut/lib/frameSource.tsindex 14e499de..e44e15f4 100644--- a/site/src/cut/lib/frameSource.ts+++ b/site/src/cut/lib/frameSource.ts@@ -1,5 +1,7 @@"use client";+import type { Retime } from "@donkeycut/effects-kit";+/*** Frames for the preview, decoded straight off the file.*@@ -148,6 +150,9 @@ export type WalkClaim = "hold" | "hop" | "restart";export function walkClaim(w: {/** The moment being asked for. */t: number;+ /** Source seconds the walk is kept ahead of the reader, for the clip's+ * rate; the 1× lookahead when absent. */+ ahead?: number;/** Where the reader stood when this walk was sent — at or behind `from`. */walkFor: number;/** Where the walk was anchored. */@@ -183,7 +188,7 @@ export function walkClaim(w: {// the lookahead's slack so a walk merely holding its lead is left alone.const behind =w.playing &&- (w.t < w.walkFor - SAME || (!w.heldBefore && w.t < w.tail - DECODE_AHEAD_S));+ (w.t < w.walkFor - SAME || (!w.heldBefore && w.t < w.tail - (w.ahead ?? DECODE_AHEAD_S)));if (!w.landed) {// Nothing landed means the walk is still doing the only work that can// produce a picture — finding the keyframe before its anchor, fetching the@@ -532,6 +537,8 @@ export class ClipFrameSource {* has since left. */private aim = 0;private closed = false;+ /** Source seconds the walk keeps ahead of the reader, for the clip's rate. */+ private ahead = DECODE_AHEAD_S;/** Set when the file turns out to hold no picture this browser can read. */private unreadable = false;/** Failed opens so far, and the timer that will clear `unreadable` for the@@ -849,8 +856,12 @@ export class ClipFrameSource {* pointer backing into a clip arrives at its last frame, which is above* everything that source was ever asked for.*/- want(t: number, playing: boolean, backward = false): void {+ want(t: number, playing: boolean, backward = false, rate = 1): void {if (this.closed || this.unreadable) return;+ // The lookahead is source seconds tuned for a clip playing at 1×. A clip+ // running faster eats the ring that much sooner, so the walk stays the+ // same timeline distance ahead; the ring's own bound still caps it.+ this.ahead = DECODE_AHEAD_S * Math.max(1, rate);// Being read is what wakes a stood-down source; the walk below is what// gives it a decoder again.this.asleep = false;@@ -1169,6 +1180,7 @@ export class ClipFrameSource {if (this.starting || this.stream) {const claim = walkClaim({t,+ ahead: this.ahead,walkFor: this.walkFor,from: this.streamFrom,tail: this.streamTail,@@ -1486,7 +1498,7 @@ export class ClipFrameSource {// or past `t` fill the ring — pushing another would drop one that is// still wanted. Only the walk's frames count: leftovers from other// gestures merely age out of the cache as the walk pushes.- if (this.streamTail >= t + DECODE_AHEAD_S) break;+ if (this.streamTail >= t + this.ahead) break;if (this.ring.between(Math.max(t, this.streamFrom), this.streamTail) >= RING - 1) break;const pullAt = performance.now();this.nextStartedAt = pullAt;@@ -1879,14 +1891,19 @@ export class FrameSourcePool {/*** The decode identity of a clip: the frames it shows are a function of its- * file, its speed, and where its source time stands at timeline zero. Two clips- * agreeing on all three show the same picture at every instant.+ * file, its rate, and where its source time stands at timeline zero. Two clips+ * agreeing on all three show the same picture at every instant. A clip whose+ * rate changes through the footage is its own map: its curve and its start.*/export function mappingKey(assetId: string,- speed: number,+ retime: Retime,inPoint: number,start: number): string {- return `${assetId}|${speed}|${(inPoint - start * speed).toFixed(3)}`;+ if (retime.uniform) {+ const speed = retime.rate;+ return `${assetId}|${speed}|${(inPoint - start * speed).toFixed(3)}`;+ }+ return `${assetId}|${retime.key}|${inPoint.toFixed(3)}|${start.toFixed(3)}`;}diff --git a/site/src/cut/lib/genvideo/docWriter.ts b/site/src/cut/lib/genvideo/docWriter.tsindex 03492791..0465e0e7 100644--- a/site/src/cut/lib/genvideo/docWriter.ts+++ b/site/src/cut/lib/genvideo/docWriter.ts@@ -1,5 +1,7 @@"use client";+import { retimeOf } from "@donkeycut/effects-kit";+/*** Background writes to a project that is NOT open in the editor. A scene run* outlives the open project: when the user switches away, its placements,@@ -175,10 +177,7 @@ export function docPlaceGenClip(: 0;const { out, speed } = fillSlot(asset.type, Math.max(MIN_LEN, asset.duration - srcIn), slot, SPEED_MIN);const row = doc.clips.filter((c) => c.track === 0);- const spans = row.map((c) => {- const sp = c.speed && c.speed > 0 ? c.speed : 1;- return { id: c.id, start: c.start, end: c.start + (c.out - c.in) / sp };- });+ const spans = row.map((c) => ({ id: c.id, start: c.start, end: c.start + retimeOf(c).len }));const anchorIds = new Set(opts?.anchorAfterIds ?? []);const prevEnd = spans.filter((s) => anchorIds.has(s.id)).reduce((m, s) => Math.max(m, s.end), -1);const len = speed !== undefined && speed > 0 ? out / speed : out;diff --git a/site/src/cut/lib/genvideo/editorBridge.ts b/site/src/cut/lib/genvideo/editorBridge.tsindex d3b5cd6b..fba0c590 100644--- a/site/src/cut/lib/genvideo/editorBridge.ts+++ b/site/src/cut/lib/genvideo/editorBridge.ts@@ -1,5 +1,7 @@"use client";+import { retimeOf, type SpeedNode } from "@donkeycut/effects-kit";+/*** The real EditorBridge: the orchestrator's typed hands on the live Cut* timeline. It is the single frames↔seconds conversion boundary — the plan@@ -35,10 +37,9 @@ export const GEN_FPS = 30;const toSec = (frames: number) => frames / GEN_FPS;-/** Effective timeline footprint of a clip (seconds), honoring speed. */-function footprintSec(c: { in: number; out: number; speed?: number }): number {- const src = c.out - c.in;- return c.speed && c.speed > 0 ? src / c.speed : src;+/** Effective timeline footprint of a clip (seconds), through its rate. */+function footprintSec(c: { in: number; out: number; speed?: number; speedCurve?: SpeedNode[] }): number {+ return retimeOf(c).len;}export class StoreEditorBridge implements EditorBridge {@@ -131,16 +132,14 @@ export class StoreEditorBridge implements EditorBridge {if ((await this.mode()) === "store") {const clip = useEditor.getState().clips.find((c) => c.id === clipId);if (!clip) return;- // Trim the tail so the footprint matches, at whatever speed the clip carries.- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;- useEditor.getState().setClipTrim(clipId, clip.in, clip.in + toSec(durationFrames) * speed);+ // Trim the tail so the footprint matches, at whatever rate the clip carries.+ useEditor.getState().setClipTrim(clipId, clip.in, retimeOf(clip).srcAt(toSec(durationFrames)));return;}await withProjectDoc(this.projectId, (doc) => {doc.clips = doc.clips.map((c) => {if (c.id !== clipId) return c;- const speed = c.speed && c.speed > 0 ? c.speed : 1;- return { ...c, out: c.in + toSec(durationFrames) * speed };+ return { ...c, out: retimeOf(c).srcAt(toSec(durationFrames)) };});});}diff --git a/site/src/cut/lib/laneTracks.ts b/site/src/cut/lib/laneTracks.tsindex f0135fc1..c5e38c27 100644--- a/site/src/cut/lib/laneTracks.ts+++ b/site/src/cut/lib/laneTracks.ts@@ -1,5 +1,7 @@"use client";+import { retimeOf } from "@donkeycut/effects-kit";+/*** The lane-track coordinator: the one place for how items on the timeline's* free-positioned tracks behave. Audio, titles, upper video layers, and@@ -121,8 +123,6 @@ interface LaneAdapter<T> {afterMove?(raw: T, delta: number): void;}-const speedOf = (c: { speed?: number }) => (c.speed && c.speed > 0 ? c.speed : 1);-/** The asset's beats inside a media item's trimmed range, in timeline* seconds. Beats live on the asset in source seconds, so every clip showing* the source maps its own window of them. */@@ -132,9 +132,9 @@ function mediaBeatTimes(): number[] {const beats = s.assets.find((a) => a.id === c.assetId)?.beats?.beats;if (!beats?.length) return [];- const speed = speedOf(c);+ const rt = retimeOf(c);const times: number[] = [];- for (const b of beats) if (b >= c.in && b <= c.out) times.push(c.start + (b - c.in) / speed);+ for (const b of beats) if (b >= c.in && b <= c.out) times.push(c.start + rt.tAt(b));return times;}@@ -142,7 +142,7 @@ function videoMaxLen(s: S, c: VideoClip): number {const a = s.assets.find((x) => x.id === c.assetId);// A still has no source length, so its clip can stretch to any duration.if (a?.type === "image") return Infinity;- return ((a?.duration ?? c.out) - c.in) / speedOf(c);+ return retimeOf({ ...c, out: a?.duration ?? c.out }).len;}const clipAdapter: LaneAdapter<VideoClip> = {@@ -156,17 +156,17 @@ const clipAdapter: LaneAdapter<VideoClip> = {movePatch: (c, start) => ({ id: c.id, patch: { start } }),trimLeftPatch: (c, newStart) => ({id: c.id,- patch: { start: newStart, in: c.in + (newStart - c.start) * speedOf(c) },+ patch: { start: newStart, in: retimeOf(c).srcAt(newStart - c.start) },}),trimRightPatch: (c, newEnd) => ({id: c.id,- patch: { out: c.in + (newEnd - c.start) * speedOf(c) },+ patch: { out: retimeOf(c).srcAt(newEnd - c.start) },}),revealLeftPatch: (c, start, reveal) => ({id: c.id,- patch: { start, in: c.in + (reveal - c.start) * speedOf(c) },+ patch: { start, in: retimeOf(c).srcAt(reveal - c.start) },}),- leftFloor: (c) => Math.max(0, c.start - c.in / speedOf(c)),+ leftFloor: (c) => Math.max(0, c.start + retimeOf(c).tAt(0)),maxLen: videoMaxLen,closesGap: true,assetOf: (s, c) => s.assets.find((x) => x.id === c.assetId),@@ -183,19 +183,19 @@ const audioAdapter: LaneAdapter<AudioClip> = {movePatch: (a, start) => ({ id: a.id, patch: { start } }),trimLeftPatch: (a, newStart) => ({id: a.id,- patch: { start: newStart, in: a.in + (newStart - a.start) * speedOf(a) },+ patch: { start: newStart, in: retimeOf(a).srcAt(newStart - a.start) },}),trimRightPatch: (a, newEnd) => ({id: a.id,- patch: { out: a.in + (newEnd - a.start) * speedOf(a) },+ patch: { out: retimeOf(a).srcAt(newEnd - a.start) },}),revealLeftPatch: (a, start, reveal) => ({id: a.id,- patch: { start, in: a.in + (reveal - a.start) * speedOf(a) },+ patch: { start, in: retimeOf(a).srcAt(reveal - a.start) },}),- leftFloor: (a) => Math.max(0, a.start - a.in / speedOf(a)),+ leftFloor: (a) => Math.max(0, a.start + retimeOf(a).tAt(0)),maxLen: (s, a) =>- ((s.assets.find((x) => x.id === a.assetId)?.duration ?? a.out) - a.in) / speedOf(a),+ retimeOf({ ...a, out: s.assets.find((x) => x.id === a.assetId)?.duration ?? a.out }).len,lanePatch: (a, lane) => ({ id: a.id, patch: { lane: lane > 0 ? lane : undefined } }),assetOf: (s, a) => s.assets.find((x) => x.id === a.assetId),beatTimes: mediaBeatTimes,@@ -227,17 +227,17 @@ const overlayClipAdapter: LaneAdapter<VideoClip> = {movePatch: (c, start) => ({ id: c.id, patch: { start } }),trimLeftPatch: (c, newStart) => ({id: c.id,- patch: { start: newStart, in: c.in + (newStart - c.start) * speedOf(c) },+ patch: { start: newStart, in: retimeOf(c).srcAt(newStart - c.start) },}),trimRightPatch: (c, newEnd) => ({id: c.id,- patch: { out: c.in + (newEnd - c.start) * speedOf(c) },+ patch: { out: retimeOf(c).srcAt(newEnd - c.start) },}),revealLeftPatch: (c, start, reveal) => ({id: c.id,- patch: { start, in: c.in + (reveal - c.start) * speedOf(c) },+ patch: { start, in: retimeOf(c).srcAt(reveal - c.start) },}),- leftFloor: (c) => Math.max(0, c.start - c.in / speedOf(c)),+ leftFloor: (c) => Math.max(0, c.start + retimeOf(c).tAt(0)),maxLen: videoMaxLen,closesGap: true,assetOf: (s, c) => s.assets.find((x) => x.id === c.assetId),diff --git a/site/src/cut/lib/maskVideo.ts b/site/src/cut/lib/maskVideo.tsindex 631432f8..e236cf4d 100644--- a/site/src/cut/lib/maskVideo.ts+++ b/site/src/cut/lib/maskVideo.ts@@ -1,5 +1,7 @@"use client";+import { retimeOf } from "@donkeycut/effects-kit";+/*** The person-matte video for the ffmpeg export path. The page — the only* place that can run segmentation — composes the video layers over the@@ -37,10 +39,8 @@ export const MASK_FPS = 15;const MASK_SHORT = 480;/** Timeline second inside a span mapped to its source second. */-const sourceTimeAt = (sp: ClipSpan, t: number) => {- const speed = sp.clip.speed && sp.clip.speed > 0 ? sp.clip.speed : 1;- return Math.min(sp.clip.out, sp.clip.in + Math.max(0, t - sp.start) * speed);-};+const sourceTimeAt = (sp: ClipSpan, t: number) =>+ Math.min(sp.clip.out, retimeOf(sp.clip).srcAt(Math.max(0, t - sp.start)));/** A clip whose picture trims by the person matte (either direction). */const subjectClip = (c: { mask?: { kind: string } }) => c.mask?.kind === "subject";diff --git a/site/src/cut/lib/previewMixer.test.ts b/site/src/cut/lib/previewMixer.test.tsindex 04faa7ee..31f4599b 100644--- a/site/src/cut/lib/previewMixer.test.ts+++ b/site/src/cut/lib/previewMixer.test.ts@@ -1,4 +1,5 @@import { describe, expect, mock, test } from "bun:test";+import { retimeOf } from "@donkeycut/effects-kit";import { AudioBuffer } from "node-web-audio-api";/**@@ -254,7 +255,7 @@ const voice = {start: 0,in: 0,out: CLIP_S,- speed: 1,+ retime: retimeOf({ in: 0, out: CLIP_S, speed: 1 }),gain: 1,};diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.tsindex 306f2fc7..1cf4759f 100644--- a/site/src/cut/lib/previewMixer.ts+++ b/site/src/cut/lib/previewMixer.ts@@ -41,12 +41,13 @@ import {soundRecipe,type AudioFxNodes,type ClipSound,+ type Retime,} from "@donkeycut/effects-kit";import type { WrappedAudioBuffer } from "mediabunny";import { assembleAudio, decodeAudioSpan, openAudioWalk, type AudioWalk } from "./mediaRead";import { allowance, holdMemory } from "./memoryBudget";import { meterAudioLate, meterClockJump } from "./perfTrace";-import { timeStretch } from "./timeStretch";+import { timeStretch, type StretchSchedule } from "./timeStretch";/** How much of a voice is pulled off its walk and scheduled at once, in source* seconds. Long enough that a stretch of sound is one scheduling decision;@@ -111,7 +112,8 @@ export interface Voice {start: number;in: number;out: number;- speed: number;+ /** The map from this voice's timeline span onto its source span. */+ retime: Retime;/** Everything the frame plan says about how loud it is right now. */gain: number;/** The clip's own treatment, run on this voice alone before it joins the@@ -127,7 +129,7 @@ interface Scheduled {interface LiveVoice {url: string;- speed: number;+ retime: Retime;in: number;out: number;start: number;@@ -452,7 +454,7 @@ export class PreviewMixer {// read aims the walk and pulls the bytes it sits in — the primer a// parked playhead uses — so the release finds an open file already// read where the first window starts.- const source = live.in + (from - live.start) * live.speed;+ const source = live.retime.srcAt(from - live.start);if (!live.filling && source < live.out - 1e-3) void this.prime(live, source);}}@@ -580,7 +582,7 @@ export class PreviewMixer {(prev.start !== v.start ||prev.in !== v.in ||prev.out !== v.out ||- prev.speed !== v.speed)+ prev.retime.key !== v.retime.key)) {this.release(prev);this.voices.delete(v.id);@@ -660,7 +662,7 @@ export class PreviewMixer {prev.start !== v.start ||prev.in !== v.in ||prev.out !== v.out ||- prev.speed !== v.speed)+ prev.retime.key !== v.retime.key)) {this.release(prev);this.voices.delete(v.id);@@ -671,7 +673,7 @@ export class PreviewMixer {this.treat(live, v.sound);const from = Math.max(v.start, t);if (live.scheduled !== from) this.reaim(live, from);- const source = live.in + (from - live.start) * live.speed;+ const source = live.retime.srcAt(from - live.start);// Already reading, at the end of its source, or standing close enough// that the play's first read is the one it was going to make anyway.if (live.filling || source >= live.out - 1e-3) continue;@@ -746,7 +748,7 @@ export class PreviewMixer {head.connect(gain);const live: LiveVoice = {url: v.url,- speed: v.speed,+ retime: v.retime,in: v.in,out: v.out,start: v.start,@@ -843,7 +845,7 @@ export class PreviewMixer {const current = () => this.voices.get(id) === live && live.gen === gen;try {const from = live.scheduled;- const sourceFrom = live.in + (from - live.start) * live.speed;+ const sourceFrom = live.retime.srcAt(from - live.start);if (sourceFrom >= live.out - 1e-3) {live.ended = true;return;@@ -855,7 +857,7 @@ export class PreviewMixer {// waiting. The first window is short so the sound joins it now.const group = live.windows.length ? GROUP_S : FIRST_GROUP_S;buffer =- Math.abs(live.speed - 1) > 1e-3+ !live.retime.uniform || Math.abs(live.retime.rate - 1) > 1e-3? await this.stretchedWindow(live, sourceFrom): await this.walkGroup(live, sourceFrom, current, group);} catch {@@ -972,13 +974,25 @@ export class PreviewMixer {* everything older is a full decode away anyway.*/private async stretchedWindow(live: LiveVoice, sourceFrom: number): Promise<AudioBuffer | null> {- const sourceTo = Math.min(live.out, sourceFrom + WINDOW_STRETCHED_S * live.speed);- const key = `${live.url}|${sourceFrom.toFixed(3)}|${sourceTo.toFixed(3)}|${live.speed}`;+ const rt = live.retime;+ const tFrom = rt.tAt(sourceFrom);+ const sourceTo = Math.min(live.out, rt.srcAt(tFrom + WINDOW_STRETCHED_S));+ const key = `${live.url}|${sourceFrom.toFixed(3)}|${sourceTo.toFixed(3)}|${rt.key}`;const held = this.decoded.get(key);if (held) return held;const raw = await decodeAudioSpan(live.url, sourceFrom, sourceTo);if (!raw) return null;- const buffer = this.stretch(raw, 1 / live.speed);+ // A curve's window is laid by the clip's own map from where the window+ // starts, so its length is exactly the timeline the picture spends on it.+ const buffer = this.stretch(+ raw,+ rt.uniform+ ? 1 / rt.rate+ : {+ factorAt: (sec) => 1 / rt.rateAtSrc(sourceFrom + sec),+ outLength: Math.round((rt.tAt(sourceFrom + raw.duration) - tFrom) * raw.sampleRate),+ }+ );// Oldest first until the cache is back inside its share. Emptying the// whole cache would throw away the window the playhead is standing on along// with the ones it has finished, and the next frame would pay a full decode@@ -1011,7 +1025,7 @@ export class PreviewMixer {live.filling = false;live.ended = false;live.scheduled = timeline;- live.walk?.seek(live.in + (timeline - live.start) * live.speed);+ live.walk?.seek(live.retime.srcAt(timeline - live.start));}/** A read that failed, came back empty, or stopped answering. The voice@@ -1038,7 +1052,7 @@ export class PreviewMixer {}/** Re-lay a buffer at a different length, keeping its pitch. */- private stretch(buffer: AudioBuffer, factor: number): AudioBuffer {+ private stretch(buffer: AudioBuffer, factor: number | StretchSchedule): AudioBuffer {const channels: Float32Array[] = [];for (let c = 0; c < buffer.numberOfChannels; c++) channels.push(buffer.getChannelData(c));const out = timeStretch(channels, buffer.sampleRate, factor);diff --git a/site/src/cut/lib/removalVideo.ts b/site/src/cut/lib/removalVideo.tsindex 3ad32ce6..640335fa 100644--- a/site/src/cut/lib/removalVideo.ts+++ b/site/src/cut/lib/removalVideo.ts@@ -10,11 +10,10 @@* channel, which is why the layer travels as a pair.*/-import { matteLumaToAlpha, removalActive } from "@donkeycut/effects-kit";+import { matteLumaToAlpha, removalActive, retimeOf } from "@donkeycut/effects-kit";import { openCanvasVideo, scaledEvenSize } from "./canvasVideo";import { FrameCompositor, type Frame } from "./composite";import { createRasterCanvas, decodeRasterImageUrl, type RasterSurface } from "./raster";-import { clipSpeed } from "./store";import type { MediaAsset, VideoClip } from "./types";import { liveReader } from "./liveReader";@@ -72,9 +71,9 @@ export async function renderRemovalPieces(const matteAsset = r.matte ? assets.find((a) => a.id === r.matte!.assetId) : undefined;if (!matteAsset) return null;- const speed = clipSpeed(clip);+ const rt = retimeOf(clip);const still = asset.type === "image";- const dur = Math.max(0.1, (clip.out - clip.in) / speed);+ const dur = Math.max(0.1, rt.len);const frames = Math.max(1, Math.ceil(dur * opts.fps));const baked = clip.removal!.matte;@@ -143,7 +142,7 @@ export async function renderRemovalPieces(for (let i = 0; i < frames; i++) {const s = Math.min(dur - 1 / (opts.fps * 2), i / opts.fps);- const srcT = still ? 0 : clip.in + s * speed;+ const srcT = still ? 0 : rt.srcAt(s);const frame = await reader.frameAt(srcT);if (frame.kind !== "ready") {rgbCtx.fillStyle = "#000000";diff --git a/site/src/cut/lib/store.test.ts b/site/src/cut/lib/store.test.tsindex e569a22e..bcac4d13 100644--- a/site/src/cut/lib/store.test.ts+++ b/site/src/cut/lib/store.test.ts@@ -1874,6 +1874,19 @@ describe("transcribe spec", () => {expect(spec.clips.find((c) => c.file === video.fileName)?.muted).toBe(false);});+ test("a curved clip ships its curve, so the mix lays it at the length it plays", () => {+ const video = asset(6);+ const clip = vclip({ assetId: video.id, start: 0, in: 0, out: 4, speedCurve: [[0, 0.5], [4, 0.5]] });+ const spec = cutTranscribeSpec({+ clips: [clip],+ assets: [video],+ audioClips: [],+ overlays: [],+ })!;+ expect(spec.clips[0].speedCurve).toEqual([[0, 0.5], [4, 0.5]]);+ expect(spec.duration).toBeCloseTo(8, 3);+ });+test("leaves a still on an upper track out of the audio items", () => {const video = asset(6);const still = { ...asset(0, "image"), fileName: "still.png" };@@ -1923,3 +1936,84 @@ describe("assetIdsInUse", () => {expect(assetIdsInUse({ clips: [], audioClips: [], overlays: [] }).size).toBe(0);});});++describe("speed curves", () => {+ test("a curve that lengthens a track-0 clip pushes the run behind it", () => {+ const a = asset(4);+ const c1 = vclip({ assetId: a.id, start: 0, in: 0, out: 2 });+ const c2 = vclip({ assetId: a.id, start: 2, in: 0, out: 2 });+ useEditor.setState({ assets: [a], clips: [c1, c2] });+ useEditor.getState().setClipSpeedCurve(c1.id, [+ [0, 0.5],+ [2, 0.5],+ ]);+ const s = useEditor.getState();+ expect(clipLen(s.clips[0])).toBeCloseTo(4, 3);+ expect(s.clips[1].start).toBeCloseTo(4, 3);+ // Leaving curve mode keeps the length as a uniform rate.+ useEditor.getState().setClipSpeedCurve(c1.id, undefined);+ const t = useEditor.getState();+ expect(t.clips[0].speedCurve).toBeUndefined();+ expect(t.clips[0].speed).toBeCloseTo(0.5, 3);+ expect(t.clips[1].start).toBeCloseTo(4, 3);+ });++ test("a crossing's handles read the rate at their own edge", () => {+ const a = asset(10);+ // Trimmed the same distance from both ends, so an edge picked by the+ // amount of source left would read the wrong one.+ const c1 = vclip({+ assetId: a.id,+ start: 0,+ in: 2,+ out: 8,+ speedCurve: [[2, 1], [8, 4]],+ });+ // The next clip abuts, so there is a cut for the crossing to play on.+ const c2 = vclip({ assetId: a.id, start: clipLen(c1), in: 2, out: 8 });+ useEditor.setState({ assets: [a], clips: [c1, c2] });+ useEditor.getState().setClipTransition(c1.id, 1.4, "audiocross");+ const s = useEditor.getState();+ const [sp] = getClipSpans(s.clips, s.assets);+ // Two source seconds past the out point, played at the tail's 4x: half a+ // timeline second, short of the 0.7 the crossing wanted.+ expect(sp.soundAhead).toBeCloseTo(0.5, 3);+ });++ test("a picked speed node that is gone leaves the clip's pose keys alone", () => {+ const a = asset(4);+ const c1 = vclip({+ assetId: a.id,+ start: 0,+ in: 0,+ out: 2,+ kf: [{ t: 1, x: 0.5, y: 0.5, scale: 1.2, rotation: 0, opacity: 1 }],+ });+ useEditor.setState({+ assets: [a],+ clips: [c1],+ selection: { kind: "clip", id: c1.id },+ // The strip left this behind; the clip carries no curve any more.+ selectedKey: { kind: "clip", id: c1.id, t: 1, track: "speed" },+ });+ useEditor.getState().deleteSelection();+ const s = useEditor.getState();+ expect(s.clips).toHaveLength(1);+ expect(s.clips[0].kf).toHaveLength(1);+ });++ test("Delete takes a picked node, never the clip", () => {+ const a = asset(4);+ const c1 = vclip({ assetId: a.id, start: 0, in: 0, out: 2, speedCurve: [[0, 1], [1, 2], [2, 1]] });+ useEditor.setState({+ assets: [a],+ clips: [c1],+ selection: { kind: "clip", id: c1.id },+ selectedKey: { kind: "clip", id: c1.id, t: 1, track: "speed" },+ });+ useEditor.getState().deleteSelection();+ const s = useEditor.getState();+ expect(s.clips).toHaveLength(1);+ expect(s.clips[0].speedCurve?.map((n) => n[0])).toEqual([0, 2]);+ });+});diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex 997280ac..1f5826f6 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -8,10 +8,14 @@ import {lineLikeShape,maskKeyAt,removeKeyAt,+ retimeOf,+ speedCurveOf,+ speedCurvePreset,upsertKey,type EffectId,type MaskKey,type OverlayKey,+ type SpeedNode,} from "@donkeycut/effects-kit";import { create } from "zustand";import type {@@ -299,8 +303,9 @@ export interface EditorState {selectedKey: {kind: "overlay" | "clip";id: string;+ /** The key's time; for a speed node, the source second it sits on. */t: number;- track: "pose" | "mask";+ track: "pose" | "mask" | "speed";} | null;playing: boolean;/** Set while a play is held at a standstill because the cut has nothing to@@ -439,6 +444,12 @@ export interface EditorState {/** Set a clip's playback rate (0.25–4). The footprint change carries the* timeline behind it the way a trim does (see resizeClipFootprint). */setClipSpeed: (id: string, speed: number) => void;+ /** Replace a clip's speed curve (undefined clears it back to a uniform rate+ * at the curve's average). Nodes are [source second, rate]; the footprint+ * change carries the timeline like setClipSpeed does. */+ setClipSpeedCurve: (id: string, nodes: SpeedNode[] | undefined) => void;+ /** Lay a preset ramp over the clip's trimmed span. */+ setClipSpeedPreset: (id: string, preset: string) => void;/** Set a clip's source trim points. While track 0 is the only video track,* a track-0 resize ripples: everything past the clip's tail rides the moved* edge both directions, gaps keeping their width. Otherwise the run rules@@ -815,7 +826,7 @@ const touches = (patch: object, keys: readonly string[]) => keys.some((k) => k i*/function settleClipFootprint(id: string, patch: Partial<VideoClip>) {const moved = touches(patch, ["start", "track"]);- if (!moved && !touches(patch, ["in", "out", "speed"])) return;+ if (!moved && !touches(patch, ["in", "out", "speed", "speedCurve"])) return;const st = useEditor.getState();const clip = st.clips.find((c) => c.id === id);if (!clip) return;@@ -1300,6 +1311,7 @@ export function cutTranscribeSpec(start: a.start,volume: a.volume,speed: a.speed,+ speedCurve: a.speedCurve,})).concat(overlayLayers(s.clips)@@ -1318,6 +1330,7 @@ export function cutTranscribeSpec(start: c.start,volume: 1,speed: c.speed,+ speedCurve: c.speedCurve,})));if (spans.length === 0 && audio.length === 0) return null;@@ -1339,6 +1352,7 @@ export function cutTranscribeSpec(out: sp.clip.out,muted: sp.clip.muted || assetIsSilent(sp.asset),speed: clipSpeed(sp.clip),+ speedCurve: sp.clip.speedCurve,// The clamped cross-dissolve overlap, so the mix overlaps clip// audio the same way the timeline does and cues stay in sync.transition: sp.transitionOut,@@ -2567,7 +2581,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {if (get().transitions.length &&!touches(patch, ["start", "track"]) &&- touches(patch, ["in", "out", "speed"])+ touches(patch, ["in", "out", "speed", "speedCurve"]))set((s) => ({ transitions: reanchorTransitions(before, s.clips, s.transitions) }));},@@ -2576,15 +2590,49 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {const clip = get().clips.find((c) => c.id === id);if (!clip) return;const clamped = Math.max(SPEED_FLOOR, speed);- if (Math.abs(clamped - clipSpeed(clip)) < 1e-4) return;- resizeClipFootprint(clip, { speed: clamped }, Math.max(MIN_LEN, (clip.out - clip.in) / clamped));+ // A uniform rate replaces any curve the clip carried.+ if (!clip.speedCurve && Math.abs(clamped - clipSpeed(clip)) < 1e-4) return;+ resizeClipFootprint(+ clip,+ { speed: clamped, speedCurve: undefined },+ Math.max(MIN_LEN, (clip.out - clip.in) / clamped)+ );+ },++ setClipSpeedCurve: (id, nodes) => {+ const clip = get().clips.find((c) => c.id === id);+ if (!clip) return;+ const cleaned = nodes ? speedCurveOf({ speedCurve: nodes }) : undefined;+ if (!cleaned) {+ if (!clip.speedCurve) return;+ // Leaving curve mode keeps the clip's length: the uniform rate is the+ // curve's average.+ const avg = Math.max(SPEED_FLOOR, retimeOf(clip).rate);+ resizeClipFootprint(clip, { speed: avg, speedCurve: undefined }, Math.max(MIN_LEN, retimeOf({ ...clip, speed: avg, speedCurve: undefined }).len));+ return;+ }+ const next = { ...clip, speedCurve: cleaned };+ if (retimeOf(next).key === retimeOf(clip).key) return;+ resizeClipFootprint(clip, { speedCurve: cleaned }, Math.max(MIN_LEN, retimeOf(next).len));+ },++ setClipSpeedPreset: (id, preset) => {+ const clip = get().clips.find((c) => c.id === id);+ if (!clip) return;+ const nodes = speedCurvePreset(preset, clip.in, clip.out);+ if (!nodes) return;+ get().setClipSpeedCurve(id, nodes);},setClipTrim: (id, nextIn, nextOut) => {const clip = get().clips.find((c) => c.id === id);if (!clip) return;if (Math.abs(nextIn - clip.in) < 1e-6 && Math.abs(nextOut - clip.out) < 1e-6) return;- resizeClipFootprint(clip, { in: nextIn, out: nextOut }, (nextOut - nextIn) / clipSpeed(clip));+ resizeClipFootprint(+ clip,+ { in: nextIn, out: nextOut },+ retimeOf({ ...clip, in: nextIn, out: nextOut }).len+ );},setClipTransition: (id, seconds, style) => {@@ -2951,7 +2999,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {setClipMaskKey: (id, tLocal, patch, opts) => {const c = get().clips.find((x) => x.id === id);if (!c?.mask) return;- const len = Math.max(0.1, (c.out - c.in) / clipSpeed(c));+ const len = clipLen(c);const t = Math.max(0, Math.min(tLocal, len));const existing = c.mask.kf?.find((k) => Math.abs(k.t - t) <= KEY_EPSILON);const next = { ...(existing ?? maskKeyAt(c.mask, t)), ...patch, t };@@ -3289,6 +3337,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {// Carry the clip's rate so the detached track keeps the muted picture's// length and stays in sync (an AudioClip plays at its own `speed`)....(clipSpeed(clip) !== 1 ? { speed: clipSpeed(clip) } : {}),+ ...(clip.speedCurve ? { speedCurve: clip.speedCurve } : {}),};set((s) => ({audioClips: [...s.audioClips, audio],@@ -3325,11 +3374,11 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {if (selection?.kind === "clip") {const c = get().clips.find((x) => x.id === selection.id);if (c && c.track !== 0) {- const sp = c.speed && c.speed > 0 ? c.speed : 1;- const eff = (c.out - c.in) / sp;+ const rt = retimeOf(c);+ const eff = rt.len;if (t > c.start + 0.05 && t < c.start + eff - 0.05) {push();- const cutIn = c.in + (t - c.start) * sp;+ const cutIn = rt.srcAt(t - c.start);// The left half hard-cuts into the right; the right keeps the// original dissolve into whatever came after (same as track 0).// The cut lands mid-footage, so the edges it creates stay plain:@@ -3403,8 +3452,8 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {);if (!span) return;push();- // Source time advances `speed`× faster than timeline time.- const cutAt = span.clip.in + (t - span.start) * clipSpeed(span.clip);+ // The clip's own map says which source second plays at `t`.+ const cutAt = retimeOf(span.clip).srcAt(t - span.start);// The left half hard-cuts into the right; the right keeps the original// dissolve into whatever came after. Both halves stay in place. The cut// lands mid-footage, so the edges it creates stay plain: the exit@@ -3425,6 +3474,18 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {// A picked keyframe is the smaller thing under the cursor: Delete takes// the key and leaves the item it belongs to alone.const key = st.selectedKey;+ if (key?.track === "speed") {+ // A speed node is the whole of this branch: its times are source+ // seconds, so falling through to the pose track would delete a+ // keyframe that merely shares the number.+ set({ selectedKey: null });+ const clip = st.clips.find((x) => x.id === key.id);+ const nodes = clip ? speedCurveOf(clip) : undefined;+ if (nodes && nodes.length > 1 && nodes.some((nd) => Math.abs(nd[0] - key.t) <= KEY_EPSILON)) {+ st.setClipSpeedCurve(key.id, nodes.filter((nd) => Math.abs(nd[0] - key.t) > KEY_EPSILON));+ }+ return;+ }if (key) {set({ selectedKey: null });const item =@@ -3596,10 +3657,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {// A layer clip carries no span (spans are track 0); use its// own footprint.const c = s.clips.find((x) => x.id === sel.id);- if (c) {- const speed = c.speed && c.speed > 0 ? c.speed : 1;- add(c.start, c.start + Math.max(0.1, (c.out - c.in) / speed));- }+ if (c) add(c.start, c.start + clipLen(c));}} else if (sel.kind === "audio") {const c = s.audioClips.find((x) => x.id === sel.id);@@ -3650,20 +3708,20 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {if (mi == null) continue;// Track-0 clips re-materialize onto track 0 (asClip), so a template// stands up its own video instead of an empty timeline.- layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, sound: sp.clip.sound, track: 1, asClip: true });+ layers.push({ media: mi, start: sp.start - start0, in: sp.clip.in, out: sp.clip.out, ...framingOf(sp.clip), muted: sp.clip.muted, speed: sp.clip.speed, speedCurve: sp.clip.speedCurve, sound: sp.clip.sound, track: 1, asClip: true });} else {const c = s.clips.find((x) => x.id === sel.id);if (!c) continue;const mi = mediaFor(c.assetId);if (mi == null) continue;- layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, sound: c.sound, track: c.track + 1 });+ layers.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, ...framingOf(c), muted: c.muted, speed: c.speed, speedCurve: c.speedCurve, sound: c.sound, track: c.track + 1 });}} else if (sel.kind === "audio") {const c = s.audioClips.find((x) => x.id === sel.id);if (!c) continue;const mi = mediaFor(c.assetId);if (mi == null) continue;- audio.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, volume: c.volume, fadeIn: c.fadeIn, fadeOut: c.fadeOut, speed: c.speed, sound: c.sound, duck: c.duck, lane: c.lane });+ audio.push({ media: mi, start: c.start - start0, in: c.in, out: c.out, volume: c.volume, fadeIn: c.fadeIn, fadeOut: c.fadeOut, speed: c.speed, speedCurve: c.speedCurve, sound: c.sound, duck: c.duck, lane: c.lane });} else if (sel.kind === "overlay") {const o = s.overlays.find((x) => x.id === sel.id);// Asset-backed stickers stay out: a template copies only the media@@ -3793,6 +3851,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {muted: l.muted,...templateFraming(l),...(l.speed ? { speed: l.speed } : {}),+ ...(l.speedCurve ? { speedCurve: l.speedCurve } : {}),...(l.sound ? { sound: l.sound } : {}),}));const topTrack = Math.max(0, ...overlayLayers(get().clips).map((c) => c.track));@@ -3811,6 +3870,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {muted: l.muted,...templateFraming(l),...(l.speed ? { speed: l.speed } : {}),+ ...(l.speedCurve ? { speedCurve: l.speedCurve } : {}),...(l.sound ? { sound: l.sound } : {}),}));const newAudio: AudioClip[] = template.audio@@ -3825,6 +3885,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {...(a.fadeIn ? { fadeIn: a.fadeIn } : {}),...(a.fadeOut ? { fadeOut: a.fadeOut } : {}),...(a.speed ? { speed: a.speed } : {}),+ ...(a.speedCurve ? { speedCurve: a.speedCurve } : {}),...(a.sound ? { sound: a.sound } : {}),...(a.duck !== undefined && a.duck < 1 ? { duck: a.duck } : {}),...(a.lane ? { lane: a.lane } : {}),@@ -4036,6 +4097,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {out: sp.clip.out,muted: false,speed: clipSpeed(sp.clip),+ speedCurve: sp.clip.speedCurve,transition: 0,},],@@ -5035,18 +5097,17 @@ export function serializeDoc(s: {};}-/** Effective playback rate of a video clip (>0, default 1). */+/** A clip's uniform playback rate (>0, default 1). A clip carrying a speed+ * curve has no single rate; read `retimeOf(c)` for its map. */export function clipSpeed(c: VideoClip) {const s = c.speed ?? 1;return s > 0 ? s : 1;}+/** Timeline footprint of a clip: its source span through its rate, or through+ * its speed curve when it carries one (see retimeOf). */export function clipLen(c: VideoClip | AudioClip) {- const src = c.out - c.in;- // Video clips play their source at `speed`, so the timeline footprint is- // shorter/longer than the source. Audio clips have no speed.- const eff = "speed" in c && c.speed && c.speed > 0 ? src / c.speed : src;- return Math.max(MIN_LEN, eff);+ return Math.max(MIN_LEN, retimeOf(c).len);}/** The longest fade a clip takes: half its footprint, so a fade in and a@@ -5504,8 +5565,8 @@ function buildClipSpans(clips: VideoClip[], assets: MediaAsset[], track: number)len,transitionOut: onSound ? 0 : overlap,soundOut,- soundAhead: handleSeconds(soundOut, asset.duration - clip.out, clip),- soundBack: handleSeconds(prevSound, clip.in, clip),+ soundAhead: handleSeconds(soundOut, asset.duration - clip.out, clip, "tail"),+ soundBack: handleSeconds(prevSound, clip.in, clip, "head"),});}return spans;@@ -5513,9 +5574,18 @@ function buildClipSpans(clips: VideoClip[], assets: MediaAsset[], track: number)/** How far a cross dissolve reaches into a clip's handle: the ramp half it* wants, capped by the source seconds actually there, in timeline seconds. */-function handleSeconds(want: number, sourceLeft: number, clip: VideoClip): number {+function handleSeconds(+ want: number,+ sourceLeft: number,+ clip: VideoClip,+ edge: "head" | "tail"+): number {if (want <= 0.01) return 0;- return Math.max(0, Math.min(want, Math.max(0, sourceLeft) / clipSpeed(clip)));+ // The handle plays at the pace of the footage beside the edge: a source+ // second past `out` lasts as long as the last source second inside it.+ const rt = retimeOf(clip);+ const edgeRate = rt.rateAtSrc(edge === "head" ? clip.in : clip.out);+ return Math.max(0, Math.min(want, Math.max(0, sourceLeft) / edgeRate));}/** One row of the timeline, whichever kind of thing sits on it. Video rows arediff --git a/site/src/cut/lib/visualFrames.ts b/site/src/cut/lib/visualFrames.tsindex 8ed968cb..6b9c0b4a 100644--- a/site/src/cut/lib/visualFrames.ts+++ b/site/src/cut/lib/visualFrames.ts@@ -1,5 +1,7 @@"use client";+import { retimeOf } from "@donkeycut/effects-kit";+import { framesAt } from "./mediaRead";import { rasterCanvasToDataUrl } from "./raster";import type { ClipSpan } from "./types";@@ -76,8 +78,7 @@ export async function captureTimelineFrames(spans: ClipSpan[]): Promise<Capturedconst at = ((i + 0.5) * total) / count;const span = visible.find((sp) => at >= sp.start && at < sp.start + sp.len);if (!span) continue;- const speed = span.clip.speed && span.clip.speed > 0 ? span.clip.speed : 1;- const srcTime = Math.max(0, span.clip.in + (at - span.start) * speed);+ const srcTime = Math.max(0, retimeOf(span.clip).srcAt(at - span.start));const group = bySource.get(span.asset.url) ?? { url: span.asset.url, moments: [] };group.moments.push({ at, srcTime });bySource.set(span.asset.url, group);