Open source CapCut alternative.
TypeScript
feat(site): a clip's rate is a curve over its footage [rebuild]
David Lebab7db2f8358
diff --git a/site/packages/effects-kit/src/gradeMath.ts b/site/packages/effects-kit/src/gradeMath.tsindex 8f394bed..3b662f7a 100644--- a/site/packages/effects-kit/src/gradeMath.ts+++ b/site/packages/effects-kit/src/gradeMath.ts@@ -34,6 +34,7 @@ import {normalizeGrade,} from "./colorGrade";import { GRADE_PRESETS } from "./gradePresets";+import { monotoneCubic } from "./monotone";const clamp01 = (v: number) => (v < 0 ? 0 : v > 1 ? 1 : v);const luma = (r: number, g: number, b: number) => 0.2126 * r + 0.7152 * g + 0.0722 * b;@@ -48,64 +49,11 @@ const luma = (r: number, g: number, b: number) => 0.2126 * r + 0.7152 * g + 0.07* The curve holds flat outside its first and last points.*/export function curveLut(points: CurvePoint[]): Float32Array {- const n = points.length;const xs = points.map((p) => p[0] / 255);const ys = points.map((p) => p[1] / 255);+ const at = monotoneCubic(xs, ys);const out = new Float32Array(256);- if (n === 1) {- out.fill(clamp01(ys[0] ?? 0));- return out;- }- const h: number[] = [];- const d: number[] = [];- for (let i = 0; i < n - 1; i++) {- const dx = Math.max(1e-6, xs[i + 1] - xs[i]);- h.push(dx);- d.push((ys[i + 1] - ys[i]) / dx);- }- const m: number[] = [d[0]];- for (let i = 1; i < n - 1; i++) {- m.push(d[i - 1] * d[i] <= 0 ? 0 : (d[i - 1] + d[i]) / 2);- }- m.push(d[n - 2]);- for (let i = 0; i < n - 1; i++) {- if (d[i] === 0) {- m[i] = 0;- m[i + 1] = 0;- continue;- }- const a = m[i] / d[i];- const b = m[i + 1] / d[i];- const s = a * a + b * b;- if (s > 9) {- const t = 3 / Math.sqrt(s);- m[i] = t * a * d[i];- m[i + 1] = t * b * d[i];- }- }- for (let s = 0; s < 256; s++) {- const x = s / 255;- if (x <= xs[0]) {- out[s] = clamp01(ys[0]);- continue;- }- if (x >= xs[n - 1]) {- out[s] = clamp01(ys[n - 1]);- continue;- }- let i = 0;- while (i < n - 2 && x > xs[i + 1]) i++;- const t = (x - xs[i]) / h[i];- const t2 = t * t;- const t3 = t2 * t;- out[s] =- clamp01(- ys[i] * (2 * t3 - 3 * t2 + 1) +- h[i] * m[i] * (t3 - 2 * t2 + t) +- ys[i + 1] * (-2 * t3 + 3 * t2) +- h[i] * m[i + 1] * (t3 - t2)- );- }+ for (let s = 0; s < 256; s++) out[s] = clamp01(at(s / 255));return out;}diff --git a/site/packages/effects-kit/src/index.ts b/site/packages/effects-kit/src/index.tsindex 38911f51..b30b9cad 100644--- a/site/packages/effects-kit/src/index.ts+++ b/site/packages/effects-kit/src/index.ts@@ -16,6 +16,8 @@ export * from "./lottie";export * from "./looks";export * from "./colorGrade";export * from "./gradeMath";+export * from "./monotone";+export * from "./retime";export * from "./gradeLut";export * from "./gradePresets";export * from "./gradeMatch";diff --git a/site/packages/effects-kit/src/monotone.ts b/site/packages/effects-kit/src/monotone.tsnew file mode 100644index 00000000..d89f0f28--- /dev/null+++ b/site/packages/effects-kit/src/monotone.ts@@ -0,0 +1,65 @@+/**+ * Fritsch–Carlson monotone cubic interpolation: smooth through every control+ * point, never overshooting between two of them. The tone curves and the speed+ * curves both sample it — a tone curve that overshoots clips a highlight, a+ * speed curve that overshoots runs the footage backward.+ */++/**+ * A sampler for the curve through `(xs[i], ys[i])`, `xs` ascending. The curve+ * holds its end values outside the first and last point.+ */+export function monotoneCubic(xs: number[], ys: number[]): (x: number) => number {+ const n = Math.min(xs.length, ys.length);+ if (n === 0) return () => 0;+ if (n === 1) return () => ys[0];+ const h: number[] = [];+ const d: number[] = [];+ for (let i = 0; i < n - 1; i++) {+ const dx = Math.max(1e-6, xs[i + 1] - xs[i]);+ h.push(dx);+ d.push((ys[i + 1] - ys[i]) / dx);+ }+ const m: number[] = [d[0]];+ for (let i = 1; i < n - 1; i++) {+ m.push(d[i - 1] * d[i] <= 0 ? 0 : (d[i - 1] + d[i]) / 2);+ }+ m.push(d[n - 2]);+ for (let i = 0; i < n - 1; i++) {+ if (d[i] === 0) {+ m[i] = 0;+ m[i + 1] = 0;+ continue;+ }+ const a = m[i] / d[i];+ const b = m[i + 1] / d[i];+ const s = a * a + b * b;+ if (s > 9) {+ const t = 3 / Math.sqrt(s);+ m[i] = t * a * d[i];+ m[i + 1] = t * b * d[i];+ }+ }+ return (x: number) => {+ if (x <= xs[0]) return ys[0];+ if (x >= xs[n - 1]) return ys[n - 1];+ // Binary search for the segment holding `x`.+ let lo = 0;+ let hi = n - 2;+ while (lo < hi) {+ const mid = (lo + hi + 1) >> 1;+ if (xs[mid] <= x) lo = mid;+ else hi = mid - 1;+ }+ const i = lo;+ const t = (x - xs[i]) / h[i];+ const t2 = t * t;+ const t3 = t2 * t;+ return (+ ys[i] * (2 * t3 - 3 * t2 + 1) ++ h[i] * m[i] * (t3 - 2 * t2 + t) ++ ys[i + 1] * (-2 * t3 + 3 * t2) ++ h[i] * m[i + 1] * (t3 - t2)+ );+ };+}diff --git a/site/packages/effects-kit/src/retime.test.ts b/site/packages/effects-kit/src/retime.test.tsnew file mode 100644index 00000000..b7aaa0d6--- /dev/null+++ b/site/packages/effects-kit/src/retime.test.ts@@ -0,0 +1,169 @@+import { describe, expect, test } from "bun:test";+import {+ flatSpeedCurve,+ retimeOf,+ speedCurveOf,+ speedCurvePreset,+ SPEED_CURVE_MAX,+ SPEED_CURVE_MIN,+ type SpeedNode,+} from "./retime";++describe("retimeOf", () => {+ test("a clip with no curve reproduces (out − in) / speed exactly", () => {+ const r = retimeOf({ in: 2, out: 8, speed: 1.5 });+ expect(r.uniform).toBe(true);+ expect(r.len).toBe(6 / 1.5);+ expect(r.srcAt(1)).toBe(3.5);+ expect(r.tAt(5)).toBe(2);+ expect(r.rateAt(0)).toBe(1.5);+ expect(r.knots).toEqual([+ [0, 0],+ [6, 4],+ ]);+ });++ test("a flat curve equals the uniform map", () => {+ const u = retimeOf({ in: 1, out: 7, speed: 2 });+ const c = retimeOf({ in: 1, out: 7, speedCurve: flatSpeedCurve({ in: 1, out: 7, speed: 2 }) });+ expect(c.uniform).toBe(false);+ expect(c.len).toBeCloseTo(u.len, 6);+ for (let t = 0; t <= u.len; t += 0.25) expect(c.srcAt(t)).toBeCloseTo(u.srcAt(t), 5);+ });++ test("len equals the numerical integral of 1 / rate", () => {+ const curve: SpeedNode[] = [+ [0, 1],+ [2, 4],+ [4, 0.5],+ [6, 1],+ ];+ const r = retimeOf({ in: 0, out: 6, speedCurve: curve });+ let sum = 0;+ const n = 60000;+ for (let i = 0; i < n; i++) {+ const s = ((i + 0.5) * 6) / n;+ sum += (6 / n) / r.rateAtSrc(s);+ }+ expect(r.len).toBeCloseTo(sum, 3);+ expect(r.rate).toBeCloseTo(6 / r.len, 6);+ });++ test("srcAt and tAt round-trip within a frame", () => {+ const curve: SpeedNode[] = [+ [1, 0.25],+ [3, 8],+ [5, 1],+ ];+ const r = retimeOf({ in: 1, out: 5, speedCurve: curve });+ for (let t = 0; t <= r.len; t += r.len / 97) {+ expect(r.tAt(r.srcAt(t))).toBeCloseTo(t, 3);+ }+ for (let s = 1; s <= 5; s += 0.05) {+ expect(r.srcAt(r.tAt(s))).toBeCloseTo(s, 3);+ }+ expect(r.srcAt(0)).toBe(1);+ expect(r.srcAt(r.len)).toBeCloseTo(5, 9);+ });++ test("the map is monotone and the rate stays in range", () => {+ const curve: SpeedNode[] = [+ [0, 0.1],+ [1, 10],+ [1.2, 0.1],+ [3, 10],+ ];+ const r = retimeOf({ in: 0, out: 3, speedCurve: curve });+ let prev = -1;+ for (let t = 0; t <= r.len; t += r.len / 500) {+ const s = r.srcAt(t);+ expect(s).toBeGreaterThanOrEqual(prev);+ prev = s;+ expect(r.rateAt(t)).toBeGreaterThanOrEqual(SPEED_CURVE_MIN);+ expect(r.rateAt(t)).toBeLessThanOrEqual(SPEED_CURVE_MAX);+ }+ });++ test("extrapolates past the span at the edge rates", () => {+ const r = retimeOf({+ in: 2,+ out: 6,+ speedCurve: [+ [2, 2],+ [6, 0.5],+ ],+ });+ expect(r.srcAt(-1)).toBeCloseTo(0, 9);+ expect(r.tAt(0)).toBeCloseTo(-1, 9);+ expect(r.srcAt(r.len + 1)).toBeCloseTo(6.5, 9);+ });++ test("trimming keeps the footage's pace: nodes hold still", () => {+ const curve: SpeedNode[] = [+ [0, 1],+ [5, 4],+ [10, 1],+ ];+ const whole = retimeOf({ in: 0, out: 10, speedCurve: curve });+ const left = retimeOf({ in: 0, out: 4, speedCurve: curve });+ const right = retimeOf({ in: 4, out: 10, speedCurve: curve });+ expect(left.len + right.len).toBeCloseTo(whole.len, 4);+ expect(left.rateAtSrc(3)).toBeCloseTo(whole.rateAtSrc(3), 9);+ expect(right.srcAt(0)).toBe(4);+ });++ test("knots start at the head, end at the tail, and follow the map", () => {+ const r = retimeOf({+ in: 3,+ out: 9,+ speedCurve: [+ [3, 1],+ [6, 5],+ [9, 1],+ ],+ });+ const first = r.knots[0];+ const last = r.knots[r.knots.length - 1];+ expect(first).toEqual([0, 0]);+ expect(last[0]).toBeCloseTo(6, 9);+ expect(last[1]).toBeCloseTo(r.len, 9);+ expect(r.knots.length).toBeGreaterThan(2);+ expect(r.knots.length).toBeLessThan(200);+ for (const [s, t] of r.knots) expect(t).toBeCloseTo(r.tAt(3 + s), 2);+ });++ test("memoizes on its inputs", () => {+ const c = { in: 0, out: 4, speedCurve: [[0, 1], [4, 2]] as SpeedNode[] };+ expect(retimeOf(c)).toBe(retimeOf({ ...c, speedCurve: [...c.speedCurve] }));+ expect(retimeOf(c)).not.toBe(retimeOf({ ...c, out: 5 }));+ });+});++describe("speedCurveOf", () => {+ test("sorts, clamps, and collapses duplicate moments", () => {+ expect(+ speedCurveOf({+ speedCurve: [+ [4, 0.01],+ [1, 50],+ [1.00001, 2],+ ],+ })+ ).toEqual([+ [1.00001, 2],+ [4, SPEED_CURVE_MIN],+ ]);+ expect(speedCurveOf({ speedCurve: [] })).toBeUndefined();+ expect(speedCurveOf({})).toBeUndefined();+ });+});++describe("speedCurvePreset", () => {+ test("lays a preset over the span in source seconds", () => {+ const nodes = speedCurvePreset("whip", 10, 20)!;+ expect(nodes[0]).toEqual([10, 1]);+ expect(nodes[nodes.length - 1]).toEqual([20, 1]);+ expect(nodes.find((n) => n[1] === 6)![0]).toBe(15);+ expect(speedCurvePreset("nope", 0, 1)).toBeUndefined();+ });+});diff --git a/site/packages/effects-kit/src/retime.ts b/site/packages/effects-kit/src/retime.tsnew file mode 100644index 00000000..5ff1ff36--- /dev/null+++ b/site/packages/effects-kit/src/retime.ts@@ -0,0 +1,361 @@+/**+ * Retiming: how a clip's footage maps onto the timeline.+ *+ * A clip plays its source span `[in, out]` at a rate. The rate is one number+ * (`speed`) or a curve — nodes of `[source second, rate]` joined by the same+ * monotone spline the tone curves use, holding flat past the first and last+ * node. Nodes live in source seconds, so trimming a clip or splitting it never+ * moves them: the footage keeps the pace it was given at every moment of it.+ *+ * Everything that turns a timeline second into a source second (the preview,+ * the exports, the filmstrip, the trim handles, the assistant's context) reads+ * this one map. The uniform case is closed form and costs nothing; a curve is+ * integrated once and memoized on its inputs.+ */++import { monotoneCubic } from "./monotone";++/** One node of a speed curve: the source second it sits on, and the rate there. */+export type SpeedNode = [number, number];++export const SPEED_CURVE_MIN = 0.1;+export const SPEED_CURVE_MAX = 10;++export interface Retimable {+ in: number;+ out: number;+ speed?: number;+ speedCurve?: SpeedNode[];+}++export interface Retime {+ in: number;+ out: number;+ /** Timeline seconds the span plays in. */+ len: number;+ /** One rate across the span (no curve). */+ uniform: boolean;+ /** The uniform rate, or the span's average rate under a curve. */+ rate: number;+ /** Timeline offset from the span's head → absolute source second. Linear+ * past either end at the edge rate, so a handle reaching outside the span+ * maps like the footage beside it. */+ srcAt(tLocal: number): number;+ /** Absolute source second → timeline offset from the span's head. */+ tAt(src: number): number;+ /** Rate at a timeline offset. */+ rateAt(tLocal: number): number;+ /** Rate at a source second. */+ rateAtSrc(src: number): number;+ /** The map as a reduced polyline of `[source seconds past in, timeline+ * seconds past the head]`, first knot `[0, 0]`, last `[out − in, len]`.+ * Renderers that cannot call a function (an ffmpeg expression) read this. */+ knots: [number, number][];+ /** Fingerprint of everything the map depends on. */+ key: string;+}++const clampRate = (v: number) =>+ v < SPEED_CURVE_MIN ? SPEED_CURVE_MIN : v > SPEED_CURVE_MAX ? SPEED_CURVE_MAX : v;++/** A clip's curve, cleaned: finite nodes, rates in range, ascending by source+ * second, one node per moment. Undefined when the clip carries no usable+ * curve. */+export function speedCurveOf(c: { speedCurve?: SpeedNode[] }): SpeedNode[] | undefined {+ const raw = c.speedCurve;+ if (!raw || raw.length === 0) return undefined;+ const nodes = raw+ .filter((n) => Array.isArray(n) && Number.isFinite(n[0]) && Number.isFinite(n[1]))+ .map((n): SpeedNode => [n[0], clampRate(n[1])])+ .sort((a, b) => a[0] - b[0]);+ const out: SpeedNode[] = [];+ for (const n of nodes) {+ const last = out[out.length - 1];+ if (last && Math.abs(last[0] - n[0]) < 1e-4) out[out.length - 1] = n;+ else out.push(n);+ }+ return out.length ? out : undefined;+}++export const hasSpeedCurve = (c: { speedCurve?: SpeedNode[] }) => !!speedCurveOf(c);++/** The rate function of a curve over source seconds. */+export function speedCurveRate(nodes: SpeedNode[]): (src: number) => number {+ const at = monotoneCubic(+ nodes.map((n) => n[0]),+ nodes.map((n) => n[1])+ );+ return (src) => clampRate(at(src));+}++/** Samples per source second the integral is taken at. */+const SAMPLES_PER_S = 120;+const MIN_SAMPLES = 32;+const MAX_SAMPLES = 8192;+/** Samples the integral takes over a span of source seconds. */+const sampleCount = (span: number) =>+ Math.min(MAX_SAMPLES, Math.max(MIN_SAMPLES, Math.round(Math.max(0, span) * SAMPLES_PER_S)));++/** Knot tolerance, timeline seconds: a quarter of a 60 fps frame. */+const KNOT_EPS = 1 / 240;++const memo = new Map<string, Retime>();+const MEMO_CAP = 256;+/** The memo's ceiling in bytes. A curved entry holds three sample tables, so+ * counting entries alone would let a session of trimming hold tens of+ * megabytes; the tables are what the cap is about. */+const MEMO_BYTES = 8 << 20;+const memoBytes = new Map<string, number>();+let memoHeld = 0;++const uniformRate = (c: { speed?: number }) => (c.speed && c.speed > 0 ? c.speed : 1);++function uniformRetime(inS: number, outS: number, rate: number, key: string): Retime {+ const len = Math.max(0, outS - inS) / rate;+ return {+ in: inS,+ out: outS,+ len,+ uniform: true,+ rate,+ srcAt: (t) => inS + t * rate,+ tAt: (s) => (s - inS) / rate,+ rateAt: () => rate,+ rateAtSrc: () => rate,+ knots: [+ [0, 0],+ [Math.max(0, outS - inS), len],+ ],+ key,+ };+}++/** Douglas–Peucker on a monotone polyline, keeping both ends. */+function reduceKnots(xs: Float64Array, ys: Float64Array, eps: number): [number, number][] {+ const n = xs.length;+ const keep = new Uint8Array(n);+ keep[0] = 1;+ keep[n - 1] = 1;+ const stack: [number, number][] = [[0, n - 1]];+ while (stack.length) {+ const [a, b] = stack.pop()!;+ if (b - a < 2) continue;+ const dx = xs[b] - xs[a];+ const dy = ys[b] - ys[a];+ let worst = -1;+ let worstD = eps;+ for (let i = a + 1; i < b; i++) {+ const t = dx === 0 ? 0 : (xs[i] - xs[a]) / dx;+ const d = Math.abs(ys[i] - (ys[a] + dy * t));+ if (d > worstD) {+ worstD = d;+ worst = i;+ }+ }+ if (worst < 0) continue;+ keep[worst] = 1;+ stack.push([a, worst], [worst, b]);+ }+ const out: [number, number][] = [];+ for (let i = 0; i < n; i++) if (keep[i]) out.push([xs[i], ys[i]]);+ return out;+}++function curvedRetime(inS: number, outS: number, nodes: SpeedNode[], key: string): Retime {+ const span = Math.max(0, outS - inS);+ const rate = speedCurveRate(nodes);+ const n = sampleCount(span);+ const ds = span / n;+ const srcs = new Float64Array(n + 1);+ const ts = new Float64Array(n + 1);+ let prevInv = 1 / rate(inS);+ srcs[0] = inS;+ for (let k = 1; k <= n; k++) {+ const s = inS + k * ds;+ const inv = 1 / rate(s);+ srcs[k] = s;+ ts[k] = ts[k - 1] + ds * 0.5 * (prevInv + inv);+ prevInv = inv;+ }+ const len = ts[n];+ const rateIn = rate(inS);+ const rateOut = rate(outS);+ const lookup = (table: Float64Array, v: number) => {+ // Largest index whose value is <= v.+ let lo = 0;+ let hi = n;+ while (lo < hi) {+ const mid = (lo + hi + 1) >> 1;+ if (table[mid] <= v) lo = mid;+ else hi = mid - 1;+ }+ return Math.min(lo, n - 1);+ };+ const srcAt = (t: number) => {+ if (t <= 0) return inS + t * rateIn;+ if (t >= len) return outS + (t - len) * rateOut;+ const k = lookup(ts, t);+ const w = ts[k + 1] - ts[k];+ const f = w > 0 ? (t - ts[k]) / w : 0;+ return srcs[k] + f * ds;+ };+ const tAt = (s: number) => {+ if (s <= inS) return (s - inS) / rateIn;+ if (s >= outS) return len + (s - outS) / rateOut;+ const k = Math.min(n - 1, Math.max(0, Math.floor((s - inS) / ds)));+ const f = (s - srcs[k]) / ds;+ return ts[k] + f * (ts[k + 1] - ts[k]);+ };+ const rel = new Float64Array(n + 1);+ for (let k = 0; k <= n; k++) rel[k] = srcs[k] - inS;+ return {+ in: inS,+ out: outS,+ len,+ uniform: false,+ rate: len > 0 ? span / len : rateIn,+ srcAt,+ tAt,+ rateAt: (t) => rate(srcAt(t)),+ rateAtSrc: rate,+ knots: reduceKnots(rel, ts, KNOT_EPS),+ key,+ };+}++/** The retime of a clip's span, memoized on everything it depends on. */+export function retimeOf(c: Retimable): Retime {+ const nodes = speedCurveOf(c);+ const key = nodes+ ? `${c.in.toFixed(4)}|${c.out.toFixed(4)}|${nodes.map((nd) => `${nd[0].toFixed(4)}:${nd[1].toFixed(4)}`).join(",")}`+ : `${c.in}|${c.out}|u${uniformRate(c)}`;+ const hit = memo.get(key);+ if (hit) return hit;+ const built = nodes+ ? curvedRetime(c.in, c.out, nodes, key)+ : uniformRetime(c.in, c.out, uniformRate(c), key);+ memo.set(key, built);+ // Three Float64 tables per curved entry, plus the knots it kept.+ const bytes = nodes ? (built.knots.length + 1) * 16 + sampleCount(c.out - c.in) * 24 : 128;+ memoBytes.set(key, bytes);+ memoHeld += bytes;+ while (memo.size > MEMO_CAP || memoHeld > MEMO_BYTES) {+ const oldest = memo.keys().next().value;+ if (oldest === undefined || oldest === key) break;+ memo.delete(oldest);+ memoHeld -= memoBytes.get(oldest) ?? 0;+ memoBytes.delete(oldest);+ }+ return built;+}++/** Timeline seconds a clip occupies. */+export const retimeLen = (c: Retimable) => retimeOf(c).len;++/** A flat curve at the clip's current rate, one node on each edge of the+ * span — what a clip starts from when it enters curve editing. */+export function flatSpeedCurve(c: Retimable): SpeedNode[] {+ const rate = clampRate(uniformRate(c));+ return [+ [c.in, rate],+ [c.out, rate],+ ];+}++/* ------------------------------------------------------------------ */+/* Preset ramps */+/* ------------------------------------------------------------------ */++export interface SpeedCurvePreset {+ id: string;+ label: string;+ /** What the ramp does, for menus and the assistant. */+ hint: string;+ /** Nodes as `[position 0..1 through the span, rate]`. */+ shape: SpeedNode[];+}++export const SPEED_CURVE_PRESETS: SpeedCurvePreset[] = [+ {+ id: "flat",+ label: "Flat",+ hint: "one rate across the clip",+ shape: [+ [0, 1],+ [1, 1],+ ],+ },+ {+ id: "rampIn",+ label: "Ramp in",+ hint: "opens fast and settles to normal",+ shape: [+ [0, 3],+ [0.45, 1],+ [1, 1],+ ],+ },+ {+ id: "rampOut",+ label: "Ramp out",+ hint: "plays normal then accelerates out",+ shape: [+ [0, 1],+ [0.55, 1],+ [1, 3],+ ],+ },+ {+ id: "whip",+ label: "Whip",+ hint: "a burst of speed in the middle",+ shape: [+ [0, 1],+ [0.35, 1],+ [0.5, 6],+ [0.65, 1],+ [1, 1],+ ],+ },+ {+ id: "hold",+ label: "Hold",+ hint: "slows into a moment in the middle and races either side of it",+ shape: [+ [0, 1.6],+ [0.35, 1.6],+ [0.5, 0.3],+ [0.65, 1.6],+ [1, 1.6],+ ],+ },+ {+ id: "stutter",+ label: "Stutter",+ hint: "alternating rushes and slow beats",+ shape: [+ [0, 1],+ [0.15, 4],+ [0.25, 0.5],+ [0.4, 4],+ [0.5, 0.5],+ [0.65, 4],+ [0.75, 0.5],+ [1, 1],+ ],+ },+];++export const SPEED_CURVE_PRESET_IDS = SPEED_CURVE_PRESETS.map((p) => p.id);++/** A preset laid over a span: its positions become source seconds. */+export function speedCurvePreset(id: string, inS: number, outS: number): SpeedNode[] | undefined {+ const p = SPEED_CURVE_PRESETS.find((x) => x.id === id);+ if (!p) return undefined;+ const span = Math.max(0, outS - inS);+ return p.shape.map(([x, r]): SpeedNode => [inS + x * span, r]);+}++/** One line per preset, for the assistant's catalog. */+export const speedCurvePresetCatalogText = () =>+ SPEED_CURVE_PRESETS.map((p) => `${p.id} (${p.hint})`).join("; ");diff --git a/site/src/cut/lib/timeStretch.test.ts b/site/src/cut/lib/timeStretch.test.tsindex 0d8fbc29..b5986f9f 100644--- a/site/src/cut/lib/timeStretch.test.ts+++ b/site/src/cut/lib/timeStretch.test.ts@@ -105,3 +105,22 @@ describe("timeStretch", () => {expect(timeStretch([], RATE, 0.5)).toEqual([]);});});++describe("timeStretch with a schedule", () => {+ test("lays a varying factor to the integral of the schedule, at pitch", () => {+ // 1 s at 1×, then 1 s at half speed: 3 s out.+ const [out] = timeStretch([sine(300, 2)], RATE, {+ factorAt: (sec) => (sec < 1 ? 1 : 2),+ });+ expect(out.length).toBe(RATE * 3);+ expect(pitchOf(out)).toBeCloseTo(300, -1);+ });++ test("honours the caller's output length", () => {+ const [out] = timeStretch([sine(300, 2)], RATE, {+ factorAt: (sec) => 1 + sec / 2,+ outLength: 12345,+ });+ expect(out.length).toBe(12345);+ });+});diff --git a/site/src/cut/lib/timeStretch.ts b/site/src/cut/lib/timeStretch.tsindex ecb2449a..4e0e9945 100644--- a/site/src/cut/lib/timeStretch.ts+++ b/site/src/cut/lib/timeStretch.ts@@ -35,6 +35,15 @@ function hann(length: number): Float32Array {return w;}+/** The output seconds a varying factor produces over `seconds` of input. */+function integrateFactor(factorAt: (sec: number) => number, seconds: number): number {+ const steps = Math.max(1, Math.ceil(seconds * 1000));+ const dt = seconds / steps;+ let sum = 0;+ for (let i = 0; i < steps; i++) sum += factorAt((i + 0.5) * dt) * dt;+ return sum;+}+/** Straight linear resample to `length` — the pitch-shifting behaviour, used* only where there is too little audio for a frame-based stretch to mean* anything. */@@ -53,25 +62,52 @@ function resampleLinear(input: Float32Array, length: number): Float32Array {return out;}+/**+ * A stretch that changes along the input: `factorAt` gives the output-over-+ * input ratio at a second into the input, and `outLength` (samples) says how+ * long the result must be — the caller integrates its own map for that, so+ * the sound lands exactly where the picture's map says it does.+ */+export interface StretchSchedule {+ factorAt(inSec: number): number;+ outLength?: number;+}+/*** Stretch `channels` so the result is `factor` times as long, at the same pitch.** `factor` is the output length over the input length: 0.5 makes a clip play in- * half the time (a 2× speed-up), 2 makes it take twice as long. Every channel is- * cut at the same points — choosing them per channel would move the two sides of- * a stereo image independently and collapse it.+ * half the time (a 2× speed-up), 2 makes it take twice as long. A schedule+ * varies that along the input, which is how a speed curve's sound is laid: the+ * synthesis hop stays fixed and the analysis hop follows the rate at each+ * frame. Every channel is cut at the same points — choosing them per channel+ * would move the two sides of a stereo image independently and collapse it.*/export function timeStretch(channels: Float32Array[],sampleRate: number,- factor: number+ factor: number | StretchSchedule): Float32Array[] {if (channels.length === 0) return [];- if (!(factor > 0) || !Number.isFinite(factor)) return channels.map((c) => c.slice());- if (factor === 1) return channels.map((c) => c.slice());-const inLength = channels[0].length;- const outLength = Math.max(1, Math.round(inLength * factor));+ const schedule = typeof factor === "number" ? null : factor;+ if (typeof factor === "number") {+ if (!(factor > 0) || !Number.isFinite(factor)) return channels.map((c) => c.slice());+ if (factor === 1) return channels.map((c) => c.slice());+ }+ const factorAt = schedule+ ? (sec: number) => {+ const f = schedule.factorAt(sec);+ return f > 0 && Number.isFinite(f) ? f : 1;+ }+ : () => factor as number;+ const outLength = Math.max(+ 1,+ schedule?.outLength ??+ (schedule+ ? Math.round(integrateFactor(factorAt, inLength / sampleRate) * sampleRate)+ : Math.round(inLength * (factor as number)))+ );// An even frame keeps the synthesis hop exactly half of it, which is what// makes the windows sum to one.const frame = Math.max(4, Math.round(FRAME_SECONDS * sampleRate) & ~1);@@ -80,7 +116,7 @@ export function timeStretch(}const hop = frame >> 1;- const analysisHop = Math.max(1, Math.round(hop / factor));+ const analysisHopAt = (at: number) => Math.max(1, Math.round(hop / factorAt(at / sampleRate)));const search = Math.max(1, Math.round(SEARCH_SECONDS * sampleRate));const match = Math.max(2, Math.min(hop, Math.round(MATCH_SECONDS * sampleRate)));const window = hann(frame);@@ -124,7 +160,7 @@ export function timeStretch(expected =nextFrom + match <= inLength ? guide.subarray(nextFrom, nextFrom + match) : null;outPos += hop;- analysis += analysisHop;+ analysis += analysisHopAt(analysis);}// The first and last half-frame are only covered by one window, so they fadediff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.tsindex 7a844668..33350f19 100644--- a/site/src/cut/lib/types.ts+++ b/site/src/cut/lib/types.ts@@ -2,6 +2,7 @@ import {behindSubjectMask,overlayKind,poseAt,+ retimeOf,stampOverlayKinds,stripDefaultOverlayKinds,type ClipRemoval,@@ -16,6 +17,7 @@ import {type OverlayPose,type ShapeKind,type ShapeOverlay,+ type SpeedNode,type StickerOverlay,type TextOverlay as KitTextOverlay,type WordEffectId,@@ -443,6 +445,7 @@ export function clipPoseAt(in: number;out: number;speed?: number;+ speedCurve?: SpeedNode[];rotation?: number;opacity?: number;kf?: OverlayKey[];@@ -450,7 +453,7 @@ export function clipPoseAt(tLocal: number): OverlayPose {const rect = rectOf(clip);- const len = Math.max(0.1, (clip.out - clip.in) / (clip.speed && clip.speed > 0 ? clip.speed : 1));+ const len = Math.max(0.1, retimeOf(clip).len);return poseAt({start: 0,@@ -539,6 +542,12 @@ export interface VideoClip {/** Playback rate, default 1 (absent). The source (out-in) seconds play in* (out-in)/speed timeline seconds, so >1 is faster and shorter. */speed?: number;+ /** A rate that changes through the footage: nodes of [source second, rate]+ * joined by a smooth curve, held flat past the outermost nodes. Present, it+ * is the clip's rate and `speed` is ignored; the timeline footprint is the+ * integral of the curve over the trim (see retimeOf). Nodes sit in source+ * seconds, so trims and splits leave them where they are. */+ speedCurve?: SpeedNode[];/** Transition into the next clip on this clip's track, in timeline seconds* (absent/0 = hard cut). Every style overlaps the two clips by this much,* so the cut shortens. On upper tracks a transition blends the incoming@@ -583,7 +592,7 @@ export interface VideoClip {// Color grading (the dual-renderer math) lives in the effects kit; the model// types and ranges re-export here so doc-model consumers keep one import.export { GRADE_BASIC_FIELDS, GRADE_HUE_MAX, GRADE_MAX, HSL_BANDS } from "@donkeycut/effects-kit";-export type { ClipSound, ColorGrade, GradePresetRef, HslBand } from "@donkeycut/effects-kit";+export type { ClipSound, ColorGrade, GradePresetRef, HslBand, SpeedNode } from "@donkeycut/effects-kit";// Background removal follows the same split: math and model in the kit, the// doc-model types re-exported here.@@ -984,6 +993,8 @@ export interface AudioClip {* a sped-up video clip, so it stays the same length and in sync with the* (now muted) picture. The timeline footprint is (out-in)/speed. */speed?: number;+ /** The video clip's speed curve, carried along on detach for the same reason. */+ speedCurve?: SpeedNode[];/** Voiceover ducking: while this clip is audible, every other sound (clip* audio and other soundtrack clips) drops to this gain, 0..1. Absent = no* ducking. Ducking clips never duck each other. */@@ -1022,6 +1033,7 @@ export interface TemplateLayer {opacity?: number;muted: boolean;speed?: number;+ speedCurve?: SpeedNode[];sound?: ClipSound;track: number;/** Came from video track 0 — re-materializes as a timeline clip, not an@@ -1037,6 +1049,7 @@ export interface TemplateAudio {fadeIn?: number;fadeOut?: number;speed?: number;+ speedCurve?: SpeedNode[];sound?: ClipSound;duck?: number;lane?: number;