Skip to content

Open source CapCut alternative.

TypeScript

feat(site): a clip's rate is a curve over its footage [rebuild]

David Lebab7db2f8358
diff --git a/site/packages/effects-kit/src/gradeMath.ts b/site/packages/effects-kit/src/gradeMath.ts
index 8f394bed..3b662f7a 100644
--- a/site/packages/effects-kit/src/gradeMath.ts
+++ b/site/packages/effects-kit/src/gradeMath.ts
@@ -34,6 +34,7 @@ import {
normalizeGrade,
} from "./colorGrade";
import { GRADE_PRESETS } from "./gradePresets";
+import { monotoneCubic } from "./monotone";
const clamp01 = (v: number) => (v < 0 ? 0 : v > 1 ? 1 : v);
const luma = (r: number, g: number, b: number) => 0.2126 * r + 0.7152 * g + 0.0722 * b;
@@ -48,64 +49,11 @@ const luma = (r: number, g: number, b: number) => 0.2126 * r + 0.7152 * g + 0.07
* The curve holds flat outside its first and last points.
*/
export function curveLut(points: CurvePoint[]): Float32Array {
- const n = points.length;
const xs = points.map((p) => p[0] / 255);
const ys = points.map((p) => p[1] / 255);
+ const at = monotoneCubic(xs, ys);
const out = new Float32Array(256);
- if (n === 1) {
- out.fill(clamp01(ys[0] ?? 0));
- return out;
- }
- const h: number[] = [];
- const d: number[] = [];
- for (let i = 0; i < n - 1; i++) {
- const dx = Math.max(1e-6, xs[i + 1] - xs[i]);
- h.push(dx);
- d.push((ys[i + 1] - ys[i]) / dx);
- }
- const m: number[] = [d[0]];
- for (let i = 1; i < n - 1; i++) {
- m.push(d[i - 1] * d[i] <= 0 ? 0 : (d[i - 1] + d[i]) / 2);
- }
- m.push(d[n - 2]);
- for (let i = 0; i < n - 1; i++) {
- if (d[i] === 0) {
- m[i] = 0;
- m[i + 1] = 0;
- continue;
- }
- const a = m[i] / d[i];
- const b = m[i + 1] / d[i];
- const s = a * a + b * b;
- if (s > 9) {
- const t = 3 / Math.sqrt(s);
- m[i] = t * a * d[i];
- m[i + 1] = t * b * d[i];
- }
- }
- for (let s = 0; s < 256; s++) {
- const x = s / 255;
- if (x <= xs[0]) {
- out[s] = clamp01(ys[0]);
- continue;
- }
- if (x >= xs[n - 1]) {
- out[s] = clamp01(ys[n - 1]);
- continue;
- }
- let i = 0;
- while (i < n - 2 && x > xs[i + 1]) i++;
- const t = (x - xs[i]) / h[i];
- const t2 = t * t;
- const t3 = t2 * t;
- out[s] =
- clamp01(
- ys[i] * (2 * t3 - 3 * t2 + 1) +
- h[i] * m[i] * (t3 - 2 * t2 + t) +
- ys[i + 1] * (-2 * t3 + 3 * t2) +
- h[i] * m[i + 1] * (t3 - t2)
- );
- }
+ for (let s = 0; s < 256; s++) out[s] = clamp01(at(s / 255));
return out;
}
diff --git a/site/packages/effects-kit/src/index.ts b/site/packages/effects-kit/src/index.ts
index 38911f51..b30b9cad 100644
--- a/site/packages/effects-kit/src/index.ts
+++ b/site/packages/effects-kit/src/index.ts
@@ -16,6 +16,8 @@ export * from "./lottie";
export * from "./looks";
export * from "./colorGrade";
export * from "./gradeMath";
+export * from "./monotone";
+export * from "./retime";
export * from "./gradeLut";
export * from "./gradePresets";
export * from "./gradeMatch";
diff --git a/site/packages/effects-kit/src/monotone.ts b/site/packages/effects-kit/src/monotone.ts
new file mode 100644
index 00000000..d89f0f28
--- /dev/null
+++ b/site/packages/effects-kit/src/monotone.ts
@@ -0,0 +1,65 @@
+/**
+ * Fritsch–Carlson monotone cubic interpolation: smooth through every control
+ * point, never overshooting between two of them. The tone curves and the speed
+ * curves both sample it — a tone curve that overshoots clips a highlight, a
+ * speed curve that overshoots runs the footage backward.
+ */
+
+/**
+ * A sampler for the curve through `(xs[i], ys[i])`, `xs` ascending. The curve
+ * holds its end values outside the first and last point.
+ */
+export function monotoneCubic(xs: number[], ys: number[]): (x: number) => number {
+ const n = Math.min(xs.length, ys.length);
+ if (n === 0) return () => 0;
+ if (n === 1) return () => ys[0];
+ const h: number[] = [];
+ const d: number[] = [];
+ for (let i = 0; i < n - 1; i++) {
+ const dx = Math.max(1e-6, xs[i + 1] - xs[i]);
+ h.push(dx);
+ d.push((ys[i + 1] - ys[i]) / dx);
+ }
+ const m: number[] = [d[0]];
+ for (let i = 1; i < n - 1; i++) {
+ m.push(d[i - 1] * d[i] <= 0 ? 0 : (d[i - 1] + d[i]) / 2);
+ }
+ m.push(d[n - 2]);
+ for (let i = 0; i < n - 1; i++) {
+ if (d[i] === 0) {
+ m[i] = 0;
+ m[i + 1] = 0;
+ continue;
+ }
+ const a = m[i] / d[i];
+ const b = m[i + 1] / d[i];
+ const s = a * a + b * b;
+ if (s > 9) {
+ const t = 3 / Math.sqrt(s);
+ m[i] = t * a * d[i];
+ m[i + 1] = t * b * d[i];
+ }
+ }
+ return (x: number) => {
+ if (x <= xs[0]) return ys[0];
+ if (x >= xs[n - 1]) return ys[n - 1];
+ // Binary search for the segment holding `x`.
+ let lo = 0;
+ let hi = n - 2;
+ while (lo < hi) {
+ const mid = (lo + hi + 1) >> 1;
+ if (xs[mid] <= x) lo = mid;
+ else hi = mid - 1;
+ }
+ const i = lo;
+ const t = (x - xs[i]) / h[i];
+ const t2 = t * t;
+ const t3 = t2 * t;
+ return (
+ ys[i] * (2 * t3 - 3 * t2 + 1) +
+ h[i] * m[i] * (t3 - 2 * t2 + t) +
+ ys[i + 1] * (-2 * t3 + 3 * t2) +
+ h[i] * m[i + 1] * (t3 - t2)
+ );
+ };
+}
diff --git a/site/packages/effects-kit/src/retime.test.ts b/site/packages/effects-kit/src/retime.test.ts
new file mode 100644
index 00000000..b7aaa0d6
--- /dev/null
+++ b/site/packages/effects-kit/src/retime.test.ts
@@ -0,0 +1,169 @@
+import { describe, expect, test } from "bun:test";
+import {
+ flatSpeedCurve,
+ retimeOf,
+ speedCurveOf,
+ speedCurvePreset,
+ SPEED_CURVE_MAX,
+ SPEED_CURVE_MIN,
+ type SpeedNode,
+} from "./retime";
+
+describe("retimeOf", () => {
+ test("a clip with no curve reproduces (out − in) / speed exactly", () => {
+ const r = retimeOf({ in: 2, out: 8, speed: 1.5 });
+ expect(r.uniform).toBe(true);
+ expect(r.len).toBe(6 / 1.5);
+ expect(r.srcAt(1)).toBe(3.5);
+ expect(r.tAt(5)).toBe(2);
+ expect(r.rateAt(0)).toBe(1.5);
+ expect(r.knots).toEqual([
+ [0, 0],
+ [6, 4],
+ ]);
+ });
+
+ test("a flat curve equals the uniform map", () => {
+ const u = retimeOf({ in: 1, out: 7, speed: 2 });
+ const c = retimeOf({ in: 1, out: 7, speedCurve: flatSpeedCurve({ in: 1, out: 7, speed: 2 }) });
+ expect(c.uniform).toBe(false);
+ expect(c.len).toBeCloseTo(u.len, 6);
+ for (let t = 0; t <= u.len; t += 0.25) expect(c.srcAt(t)).toBeCloseTo(u.srcAt(t), 5);
+ });
+
+ test("len equals the numerical integral of 1 / rate", () => {
+ const curve: SpeedNode[] = [
+ [0, 1],
+ [2, 4],
+ [4, 0.5],
+ [6, 1],
+ ];
+ const r = retimeOf({ in: 0, out: 6, speedCurve: curve });
+ let sum = 0;
+ const n = 60000;
+ for (let i = 0; i < n; i++) {
+ const s = ((i + 0.5) * 6) / n;
+ sum += (6 / n) / r.rateAtSrc(s);
+ }
+ expect(r.len).toBeCloseTo(sum, 3);
+ expect(r.rate).toBeCloseTo(6 / r.len, 6);
+ });
+
+ test("srcAt and tAt round-trip within a frame", () => {
+ const curve: SpeedNode[] = [
+ [1, 0.25],
+ [3, 8],
+ [5, 1],
+ ];
+ const r = retimeOf({ in: 1, out: 5, speedCurve: curve });
+ for (let t = 0; t <= r.len; t += r.len / 97) {
+ expect(r.tAt(r.srcAt(t))).toBeCloseTo(t, 3);
+ }
+ for (let s = 1; s <= 5; s += 0.05) {
+ expect(r.srcAt(r.tAt(s))).toBeCloseTo(s, 3);
+ }
+ expect(r.srcAt(0)).toBe(1);
+ expect(r.srcAt(r.len)).toBeCloseTo(5, 9);
+ });
+
+ test("the map is monotone and the rate stays in range", () => {
+ const curve: SpeedNode[] = [
+ [0, 0.1],
+ [1, 10],
+ [1.2, 0.1],
+ [3, 10],
+ ];
+ const r = retimeOf({ in: 0, out: 3, speedCurve: curve });
+ let prev = -1;
+ for (let t = 0; t <= r.len; t += r.len / 500) {
+ const s = r.srcAt(t);
+ expect(s).toBeGreaterThanOrEqual(prev);
+ prev = s;
+ expect(r.rateAt(t)).toBeGreaterThanOrEqual(SPEED_CURVE_MIN);
+ expect(r.rateAt(t)).toBeLessThanOrEqual(SPEED_CURVE_MAX);
+ }
+ });
+
+ test("extrapolates past the span at the edge rates", () => {
+ const r = retimeOf({
+ in: 2,
+ out: 6,
+ speedCurve: [
+ [2, 2],
+ [6, 0.5],
+ ],
+ });
+ expect(r.srcAt(-1)).toBeCloseTo(0, 9);
+ expect(r.tAt(0)).toBeCloseTo(-1, 9);
+ expect(r.srcAt(r.len + 1)).toBeCloseTo(6.5, 9);
+ });
+
+ test("trimming keeps the footage's pace: nodes hold still", () => {
+ const curve: SpeedNode[] = [
+ [0, 1],
+ [5, 4],
+ [10, 1],
+ ];
+ const whole = retimeOf({ in: 0, out: 10, speedCurve: curve });
+ const left = retimeOf({ in: 0, out: 4, speedCurve: curve });
+ const right = retimeOf({ in: 4, out: 10, speedCurve: curve });
+ expect(left.len + right.len).toBeCloseTo(whole.len, 4);
+ expect(left.rateAtSrc(3)).toBeCloseTo(whole.rateAtSrc(3), 9);
+ expect(right.srcAt(0)).toBe(4);
+ });
+
+ test("knots start at the head, end at the tail, and follow the map", () => {
+ const r = retimeOf({
+ in: 3,
+ out: 9,
+ speedCurve: [
+ [3, 1],
+ [6, 5],
+ [9, 1],
+ ],
+ });
+ const first = r.knots[0];
+ const last = r.knots[r.knots.length - 1];
+ expect(first).toEqual([0, 0]);
+ expect(last[0]).toBeCloseTo(6, 9);
+ expect(last[1]).toBeCloseTo(r.len, 9);
+ expect(r.knots.length).toBeGreaterThan(2);
+ expect(r.knots.length).toBeLessThan(200);
+ for (const [s, t] of r.knots) expect(t).toBeCloseTo(r.tAt(3 + s), 2);
+ });
+
+ test("memoizes on its inputs", () => {
+ const c = { in: 0, out: 4, speedCurve: [[0, 1], [4, 2]] as SpeedNode[] };
+ expect(retimeOf(c)).toBe(retimeOf({ ...c, speedCurve: [...c.speedCurve] }));
+ expect(retimeOf(c)).not.toBe(retimeOf({ ...c, out: 5 }));
+ });
+});
+
+describe("speedCurveOf", () => {
+ test("sorts, clamps, and collapses duplicate moments", () => {
+ expect(
+ speedCurveOf({
+ speedCurve: [
+ [4, 0.01],
+ [1, 50],
+ [1.00001, 2],
+ ],
+ })
+ ).toEqual([
+ [1.00001, 2],
+ [4, SPEED_CURVE_MIN],
+ ]);
+ expect(speedCurveOf({ speedCurve: [] })).toBeUndefined();
+ expect(speedCurveOf({})).toBeUndefined();
+ });
+});
+
+describe("speedCurvePreset", () => {
+ test("lays a preset over the span in source seconds", () => {
+ const nodes = speedCurvePreset("whip", 10, 20)!;
+ expect(nodes[0]).toEqual([10, 1]);
+ expect(nodes[nodes.length - 1]).toEqual([20, 1]);
+ expect(nodes.find((n) => n[1] === 6)![0]).toBe(15);
+ expect(speedCurvePreset("nope", 0, 1)).toBeUndefined();
+ });
+});
diff --git a/site/packages/effects-kit/src/retime.ts b/site/packages/effects-kit/src/retime.ts
new file mode 100644
index 00000000..5ff1ff36
--- /dev/null
+++ b/site/packages/effects-kit/src/retime.ts
@@ -0,0 +1,361 @@
+/**
+ * Retiming: how a clip's footage maps onto the timeline.
+ *
+ * A clip plays its source span `[in, out]` at a rate. The rate is one number
+ * (`speed`) or a curve — nodes of `[source second, rate]` joined by the same
+ * monotone spline the tone curves use, holding flat past the first and last
+ * node. Nodes live in source seconds, so trimming a clip or splitting it never
+ * moves them: the footage keeps the pace it was given at every moment of it.
+ *
+ * Everything that turns a timeline second into a source second (the preview,
+ * the exports, the filmstrip, the trim handles, the assistant's context) reads
+ * this one map. The uniform case is closed form and costs nothing; a curve is
+ * integrated once and memoized on its inputs.
+ */
+
+import { monotoneCubic } from "./monotone";
+
+/** One node of a speed curve: the source second it sits on, and the rate there. */
+export type SpeedNode = [number, number];
+
+export const SPEED_CURVE_MIN = 0.1;
+export const SPEED_CURVE_MAX = 10;
+
+export interface Retimable {
+ in: number;
+ out: number;
+ speed?: number;
+ speedCurve?: SpeedNode[];
+}
+
+export interface Retime {
+ in: number;
+ out: number;
+ /** Timeline seconds the span plays in. */
+ len: number;
+ /** One rate across the span (no curve). */
+ uniform: boolean;
+ /** The uniform rate, or the span's average rate under a curve. */
+ rate: number;
+ /** Timeline offset from the span's head → absolute source second. Linear
+ * past either end at the edge rate, so a handle reaching outside the span
+ * maps like the footage beside it. */
+ srcAt(tLocal: number): number;
+ /** Absolute source second → timeline offset from the span's head. */
+ tAt(src: number): number;
+ /** Rate at a timeline offset. */
+ rateAt(tLocal: number): number;
+ /** Rate at a source second. */
+ rateAtSrc(src: number): number;
+ /** The map as a reduced polyline of `[source seconds past in, timeline
+ * seconds past the head]`, first knot `[0, 0]`, last `[out − in, len]`.
+ * Renderers that cannot call a function (an ffmpeg expression) read this. */
+ knots: [number, number][];
+ /** Fingerprint of everything the map depends on. */
+ key: string;
+}
+
+const clampRate = (v: number) =>
+ v < SPEED_CURVE_MIN ? SPEED_CURVE_MIN : v > SPEED_CURVE_MAX ? SPEED_CURVE_MAX : v;
+
+/** A clip's curve, cleaned: finite nodes, rates in range, ascending by source
+ * second, one node per moment. Undefined when the clip carries no usable
+ * curve. */
+export function speedCurveOf(c: { speedCurve?: SpeedNode[] }): SpeedNode[] | undefined {
+ const raw = c.speedCurve;
+ if (!raw || raw.length === 0) return undefined;
+ const nodes = raw
+ .filter((n) => Array.isArray(n) && Number.isFinite(n[0]) && Number.isFinite(n[1]))
+ .map((n): SpeedNode => [n[0], clampRate(n[1])])
+ .sort((a, b) => a[0] - b[0]);
+ const out: SpeedNode[] = [];
+ for (const n of nodes) {
+ const last = out[out.length - 1];
+ if (last && Math.abs(last[0] - n[0]) < 1e-4) out[out.length - 1] = n;
+ else out.push(n);
+ }
+ return out.length ? out : undefined;
+}
+
+export const hasSpeedCurve = (c: { speedCurve?: SpeedNode[] }) => !!speedCurveOf(c);
+
+/** The rate function of a curve over source seconds. */
+export function speedCurveRate(nodes: SpeedNode[]): (src: number) => number {
+ const at = monotoneCubic(
+ nodes.map((n) => n[0]),
+ nodes.map((n) => n[1])
+ );
+ return (src) => clampRate(at(src));
+}
+
+/** Samples per source second the integral is taken at. */
+const SAMPLES_PER_S = 120;
+const MIN_SAMPLES = 32;
+const MAX_SAMPLES = 8192;
+/** Samples the integral takes over a span of source seconds. */
+const sampleCount = (span: number) =>
+ Math.min(MAX_SAMPLES, Math.max(MIN_SAMPLES, Math.round(Math.max(0, span) * SAMPLES_PER_S)));
+
+/** Knot tolerance, timeline seconds: a quarter of a 60 fps frame. */
+const KNOT_EPS = 1 / 240;
+
+const memo = new Map<string, Retime>();
+const MEMO_CAP = 256;
+/** The memo's ceiling in bytes. A curved entry holds three sample tables, so
+ * counting entries alone would let a session of trimming hold tens of
+ * megabytes; the tables are what the cap is about. */
+const MEMO_BYTES = 8 << 20;
+const memoBytes = new Map<string, number>();
+let memoHeld = 0;
+
+const uniformRate = (c: { speed?: number }) => (c.speed && c.speed > 0 ? c.speed : 1);
+
+function uniformRetime(inS: number, outS: number, rate: number, key: string): Retime {
+ const len = Math.max(0, outS - inS) / rate;
+ return {
+ in: inS,
+ out: outS,
+ len,
+ uniform: true,
+ rate,
+ srcAt: (t) => inS + t * rate,
+ tAt: (s) => (s - inS) / rate,
+ rateAt: () => rate,
+ rateAtSrc: () => rate,
+ knots: [
+ [0, 0],
+ [Math.max(0, outS - inS), len],
+ ],
+ key,
+ };
+}
+
+/** Douglas–Peucker on a monotone polyline, keeping both ends. */
+function reduceKnots(xs: Float64Array, ys: Float64Array, eps: number): [number, number][] {
+ const n = xs.length;
+ const keep = new Uint8Array(n);
+ keep[0] = 1;
+ keep[n - 1] = 1;
+ const stack: [number, number][] = [[0, n - 1]];
+ while (stack.length) {
+ const [a, b] = stack.pop()!;
+ if (b - a < 2) continue;
+ const dx = xs[b] - xs[a];
+ const dy = ys[b] - ys[a];
+ let worst = -1;
+ let worstD = eps;
+ for (let i = a + 1; i < b; i++) {
+ const t = dx === 0 ? 0 : (xs[i] - xs[a]) / dx;
+ const d = Math.abs(ys[i] - (ys[a] + dy * t));
+ if (d > worstD) {
+ worstD = d;
+ worst = i;
+ }
+ }
+ if (worst < 0) continue;
+ keep[worst] = 1;
+ stack.push([a, worst], [worst, b]);
+ }
+ const out: [number, number][] = [];
+ for (let i = 0; i < n; i++) if (keep[i]) out.push([xs[i], ys[i]]);
+ return out;
+}
+
+function curvedRetime(inS: number, outS: number, nodes: SpeedNode[], key: string): Retime {
+ const span = Math.max(0, outS - inS);
+ const rate = speedCurveRate(nodes);
+ const n = sampleCount(span);
+ const ds = span / n;
+ const srcs = new Float64Array(n + 1);
+ const ts = new Float64Array(n + 1);
+ let prevInv = 1 / rate(inS);
+ srcs[0] = inS;
+ for (let k = 1; k <= n; k++) {
+ const s = inS + k * ds;
+ const inv = 1 / rate(s);
+ srcs[k] = s;
+ ts[k] = ts[k - 1] + ds * 0.5 * (prevInv + inv);
+ prevInv = inv;
+ }
+ const len = ts[n];
+ const rateIn = rate(inS);
+ const rateOut = rate(outS);
+ const lookup = (table: Float64Array, v: number) => {
+ // Largest index whose value is <= v.
+ let lo = 0;
+ let hi = n;
+ while (lo < hi) {
+ const mid = (lo + hi + 1) >> 1;
+ if (table[mid] <= v) lo = mid;
+ else hi = mid - 1;
+ }
+ return Math.min(lo, n - 1);
+ };
+ const srcAt = (t: number) => {
+ if (t <= 0) return inS + t * rateIn;
+ if (t >= len) return outS + (t - len) * rateOut;
+ const k = lookup(ts, t);
+ const w = ts[k + 1] - ts[k];
+ const f = w > 0 ? (t - ts[k]) / w : 0;
+ return srcs[k] + f * ds;
+ };
+ const tAt = (s: number) => {
+ if (s <= inS) return (s - inS) / rateIn;
+ if (s >= outS) return len + (s - outS) / rateOut;
+ const k = Math.min(n - 1, Math.max(0, Math.floor((s - inS) / ds)));
+ const f = (s - srcs[k]) / ds;
+ return ts[k] + f * (ts[k + 1] - ts[k]);
+ };
+ const rel = new Float64Array(n + 1);
+ for (let k = 0; k <= n; k++) rel[k] = srcs[k] - inS;
+ return {
+ in: inS,
+ out: outS,
+ len,
+ uniform: false,
+ rate: len > 0 ? span / len : rateIn,
+ srcAt,
+ tAt,
+ rateAt: (t) => rate(srcAt(t)),
+ rateAtSrc: rate,
+ knots: reduceKnots(rel, ts, KNOT_EPS),
+ key,
+ };
+}
+
+/** The retime of a clip's span, memoized on everything it depends on. */
+export function retimeOf(c: Retimable): Retime {
+ const nodes = speedCurveOf(c);
+ const key = nodes
+ ? `${c.in.toFixed(4)}|${c.out.toFixed(4)}|${nodes.map((nd) => `${nd[0].toFixed(4)}:${nd[1].toFixed(4)}`).join(",")}`
+ : `${c.in}|${c.out}|u${uniformRate(c)}`;
+ const hit = memo.get(key);
+ if (hit) return hit;
+ const built = nodes
+ ? curvedRetime(c.in, c.out, nodes, key)
+ : uniformRetime(c.in, c.out, uniformRate(c), key);
+ memo.set(key, built);
+ // Three Float64 tables per curved entry, plus the knots it kept.
+ const bytes = nodes ? (built.knots.length + 1) * 16 + sampleCount(c.out - c.in) * 24 : 128;
+ memoBytes.set(key, bytes);
+ memoHeld += bytes;
+ while (memo.size > MEMO_CAP || memoHeld > MEMO_BYTES) {
+ const oldest = memo.keys().next().value;
+ if (oldest === undefined || oldest === key) break;
+ memo.delete(oldest);
+ memoHeld -= memoBytes.get(oldest) ?? 0;
+ memoBytes.delete(oldest);
+ }
+ return built;
+}
+
+/** Timeline seconds a clip occupies. */
+export const retimeLen = (c: Retimable) => retimeOf(c).len;
+
+/** A flat curve at the clip's current rate, one node on each edge of the
+ * span — what a clip starts from when it enters curve editing. */
+export function flatSpeedCurve(c: Retimable): SpeedNode[] {
+ const rate = clampRate(uniformRate(c));
+ return [
+ [c.in, rate],
+ [c.out, rate],
+ ];
+}
+
+/* ------------------------------------------------------------------ */
+/* Preset ramps */
+/* ------------------------------------------------------------------ */
+
+export interface SpeedCurvePreset {
+ id: string;
+ label: string;
+ /** What the ramp does, for menus and the assistant. */
+ hint: string;
+ /** Nodes as `[position 0..1 through the span, rate]`. */
+ shape: SpeedNode[];
+}
+
+export const SPEED_CURVE_PRESETS: SpeedCurvePreset[] = [
+ {
+ id: "flat",
+ label: "Flat",
+ hint: "one rate across the clip",
+ shape: [
+ [0, 1],
+ [1, 1],
+ ],
+ },
+ {
+ id: "rampIn",
+ label: "Ramp in",
+ hint: "opens fast and settles to normal",
+ shape: [
+ [0, 3],
+ [0.45, 1],
+ [1, 1],
+ ],
+ },
+ {
+ id: "rampOut",
+ label: "Ramp out",
+ hint: "plays normal then accelerates out",
+ shape: [
+ [0, 1],
+ [0.55, 1],
+ [1, 3],
+ ],
+ },
+ {
+ id: "whip",
+ label: "Whip",
+ hint: "a burst of speed in the middle",
+ shape: [
+ [0, 1],
+ [0.35, 1],
+ [0.5, 6],
+ [0.65, 1],
+ [1, 1],
+ ],
+ },
+ {
+ id: "hold",
+ label: "Hold",
+ hint: "slows into a moment in the middle and races either side of it",
+ shape: [
+ [0, 1.6],
+ [0.35, 1.6],
+ [0.5, 0.3],
+ [0.65, 1.6],
+ [1, 1.6],
+ ],
+ },
+ {
+ id: "stutter",
+ label: "Stutter",
+ hint: "alternating rushes and slow beats",
+ shape: [
+ [0, 1],
+ [0.15, 4],
+ [0.25, 0.5],
+ [0.4, 4],
+ [0.5, 0.5],
+ [0.65, 4],
+ [0.75, 0.5],
+ [1, 1],
+ ],
+ },
+];
+
+export const SPEED_CURVE_PRESET_IDS = SPEED_CURVE_PRESETS.map((p) => p.id);
+
+/** A preset laid over a span: its positions become source seconds. */
+export function speedCurvePreset(id: string, inS: number, outS: number): SpeedNode[] | undefined {
+ const p = SPEED_CURVE_PRESETS.find((x) => x.id === id);
+ if (!p) return undefined;
+ const span = Math.max(0, outS - inS);
+ return p.shape.map(([x, r]): SpeedNode => [inS + x * span, r]);
+}
+
+/** One line per preset, for the assistant's catalog. */
+export const speedCurvePresetCatalogText = () =>
+ SPEED_CURVE_PRESETS.map((p) => `${p.id} (${p.hint})`).join("; ");
diff --git a/site/src/cut/lib/timeStretch.test.ts b/site/src/cut/lib/timeStretch.test.ts
index 0d8fbc29..b5986f9f 100644
--- a/site/src/cut/lib/timeStretch.test.ts
+++ b/site/src/cut/lib/timeStretch.test.ts
@@ -105,3 +105,22 @@ describe("timeStretch", () => {
expect(timeStretch([], RATE, 0.5)).toEqual([]);
});
});
+
+describe("timeStretch with a schedule", () => {
+ test("lays a varying factor to the integral of the schedule, at pitch", () => {
+ // 1 s at 1×, then 1 s at half speed: 3 s out.
+ const [out] = timeStretch([sine(300, 2)], RATE, {
+ factorAt: (sec) => (sec < 1 ? 1 : 2),
+ });
+ expect(out.length).toBe(RATE * 3);
+ expect(pitchOf(out)).toBeCloseTo(300, -1);
+ });
+
+ test("honours the caller's output length", () => {
+ const [out] = timeStretch([sine(300, 2)], RATE, {
+ factorAt: (sec) => 1 + sec / 2,
+ outLength: 12345,
+ });
+ expect(out.length).toBe(12345);
+ });
+});
diff --git a/site/src/cut/lib/timeStretch.ts b/site/src/cut/lib/timeStretch.ts
index ecb2449a..4e0e9945 100644
--- a/site/src/cut/lib/timeStretch.ts
+++ b/site/src/cut/lib/timeStretch.ts
@@ -35,6 +35,15 @@ function hann(length: number): Float32Array {
return w;
}
+/** The output seconds a varying factor produces over `seconds` of input. */
+function integrateFactor(factorAt: (sec: number) => number, seconds: number): number {
+ const steps = Math.max(1, Math.ceil(seconds * 1000));
+ const dt = seconds / steps;
+ let sum = 0;
+ for (let i = 0; i < steps; i++) sum += factorAt((i + 0.5) * dt) * dt;
+ return sum;
+}
+
/** Straight linear resample to `length` — the pitch-shifting behaviour, used
* only where there is too little audio for a frame-based stretch to mean
* anything. */
@@ -53,25 +62,52 @@ function resampleLinear(input: Float32Array, length: number): Float32Array {
return out;
}
+/**
+ * A stretch that changes along the input: `factorAt` gives the output-over-
+ * input ratio at a second into the input, and `outLength` (samples) says how
+ * long the result must be — the caller integrates its own map for that, so
+ * the sound lands exactly where the picture's map says it does.
+ */
+export interface StretchSchedule {
+ factorAt(inSec: number): number;
+ outLength?: number;
+}
+
/**
* Stretch `channels` so the result is `factor` times as long, at the same pitch.
*
* `factor` is the output length over the input length: 0.5 makes a clip play in
- * half the time (a 2× speed-up), 2 makes it take twice as long. Every channel is
- * cut at the same points — choosing them per channel would move the two sides of
- * a stereo image independently and collapse it.
+ * half the time (a 2× speed-up), 2 makes it take twice as long. A schedule
+ * varies that along the input, which is how a speed curve's sound is laid: the
+ * synthesis hop stays fixed and the analysis hop follows the rate at each
+ * frame. Every channel is cut at the same points — choosing them per channel
+ * would move the two sides of a stereo image independently and collapse it.
*/
export function timeStretch(
channels: Float32Array[],
sampleRate: number,
- factor: number
+ factor: number | StretchSchedule
): Float32Array[] {
if (channels.length === 0) return [];
- if (!(factor > 0) || !Number.isFinite(factor)) return channels.map((c) => c.slice());
- if (factor === 1) return channels.map((c) => c.slice());
-
const inLength = channels[0].length;
- const outLength = Math.max(1, Math.round(inLength * factor));
+ const schedule = typeof factor === "number" ? null : factor;
+ if (typeof factor === "number") {
+ if (!(factor > 0) || !Number.isFinite(factor)) return channels.map((c) => c.slice());
+ if (factor === 1) return channels.map((c) => c.slice());
+ }
+ const factorAt = schedule
+ ? (sec: number) => {
+ const f = schedule.factorAt(sec);
+ return f > 0 && Number.isFinite(f) ? f : 1;
+ }
+ : () => factor as number;
+ const outLength = Math.max(
+ 1,
+ schedule?.outLength ??
+ (schedule
+ ? Math.round(integrateFactor(factorAt, inLength / sampleRate) * sampleRate)
+ : Math.round(inLength * (factor as number)))
+ );
// An even frame keeps the synthesis hop exactly half of it, which is what
// makes the windows sum to one.
const frame = Math.max(4, Math.round(FRAME_SECONDS * sampleRate) & ~1);
@@ -80,7 +116,7 @@ export function timeStretch(
}
const hop = frame >> 1;
- const analysisHop = Math.max(1, Math.round(hop / factor));
+ const analysisHopAt = (at: number) => Math.max(1, Math.round(hop / factorAt(at / sampleRate)));
const search = Math.max(1, Math.round(SEARCH_SECONDS * sampleRate));
const match = Math.max(2, Math.min(hop, Math.round(MATCH_SECONDS * sampleRate)));
const window = hann(frame);
@@ -124,7 +160,7 @@ export function timeStretch(
expected =
nextFrom + match <= inLength ? guide.subarray(nextFrom, nextFrom + match) : null;
outPos += hop;
- analysis += analysisHop;
+ analysis += analysisHopAt(analysis);
}
// The first and last half-frame are only covered by one window, so they fade
diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.ts
index 7a844668..33350f19 100644
--- a/site/src/cut/lib/types.ts
+++ b/site/src/cut/lib/types.ts
@@ -2,6 +2,7 @@ import {
behindSubjectMask,
overlayKind,
poseAt,
+ retimeOf,
stampOverlayKinds,
stripDefaultOverlayKinds,
type ClipRemoval,
@@ -16,6 +17,7 @@ import {
type OverlayPose,
type ShapeKind,
type ShapeOverlay,
+ type SpeedNode,
type StickerOverlay,
type TextOverlay as KitTextOverlay,
type WordEffectId,
@@ -443,6 +445,7 @@ export function clipPoseAt(
in: number;
out: number;
speed?: number;
+ speedCurve?: SpeedNode[];
rotation?: number;
opacity?: number;
kf?: OverlayKey[];
@@ -450,7 +453,7 @@ export function clipPoseAt(
tLocal: number
): OverlayPose {
const rect = rectOf(clip);
- const len = Math.max(0.1, (clip.out - clip.in) / (clip.speed && clip.speed > 0 ? clip.speed : 1));
+ const len = Math.max(0.1, retimeOf(clip).len);
return poseAt(
{
start: 0,
@@ -539,6 +542,12 @@ export interface VideoClip {
/** Playback rate, default 1 (absent). The source (out-in) seconds play in
* (out-in)/speed timeline seconds, so >1 is faster and shorter. */
speed?: number;
+ /** A rate that changes through the footage: nodes of [source second, rate]
+ * joined by a smooth curve, held flat past the outermost nodes. Present, it
+ * is the clip's rate and `speed` is ignored; the timeline footprint is the
+ * integral of the curve over the trim (see retimeOf). Nodes sit in source
+ * seconds, so trims and splits leave them where they are. */
+ speedCurve?: SpeedNode[];
/** Transition into the next clip on this clip's track, in timeline seconds
* (absent/0 = hard cut). Every style overlaps the two clips by this much,
* so the cut shortens. On upper tracks a transition blends the incoming
@@ -583,7 +592,7 @@ export interface VideoClip {
// Color grading (the dual-renderer math) lives in the effects kit; the model
// types and ranges re-export here so doc-model consumers keep one import.
export { GRADE_BASIC_FIELDS, GRADE_HUE_MAX, GRADE_MAX, HSL_BANDS } from "@donkeycut/effects-kit";
-export type { ClipSound, ColorGrade, GradePresetRef, HslBand } from "@donkeycut/effects-kit";
+export type { ClipSound, ColorGrade, GradePresetRef, HslBand, SpeedNode } from "@donkeycut/effects-kit";
// Background removal follows the same split: math and model in the kit, the
// doc-model types re-exported here.
@@ -984,6 +993,8 @@ export interface AudioClip {
* a sped-up video clip, so it stays the same length and in sync with the
* (now muted) picture. The timeline footprint is (out-in)/speed. */
speed?: number;
+ /** The video clip's speed curve, carried along on detach for the same reason. */
+ speedCurve?: SpeedNode[];
/** Voiceover ducking: while this clip is audible, every other sound (clip
* audio and other soundtrack clips) drops to this gain, 0..1. Absent = no
* ducking. Ducking clips never duck each other. */
@@ -1022,6 +1033,7 @@ export interface TemplateLayer {
opacity?: number;
muted: boolean;
speed?: number;
+ speedCurve?: SpeedNode[];
sound?: ClipSound;
track: number;
/** Came from video track 0 — re-materializes as a timeline clip, not an
@@ -1037,6 +1049,7 @@ export interface TemplateAudio {
fadeIn?: number;
fadeOut?: number;
speed?: number;
+ speedCurve?: SpeedNode[];
sound?: ClipSound;
duck?: number;
lane?: number;