Skip to content

Open source CapCut alternative.

TypeScript

feat(site): build the graphics engine on the effects-kit seam [rebuild]

David Le9eeffb9c4b67
diff --git a/site/.gitignore b/site/.gitignore
index 8d42ead9..1f68472d 100644
--- a/site/.gitignore
+++ b/site/.gitignore
@@ -54,3 +54,6 @@ next-env.d.ts
# Cut engine build output
/dist/
+
+# MediaPipe wasm runtime, staged from node_modules by postinstall
+/public/mediapipe/wasm/
diff --git a/site/eslint.config.mjs b/site/eslint.config.mjs
index cf4ff092..a31d9c42 100644
--- a/site/eslint.config.mjs
+++ b/site/eslint.config.mjs
@@ -16,6 +16,24 @@ const eslintConfig = defineConfig([
// against workers types, excluded from the site's tsconfig.
"src/cut/worker/cf/**",
]),
+ {
+ // effects-kit is a standalone package: it must build for any host, so it
+ // never reaches into the app's source or aliases.
+ files: ["packages/effects-kit/**/*.ts", "packages/effects-kit/**/*.tsx"],
+ rules: {
+ "no-restricted-imports": [
+ "error",
+ {
+ patterns: [
+ {
+ group: ["@/*", "**/src/cut/*", "next", "next/*"],
+ message: "effects-kit is host-agnostic — no site/src or Next.js imports.",
+ },
+ ],
+ },
+ ],
+ },
+ },
]);
export default eslintConfig;
diff --git a/site/next.config.ts b/site/next.config.ts
index 5dafc69d..de527b67 100644
--- a/site/next.config.ts
+++ b/site/next.config.ts
@@ -3,6 +3,8 @@ import type { NextConfig } from "next";
const nextConfig: NextConfig = {
pageExtensions: ["js", "jsx", "ts", "tsx", "md", "mdx"],
+ // Workspace packages ship TypeScript source; the app build transpiles them.
+ transpilePackages: ["@donkeycut/effects-kit"],
// Cut (the video editor) uploads large media. Two independent limits apply:
// its media route reads req.formData() (a route handler), so it isn't covered
// by serverActions.bodySizeLimit; and src/proxy.ts runs on /api/cut/* on every
diff --git a/site/package-lock.json b/site/package-lock.json
index 2e87ef61..afdc1f04 100644
--- a/site/package-lock.json
+++ b/site/package-lock.json
@@ -9,6 +9,9 @@
"version": "0.1.0",
"hasInstallScript": true,
"license": "Apache-2.0",
+ "workspaces": [
+ "packages/*"
+ ],
"dependencies": {
"@ai-sdk/react": "^4.0.15",
"@anthropic-ai/claude-agent-sdk": "^0.3.201",
@@ -19,10 +22,12 @@
"@better-auth/core": "1.6.11",
"@better-auth/kysely-adapter": "1.6.11",
"@better-auth/utils": "0.4.0",
+ "@donkeycut/effects-kit": "*",
"@elevenlabs/elevenlabs-js": "^2.49.1",
"@google/genai": "^2.6.0",
"@mdx-js/loader": "^3.1.1",
"@mdx-js/react": "^3.1.1",
+ "@mediapipe/tasks-vision": "^1.0.1",
"@next/mdx": "^16.2.6",
"@posthog/react": "^1.10.3",
"@prisma/adapter-pg": "^7.8.0",
@@ -1288,6 +1293,10 @@
"dev": true,
"license": "MIT OR Apache-2.0"
},
+ "node_modules/@donkeycut/effects-kit": {
+ "resolved": "packages/effects-kit",
+ "link": true
+ },
"node_modules/@dotenvx/dotenvx": {
"version": "1.66.0",
"resolved": "https://registry.npmjs.org/@dotenvx/dotenvx/-/dotenvx-1.66.0.tgz",
@@ -2977,6 +2986,12 @@
"react": ">=16"
}
},
+ "node_modules/@mediapipe/tasks-vision": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-1.0.1.tgz",
+ "integrity": "sha512-rvRE2FmAZ6ZxKSw7wq+e+jQDpN3t1B/tD2mJz9SmAzb1msoDkd4dMoE4wAh8Z30Um0PQwLiHr9QtomhmXk3aUQ==",
+ "license": "Apache-2.0"
+ },
"node_modules/@mixmark-io/domino": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
@@ -11356,6 +11371,12 @@
"loose-envify": "cli.js"
}
},
+ "node_modules/lottie-web": {
+ "version": "5.13.0",
+ "resolved": "https://registry.npmjs.org/lottie-web/-/lottie-web-5.13.0.tgz",
+ "integrity": "sha512-+gfBXl6sxXMPe8tKQm7qzLnUy5DUPJPKIyRHwtpCpyUEYjHYRJC/5gjUvdkuO2c3JllrPtHXH5UJJK8LRYl5yQ==",
+ "license": "MIT"
+ },
"node_modules/lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
@@ -16809,6 +16830,14 @@
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
+ },
+ "packages/effects-kit": {
+ "name": "@donkeycut/effects-kit",
+ "version": "0.1.0",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "lottie-web": "^5.13.0"
+ }
}
}
}
diff --git a/site/package.json b/site/package.json
index dd486445..f2dbfdfe 100644
--- a/site/package.json
+++ b/site/package.json
@@ -4,12 +4,15 @@
"private": true,
"homepage": "https://donkeycut.com",
"license": "Apache-2.0",
+ "workspaces": [
+ "packages/*"
+ ],
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "eslint",
- "postinstall": "prisma generate",
+ "postinstall": "prisma generate && node scripts/copy-mediapipe-assets.mjs",
"db:generate": "prisma generate",
"db:pull": "prisma db pull",
"prisma:dev": "dotenv -e .env.development -- npx prisma migrate dev",
@@ -19,7 +22,7 @@
"worker:build": "bun build --target=node --minify --external=pg-native src/cut/worker/main.ts --outfile dist/cut-worker/main.js",
"eval:cut-chat": "bun run scripts/eval-cut-chat.ts",
"email:dev": "email dev --dir src/emails --port 3001",
- "test": "bun test src",
+ "test": "bun test src packages/effects-kit/src",
"test:genvideo": "bun run scripts/genvideo-selftest.ts",
"eval:genvideo": "bun run scripts/genvideo-eval.ts"
},
@@ -33,10 +36,12 @@
"@better-auth/core": "1.6.11",
"@better-auth/kysely-adapter": "1.6.11",
"@better-auth/utils": "0.4.0",
+ "@donkeycut/effects-kit": "*",
"@elevenlabs/elevenlabs-js": "^2.49.1",
"@google/genai": "^2.6.0",
"@mdx-js/loader": "^3.1.1",
"@mdx-js/react": "^3.1.1",
+ "@mediapipe/tasks-vision": "^1.0.1",
"@next/mdx": "^16.2.6",
"@posthog/react": "^1.10.3",
"@prisma/adapter-pg": "^7.8.0",
diff --git a/site/packages/effects-kit/package.json b/site/packages/effects-kit/package.json
new file mode 100644
index 00000000..c5b47e6e
--- /dev/null
+++ b/site/packages/effects-kit/package.json
@@ -0,0 +1,18 @@
+{
+ "name": "@donkeycut/effects-kit",
+ "version": "0.1.0",
+ "description": "Host-agnostic graphics elements for web video editors: overlay model, canvas painters, plugin registry.",
+ "license": "Apache-2.0",
+ "type": "module",
+ "main": "./src/index.ts",
+ "types": "./src/index.ts",
+ "exports": {
+ ".": {
+ "types": "./src/index.ts",
+ "default": "./src/index.ts"
+ }
+ },
+ "dependencies": {
+ "lottie-web": "^5.13.0"
+ }
+}
diff --git a/site/packages/effects-kit/src/anim.ts b/site/packages/effects-kit/src/anim.ts
new file mode 100644
index 00000000..b4422649
--- /dev/null
+++ b/site/packages/effects-kit/src/anim.ts
@@ -0,0 +1,186 @@
+/**
+ * Overlay animation: preset In / Out / Loop slots evaluated by one pure
+ * function. The DOM preview applies the result as a per-frame CSS transform;
+ * the export samples the same function into rasterized frames — a single
+ * evaluator is what keeps the two in lockstep.
+ */
+
+export type OverlayAnimStyle =
+ | "fade"
+ | "pop"
+ | "zoom"
+ | "slideleft"
+ | "slideright"
+ | "slideup"
+ | "slidedown"
+ | "typewriter"; // text only; other kinds render it as a fade
+
+export type OverlayLoopStyle = "pulse" | "wiggle" | "spin" | "float";
+
+export interface OverlayAnim {
+ in?: { style: OverlayAnimStyle; seconds: number };
+ out?: { style: OverlayAnimStyle; seconds: number };
+ /** Runs the element's whole duration; `speed` multiplies the base rate
+ * (1 = default, 2 = twice as fast). */
+ loop?: { style: OverlayLoopStyle; speed: number };
+}
+
+export const OVERLAY_ANIM_STYLE_IDS: OverlayAnimStyle[] = [
+ "fade",
+ "pop",
+ "zoom",
+ "slideleft",
+ "slideright",
+ "slideup",
+ "slidedown",
+ "typewriter",
+];
+
+export const OVERLAY_LOOP_STYLE_IDS: OverlayLoopStyle[] = ["pulse", "wiggle", "spin", "float"];
+
+export const OVERLAY_ANIM_STYLE_LABELS: Record<OverlayAnimStyle, string> = {
+ fade: "Fade",
+ pop: "Pop",
+ zoom: "Zoom",
+ slideleft: "Slide left",
+ slideright: "Slide right",
+ slideup: "Slide up",
+ slidedown: "Slide down",
+ typewriter: "Typewriter",
+};
+
+export const OVERLAY_LOOP_STYLE_LABELS: Record<OverlayLoopStyle, string> = {
+ pulse: "Pulse",
+ wiggle: "Wiggle",
+ spin: "Spin",
+ float: "Float",
+};
+
+/** Ramp length bounds, seconds (mirrors clip animations). */
+export const OVERLAY_ANIM_MIN_SECONDS = 0.1;
+export const OVERLAY_ANIM_MAX_SECONDS = 2;
+export const OVERLAY_ANIM_DEFAULT_SECONDS = 0.5;
+
+/** How far slides travel, in design px at the 1080 short side. */
+export const SLIDE_TRAVEL = 120;
+/** Float bob amplitude, design px. */
+export const FLOAT_TRAVEL = 12;
+/** Base loop periods in seconds at speed 1. */
+export const LOOP_PERIODS: Record<OverlayLoopStyle, number> = {
+ pulse: 1.2,
+ wiggle: 0.9,
+ spin: 4,
+ float: 2.4,
+};
+
+/** The transform an animation contributes at one moment. dx/dy are design px
+ * (1080 short side); rotate is degrees; alpha multiplies the element opacity;
+ * textProgress (0..1, typewriter only) is the share of characters shown. */
+export interface OverlayAnimState {
+ dx: number;
+ dy: number;
+ scale: number;
+ rotate: number;
+ alpha: number;
+ textProgress?: number;
+}
+
+const IDLE: OverlayAnimState = { dx: 0, dy: 0, scale: 1, rotate: 0, alpha: 1 };
+
+const easeOutCubic = (p: number) => 1 - Math.pow(1 - p, 3);
+const easeInCubic = (p: number) => p * p * p;
+const easeOutBack = (p: number) => {
+ const c1 = 1.70158;
+ const c3 = c1 + 1;
+ return 1 + c3 * Math.pow(p - 1, 3) + c1 * Math.pow(p - 1, 2);
+};
+
+/** One edge ramp. `p` runs 0→1 over the window in play direction for "in";
+ * for "out" the caller feeds remaining-share so the motion mirrors. */
+function edgeState(style: OverlayAnimStyle, p: number, exiting: boolean): OverlayAnimState {
+ const clamped = Math.min(1, Math.max(0, p));
+ switch (style) {
+ case "fade":
+ return { ...IDLE, alpha: clamped };
+ case "pop":
+ // Entering overshoots then settles; exiting shrinks away without the
+ // overshoot (a reverse overshoot reads as a stutter).
+ return {
+ ...IDLE,
+ scale: exiting ? easeInCubic(clamped) * 0.4 + 0.6 * clamped : easeOutBack(clamped),
+ alpha: Math.min(1, clamped * 2),
+ };
+ case "zoom":
+ return { ...IDLE, scale: 0.6 + 0.4 * easeOutCubic(clamped), alpha: clamped };
+ case "slideleft":
+ // The picture moves leftward: it enters from the right edge, or exits
+ // off the left one — the exit's offset runs negative.
+ return { ...IDLE, dx: (1 - easeOutCubic(clamped)) * SLIDE_TRAVEL * (exiting ? -1 : 1), alpha: clamped };
+ case "slideright":
+ return { ...IDLE, dx: (1 - easeOutCubic(clamped)) * SLIDE_TRAVEL * (exiting ? 1 : -1), alpha: clamped };
+ case "slideup":
+ return { ...IDLE, dy: (1 - easeOutCubic(clamped)) * SLIDE_TRAVEL * (exiting ? -1 : 1), alpha: clamped };
+ case "slidedown":
+ return { ...IDLE, dy: (1 - easeOutCubic(clamped)) * SLIDE_TRAVEL * (exiting ? 1 : -1), alpha: clamped };
+ case "typewriter":
+ return { ...IDLE, textProgress: clamped };
+ }
+}
+
+/**
+ * Evaluate an element's animation at `tLocal` seconds into its [0, dur]
+ * window. Pure and deterministic — the preview, the export frame renderer,
+ * and the browser compositor all call this same function.
+ */
+export function evalOverlayAnim(
+ anim: OverlayAnim | undefined,
+ tLocal: number,
+ dur: number
+): OverlayAnimState {
+ if (!anim) return IDLE;
+ let state = { ...IDLE };
+
+ const inSecs = anim.in ? Math.min(anim.in.seconds, dur) : 0;
+ const outSecs = anim.out ? Math.min(anim.out.seconds, Math.max(0, dur - inSecs)) : 0;
+
+ if (anim.in && tLocal < inSecs) {
+ state = edgeState(anim.in.style, tLocal / inSecs, false);
+ } else if (anim.out && tLocal > dur - outSecs) {
+ state = edgeState(anim.out.style, (dur - tLocal) / outSecs, true);
+ }
+
+ if (anim.loop) {
+ const speed = anim.loop.speed > 0 ? anim.loop.speed : 1;
+ const period = LOOP_PERIODS[anim.loop.style] / speed;
+ const phase = (tLocal % period) / period;
+ const wave = Math.sin(phase * Math.PI * 2);
+ switch (anim.loop.style) {
+ case "pulse":
+ state.scale *= 1 + 0.06 * wave;
+ break;
+ case "wiggle":
+ state.rotate += 4 * wave;
+ break;
+ case "spin":
+ state.rotate += phase * 360;
+ break;
+ case "float":
+ state.dy += FLOAT_TRAVEL * wave;
+ break;
+ }
+ }
+ return state;
+}
+
+/** The loop's exact cycle length in seconds (frame sequences render one cycle
+ * and repeat it), or null when the element has no loop. */
+export function loopPeriod(anim: OverlayAnim | undefined): number | null {
+ if (!anim?.loop) return null;
+ const speed = anim.loop.speed > 0 ? anim.loop.speed : 1;
+ return LOOP_PERIODS[anim.loop.style] / speed;
+}
+
+/** Whether any slot is set (an element with an empty anim object is static). */
+export function hasOverlayAnim(anim: OverlayAnim | undefined): boolean {
+ return !!anim && (!!anim.in || !!anim.out || !!anim.loop);
+}
diff --git a/site/src/cut/lib/colorGrade.test.ts b/site/packages/effects-kit/src/colorGrade.test.ts
similarity index 100%
rename from site/src/cut/lib/colorGrade.test.ts
rename to site/packages/effects-kit/src/colorGrade.test.ts
diff --git a/site/src/cut/lib/colorGrade.ts b/site/packages/effects-kit/src/colorGrade.ts
similarity index 93%
rename from site/src/cut/lib/colorGrade.ts
rename to site/packages/effects-kit/src/colorGrade.ts
index 910b8e35..b23a4be4 100644
--- a/site/src/cut/lib/colorGrade.ts
+++ b/site/packages/effects-kit/src/colorGrade.ts
@@ -1,5 +1,17 @@
-import { GRADE_HUE_MAX, GRADE_MAX } from "./types";
-import type { ColorGrade } from "./types";
+/** Per-clip manual color adjustments. Integer sliders, 0 = neutral; the whole
+ * field is absent when every value is 0, so untouched clips carry nothing. */
+export interface ColorGrade {
+ brightness?: number; // -50..50
+ contrast?: number; // -50..50
+ saturation?: number; // -50..50
+ exposure?: number; // -50..50 (±1 EV)
+ temperature?: number; // -50..50, positive = warm
+ hue?: number; // -180..180 degrees
+}
+
+/** Slider range for every grade parameter except hue. */
+export const GRADE_MAX = 50;
+export const GRADE_HUE_MAX = 180;
/**
* One mapping, two renderers. The preview canvas and the export filtergraph
diff --git a/site/packages/effects-kit/src/cutout.test.ts b/site/packages/effects-kit/src/cutout.test.ts
new file mode 100644
index 00000000..6d1eecbe
--- /dev/null
+++ b/site/packages/effects-kit/src/cutout.test.ts
@@ -0,0 +1,208 @@
+import { describe, expect, test } from "bun:test";
+import { flatBackdropAlpha, keepLargestSubject } from "./cutout";
+
+/** A picture with a flat backdrop and a solid block in the middle. */
+function scene(
+ w: number,
+ h: number,
+ bg: [number, number, number],
+ subject: [number, number, number],
+ box: { x0: number; y0: number; x1: number; y1: number },
+ noise = 0
+): Uint8ClampedArray {
+ const data = new Uint8ClampedArray(w * h * 4);
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ const inside = x >= box.x0 && x < box.x1 && y >= box.y0 && y < box.y1;
+ const c = inside ? subject : bg;
+ // Deterministic dither, so "a slightly noisy backdrop" is reproducible.
+ const n = inside ? 0 : ((x * 7 + y * 13) % 3) * noise;
+ const i = (y * w + x) * 4;
+ data[i] = c[0] + n;
+ data[i + 1] = c[1] + n;
+ data[i + 2] = c[2] + n;
+ data[i + 3] = 255;
+ }
+ }
+ return data;
+}
+
+const at = (alpha: Float32Array, w: number, x: number, y: number) => alpha[y * w + x];
+
+describe("flat backdrop knockout", () => {
+ test("drops the backdrop and keeps the subject", () => {
+ const w = 40;
+ const h = 40;
+ const data = scene(w, h, [208, 208, 208], [220, 120, 40], { x0: 12, y0: 12, x1: 28, y1: 28 });
+ const alpha = flatBackdropAlpha(data, w, h)!;
+ expect(alpha).not.toBe(null);
+ expect(at(alpha, w, 1, 1)).toBe(0); // corner: backdrop
+ expect(at(alpha, w, 20, 20)).toBe(1); // middle: subject
+ });
+
+ test("a light subject on a light backdrop survives, edge included", () => {
+ // The case that matters for stickers: a white outline against a gray
+ // sweep. A generous threshold would eat the outline and then the subject.
+ const w = 40;
+ const h = 40;
+ const data = scene(w, h, [208, 208, 208], [255, 255, 255], { x0: 12, y0: 12, x1: 28, y1: 28 });
+ const alpha = flatBackdropAlpha(data, w, h)!;
+ expect(at(alpha, w, 20, 20)).toBe(1);
+ expect(at(alpha, w, 12, 20)).toBe(1); // the outermost white pixel stays
+ expect(at(alpha, w, 1, 20)).toBe(0);
+ });
+
+ test("backdrop-coloured pixels inside the subject are kept", () => {
+ // The fill spreads from the border, so an enclosed pocket of backdrop
+ // colour — a gray eye on a gray-free face — is never punched out.
+ const w = 40;
+ const h = 40;
+ const data = scene(w, h, [40, 40, 40], [230, 90, 30], { x0: 8, y0: 8, x1: 32, y1: 32 });
+ for (const [x, y] of [
+ [19, 19],
+ [19, 20],
+ [20, 19],
+ [20, 20],
+ ]) {
+ const i = (y * w + x) * 4;
+ data[i] = 40;
+ data[i + 1] = 40;
+ data[i + 2] = 40;
+ }
+ const alpha = flatBackdropAlpha(data, w, h)!;
+ expect(at(alpha, w, 20, 20)).toBe(1);
+ });
+
+ test("a noisy backdrop still reads as flat", () => {
+ const w = 40;
+ const h = 40;
+ const data = scene(w, h, [200, 200, 200], [10, 90, 200], { x0: 14, y0: 14, x1: 26, y1: 26 }, 4);
+ const alpha = flatBackdropAlpha(data, w, h)!;
+ expect(alpha).not.toBe(null);
+ expect(at(alpha, w, 1, 1)).toBeLessThan(0.5);
+ expect(at(alpha, w, 20, 20)).toBe(1);
+ });
+
+ test("a backdrop that drifts across the frame is still one backdrop", () => {
+ // What a generator actually paints when asked for a solid sweep: a slow
+ // ramp corner to corner. Measured against one global colour the far side
+ // reads as a subject; measured against the colour beside it, it does not.
+ const w = 64;
+ const h = 64;
+ const data = new Uint8ClampedArray(w * h * 4);
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ const inside = x >= 24 && x < 40 && y >= 24 && y < 40;
+ const ramp = 150 + Math.round(((x + y) / (w + h - 2)) * 70); // 150 → 220
+ const i = (y * w + x) * 4;
+ data[i] = inside ? 250 : ramp;
+ data[i + 1] = inside ? 250 : ramp;
+ data[i + 2] = inside ? 250 : ramp;
+ data[i + 3] = 255;
+ }
+ }
+ const alpha = flatBackdropAlpha(data, w, h)!;
+ expect(alpha).not.toBe(null);
+ expect(at(alpha, w, 1, 1)).toBe(0); // dark corner
+ expect(at(alpha, w, w - 2, h - 2)).toBe(0); // light corner, 70 off the other
+ expect(at(alpha, w, 32, 32)).toBe(1); // the subject survives both
+ });
+
+ test("no flat backdrop, no answer", () => {
+ const w = 40;
+ const h = 40;
+ // A photograph's border wanders; the guess would be wrong, so it declines
+ // and a real matte gets asked instead.
+ const data = new Uint8ClampedArray(w * h * 4);
+ for (let i = 0; i < w * h; i++) {
+ data[i * 4] = (i * 37) % 255;
+ data[i * 4 + 1] = (i * 53) % 255;
+ data[i * 4 + 2] = (i * 91) % 255;
+ data[i * 4 + 3] = 255;
+ }
+ expect(flatBackdropAlpha(data, w, h) === null).toBe(true);
+ });
+
+ test("a picture that is all backdrop declines too", () => {
+ const w = 24;
+ const h = 24;
+ const data = scene(w, h, [128, 128, 128], [128, 128, 128], { x0: 0, y0: 0, x1: 0, y1: 0 });
+ expect(flatBackdropAlpha(data, w, h) === null).toBe(true);
+ });
+});
+
+/** A stand-in canvas: `keepLargestSubject` only ever reads and writes alpha. */
+function alphaCanvas(w: number, h: number, opaque: (x: number, y: number) => boolean) {
+ const data = new Uint8ClampedArray(w * h * 4);
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ const i = (y * w + x) * 4;
+ data[i] = data[i + 1] = data[i + 2] = 255;
+ data[i + 3] = opaque(x, y) ? 255 : 0;
+ }
+ }
+ const image = { data, width: w, height: h };
+ const ctx = {
+ getImageData: () => image,
+ putImageData: () => {},
+ } as unknown as CanvasRenderingContext2D;
+ const canvas = { width: w, height: h, getContext: () => ctx } as unknown as HTMLCanvasElement;
+ return { canvas, alphaAt: (x: number, y: number) => data[(y * w + x) * 4 + 3] };
+}
+
+describe("keep the largest subject", () => {
+ test("drops the strokes and specks around a sticker", () => {
+ const w = 80;
+ const h = 80;
+ const body = (x: number, y: number) => x >= 24 && x < 56 && y >= 24 && y < 56;
+ // A hairline stroke arcing past the body and a speck off in the corner —
+ // what a generator scribbles around what it was asked for.
+ const stroke = (x: number, y: number) => y === 12 && x >= 10 && x < 70;
+ const speck = (x: number, y: number) => x >= 70 && x < 73 && y >= 70 && y < 73;
+ const { canvas, alphaAt } = alphaCanvas(
+ w,
+ h,
+ (x, y) => body(x, y) || stroke(x, y) || speck(x, y)
+ );
+ expect(keepLargestSubject(canvas)).toBe(true);
+ expect(alphaAt(40, 40)).toBe(255); // the sticker stays
+ expect(alphaAt(30, 12)).toBe(0); // the stroke goes
+ expect(alphaAt(71, 71)).toBe(0); // the speck goes
+ });
+
+ test("a stroke touching the sticker goes with the rest", () => {
+ // Erosion is what makes this separable: the thread is thinner than the
+ // radius, so it parts from the body before the regions are counted.
+ const w = 80;
+ const h = 80;
+ const body = (x: number, y: number) => x >= 20 && x < 60 && y >= 20 && y < 60;
+ const tail = (x: number, y: number) => y === 40 && x >= 60 && x < 78;
+ const { canvas, alphaAt } = alphaCanvas(w, h, (x, y) => body(x, y) || tail(x, y));
+ expect(keepLargestSubject(canvas)).toBe(true);
+ expect(alphaAt(40, 40)).toBe(255);
+ expect(alphaAt(74, 40)).toBe(0);
+ });
+
+ test("one clean subject is left alone", () => {
+ const w = 60;
+ const h = 60;
+ const { canvas, alphaAt } = alphaCanvas(
+ w,
+ h,
+ (x, y) => x >= 15 && x < 45 && y >= 15 && y < 45
+ );
+ expect(keepLargestSubject(canvas)).toBe(false);
+ expect(alphaAt(30, 30)).toBe(255);
+ expect(alphaAt(16, 16)).toBe(255); // the true edge is never trimmed
+ });
+
+ test("an all-hairline picture is left alone", () => {
+ // Nothing survives the erosion, so there is no body to judge strays
+ // against; better to ship the matte as it came than to blank it.
+ const w = 60;
+ const h = 60;
+ const { canvas, alphaAt } = alphaCanvas(w, h, (x, y) => y % 10 === 0 && x > 4 && x < 56);
+ expect(keepLargestSubject(canvas)).toBe(false);
+ expect(alphaAt(30, 10)).toBe(255);
+ });
+});
diff --git a/site/packages/effects-kit/src/cutout.ts b/site/packages/effects-kit/src/cutout.ts
new file mode 100644
index 00000000..685d48b8
--- /dev/null
+++ b/site/packages/effects-kit/src/cutout.ts
@@ -0,0 +1,362 @@
+/**
+ * Cutout finishing: the canvas work that turns a matted picture into a
+ * sticker. Where the alpha came from — an on-device segmenter, a hosted vision
+ * model, a hand-drawn selection — is the host's business; everything here just
+ * shapes pixels, so any host with a canvas can reuse it.
+ */
+
+/** Die-cut outline radius as a share of the sticker's long side. */
+export const OUTLINE_SHARE = 0.03;
+
+/** Number of stamps around the circle that make the outline read as smooth. */
+const OUTLINE_STAMPS = 24;
+
+/**
+ * Knock out a plain backdrop.
+ *
+ * A generated sticker arrives on a plain studio background, and no segmenter
+ * is needed to see it: the border tells you the colour, and the subject is
+ * whatever the colour does not reach. The fill spreads inward from the edges,
+ * so a patch of backdrop-coloured pixels *inside* the subject — a white shirt
+ * against a white sweep — is kept rather than punched out.
+ *
+ * Each pixel is judged against the backdrop colour beside it, carried along by
+ * the fill, so a backdrop that drifts across the frame stays backdrop the whole
+ * way. An image generator paints a "solid" sweep with a gradient and a little
+ * noise in it, and one global colour would call the far corner a subject.
+ * Drifting is bounded: a pixel far enough from the border colour is a subject
+ * however smoothly it was reached, so a soft shadow cannot leak into the cat.
+ *
+ * Returns per-pixel coverage (1 keeps, 0 drops, between the two for the soft
+ * edge), or null when the picture is not one of these: a chaotic border, or a
+ * fill that swallows nearly everything, means the guess was wrong and a real
+ * matte should answer instead.
+ */
+export interface FlatBackdropOptions {
+ /** Colour distance from the neighbouring backdrop that is still backdrop
+ * (0-255, per channel). Default: scaled to how much the border itself
+ * varies. */
+ near?: number;
+ /** Distance past which a pixel is certainly subject; between the two the
+ * edge fades, which is what keeps the cutout from looking cut with pinking
+ * shears. */
+ far?: number;
+ /** How far the fill may drift from the border colour in total. */
+ maxDrift?: number;
+ /** How much the border may vary and still count as a backdrop at all. */
+ borderSpread?: number;
+}
+
+export function flatBackdropAlpha(
+ data: Uint8ClampedArray,
+ w: number,
+ h: number,
+ opts: FlatBackdropOptions = {}
+): Float32Array | null {
+ if (w < 4 || h < 4) return null;
+
+ // The backdrop colour: the median of the border ring, which shrugs off a
+ // subject that runs off one edge.
+ const border: number[][] = [];
+ const at = (x: number, y: number) => (y * w + x) * 4;
+ for (let x = 0; x < w; x++) {
+ for (const y of [0, h - 1]) border.push([data[at(x, y)], data[at(x, y) + 1], data[at(x, y) + 2]]);
+ }
+ for (let y = 1; y < h - 1; y++) {
+ for (const x of [0, w - 1]) border.push([data[at(x, y)], data[at(x, y) + 1], data[at(x, y) + 2]]);
+ }
+ const median = (values: number[]) => {
+ const sorted = [...values].sort((a, b) => a - b);
+ return sorted[sorted.length >> 1];
+ };
+ const gap = (p: number[], r: number, g: number, b: number) =>
+ Math.max(Math.abs(p[0] - r), Math.abs(p[1] - g), Math.abs(p[2] - b));
+ const bg = [0, 1, 2].map((c) => median(border.map((p) => p[c])));
+
+ // How ragged the border is, as its own typical departure from that median.
+ // Half the ring can sit on a gradient's dark end without being ragged, so
+ // the median of the departures answers where a share-of-pixels count would
+ // call a smooth sweep chaotic.
+ const ragged = median(border.map((p) => gap(p, bg[0], bg[1], bg[2])));
+ // A photographed wall wanders pixel to pixel; a rendered backdrop does not.
+ if (ragged > (opts.borderSpread ?? 24)) return null;
+
+ const near = opts.near ?? Math.max(10, Math.min(30, ragged * 2 + 8));
+ const far = opts.far ?? near + 18;
+ const maxDrift = opts.maxDrift ?? Math.max(72, far * 2);
+
+ const alpha = new Float32Array(w * h).fill(1);
+ const seen = new Uint8Array(w * h);
+ // The backdrop colour beside each filled pixel, carried inward so a drifting
+ // sweep is measured against itself.
+ const ref = new Uint8Array(w * h * 3);
+ const stack: number[] = [];
+
+ /** Take pixel `i` against a backdrop colour, and spread from it when it is
+ * wholly backdrop. An edge pixel keeps its part-coverage and spreads no
+ * further, so a soft rim cannot walk the fill into the subject. */
+ const take = (i: number, r: number, g: number, b: number) => {
+ if (seen[i]) return;
+ seen[i] = 1;
+ const p = [data[i * 4], data[i * 4 + 1], data[i * 4 + 2]];
+ if (gap(p, bg[0], bg[1], bg[2]) > maxDrift) return; // too far from where it started
+ const d = gap(p, r, g, b);
+ if (d >= far) return; // subject: the fill stops here
+ if (d > near) {
+ alpha[i] = (d - near) / (far - near);
+ return;
+ }
+ alpha[i] = 0;
+ ref[i * 3] = p[0];
+ ref[i * 3 + 1] = p[1];
+ ref[i * 3 + 2] = p[2];
+ stack.push(i);
+ };
+
+ for (let x = 0; x < w; x++) {
+ take(x, bg[0], bg[1], bg[2]);
+ take((h - 1) * w + x, bg[0], bg[1], bg[2]);
+ }
+ for (let y = 0; y < h; y++) {
+ take(y * w, bg[0], bg[1], bg[2]);
+ take(y * w + w - 1, bg[0], bg[1], bg[2]);
+ }
+ while (stack.length) {
+ const i = stack.pop()!;
+ const r = ref[i * 3];
+ const g = ref[i * 3 + 1];
+ const b = ref[i * 3 + 2];
+ const x = i % w;
+ const y = (i / w) | 0;
+ if (x > 0) take(i - 1, r, g, b);
+ if (x < w - 1) take(i + 1, r, g, b);
+ if (y > 0) take(i - w, r, g, b);
+ if (y < h - 1) take(i + w, r, g, b);
+ }
+
+ let dropped = 0;
+ for (let i = 0; i < alpha.length; i++) if (alpha[i] < 0.5) dropped++;
+ const share = dropped / alpha.length;
+ // Too little and there was no backdrop to speak of; too much and the
+ // subject went with it.
+ if (share < 0.08 || share > 0.94) return null;
+ return alpha;
+}
+
+/** Apply `flatBackdropAlpha` to a canvas in place. False when the picture has
+ * no flat backdrop to knock out — the caller falls through to a real matte. */
+export function removeFlatBackdrop(
+ canvas: HTMLCanvasElement,
+ opts?: FlatBackdropOptions
+): boolean {
+ const ctx = canvas.getContext("2d", { willReadFrequently: true })!;
+ const { width: w, height: h } = canvas;
+ const image = ctx.getImageData(0, 0, w, h);
+ const alpha = flatBackdropAlpha(image.data, w, h, opts);
+ if (!alpha) return false;
+ for (let i = 0; i < alpha.length; i++) {
+ image.data[i * 4 + 3] = Math.round(image.data[i * 4 + 3] * alpha[i]);
+ }
+ ctx.putImageData(image, 0, 0);
+ return true;
+}
+
+/** Square min/max filters, run as two 1D passes. A square structuring element
+ * separates, so an r-pixel erode costs two scans rather than r². */
+function boxFilter(src: Uint8Array, w: number, h: number, r: number, want: 0 | 1): Uint8Array {
+ const pick = (a: number, b: number) => (want === 1 ? (a > b ? a : b) : a < b ? a : b);
+ const mid = new Uint8Array(w * h);
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ let v = src[y * w + x];
+ for (let d = -r; d <= r; d++) {
+ const nx = x + d;
+ if (nx >= 0 && nx < w) v = pick(v, src[y * w + nx]);
+ }
+ mid[y * w + x] = v;
+ }
+ }
+ const out = new Uint8Array(w * h);
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ let v = mid[y * w + x];
+ for (let d = -r; d <= r; d++) {
+ const ny = y + d;
+ if (ny >= 0 && ny < h) v = pick(v, mid[ny * w + x]);
+ }
+ out[y * w + x] = v;
+ }
+ }
+ return out;
+}
+
+/**
+ * Keep the sticker and drop what is loitering around it.
+ *
+ * A generator asked for a die-cut sticker draws one — and then sketches stray
+ * strokes and specks around it, which a backdrop knockout rightly keeps: they
+ * are not the backdrop colour. They are, though, the only marks not part of
+ * the sticker's one solid body.
+ *
+ * Eroding the matte first is what makes that separable. Hairline strokes are
+ * thinner than the erosion radius, so they vanish or pull away from the body,
+ * and the body — a filled silhouette with its outline around it — survives.
+ * The largest surviving region is the sticker; dilating it back and taking it
+ * against the original alpha restores the true edge without ever adding a
+ * pixel the matte had not already kept.
+ *
+ * A picture whose subject is genuinely several pieces would lose all but the
+ * biggest, so this is the sticker pipeline's step rather than every matte's.
+ */
+export function keepLargestSubject(
+ canvas: HTMLCanvasElement,
+ opts: { radius?: number } = {}
+): boolean {
+ const ctx = canvas.getContext("2d", { willReadFrequently: true })!;
+ const { width: w, height: h } = canvas;
+ if (w < 8 || h < 8) return false;
+ const image = ctx.getImageData(0, 0, w, h);
+ const data = image.data;
+ // Thin enough to leave the sticker's body untouched, thick enough to bite
+ // through a sketch line.
+ const r = Math.max(1, opts.radius ?? Math.round(Math.min(w, h) * 0.006));
+
+ const solid = new Uint8Array(w * h);
+ let solidCount = 0;
+ for (let i = 0; i < w * h; i++) {
+ if (data[i * 4 + 3] > 127) {
+ solid[i] = 1;
+ solidCount++;
+ }
+ }
+ if (solidCount === 0) return false;
+
+ const eroded = boxFilter(solid, w, h, r, 0);
+
+ // Largest connected region of what survived the erosion, 4-connectivity —
+ // a diagonal thread of pixels is exactly the kind of join worth breaking.
+ const seen = new Uint8Array(w * h);
+ let best: number[] = [];
+ const stack: number[] = [];
+ for (let start = 0; start < w * h; start++) {
+ if (!eroded[start] || seen[start]) continue;
+ const region: number[] = [];
+ seen[start] = 1;
+ stack.push(start);
+ while (stack.length) {
+ const i = stack.pop()!;
+ region.push(i);
+ const x = i % w;
+ const y = (i / w) | 0;
+ const push = (j: number) => {
+ if (!seen[j] && eroded[j]) {
+ seen[j] = 1;
+ stack.push(j);
+ }
+ };
+ if (x > 0) push(i - 1);
+ if (x < w - 1) push(i + 1);
+ if (y > 0) push(i - w);
+ if (y < h - 1) push(i + w);
+ }
+ if (region.length > best.length) best = region;
+ }
+ // Erosion ate everything: the matte is all thin work, and there is no body
+ // to tell the strays apart from.
+ if (best.length === 0) return false;
+
+ const body = new Uint8Array(w * h);
+ for (const i of best) body[i] = 1;
+ const keep = boxFilter(body, w, h, r, 1);
+
+ let dropped = 0;
+ for (let i = 0; i < w * h; i++) {
+ if (solid[i] && !keep[i]) dropped++;
+ if (!keep[i]) data[i * 4 + 3] = 0;
+ }
+ if (dropped === 0) return false;
+ ctx.putImageData(image, 0, 0);
+ return true;
+}
+
+/** A blank canvas and its 2D context, set up for pixel reads. */
+export function canvasOf(w: number, h: number): [HTMLCanvasElement, CanvasRenderingContext2D] {
+ const c = document.createElement("canvas");
+ c.width = w;
+ c.height = h;
+ return [c, c.getContext("2d", { willReadFrequently: true })!];
+}
+
+/**
+ * Scale every pixel's alpha by `alphaAt`, which reads a pixel index and
+ * answers how much of the subject is there (0 = background, 1 = solid).
+ * Returns whether enough survived to be a subject at all, so a caller can
+ * treat "the matte found nothing" as its own outcome rather than shipping a
+ * blank sticker.
+ */
+export function applyAlpha(
+ ctx: CanvasRenderingContext2D,
+ w: number,
+ h: number,
+ alphaAt: (i: number) => number
+): boolean {
+ const data = ctx.getImageData(0, 0, w, h);
+ let kept = 0;
+ for (let i = 0; i < w * h; i++) {
+ const a = alphaAt(i);
+ data.data[i * 4 + 3] = Math.round(data.data[i * 4 + 3] * a);
+ if (a > 0.5) kept++;
+ }
+ ctx.putImageData(data, 0, 0);
+ return kept > w * h * 0.005;
+}
+
+/** Crop a canvas to its non-transparent content plus a small margin. Returns
+ * the canvas untouched when nothing is left to crop to. */
+export function cropToContent(canvas: HTMLCanvasElement): HTMLCanvasElement {
+ const ctx = canvas.getContext("2d", { willReadFrequently: true })!;
+ const { width: w, height: h } = canvas;
+ const data = ctx.getImageData(0, 0, w, h).data;
+ let x0 = w,
+ y0 = h,
+ x1 = 0,
+ y1 = 0;
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ if (data[(y * w + x) * 4 + 3] > 8) {
+ if (x < x0) x0 = x;
+ if (x > x1) x1 = x;
+ if (y < y0) y0 = y;
+ if (y > y1) y1 = y;
+ }
+ }
+ }
+ if (x1 <= x0 || y1 <= y0) return canvas;
+ const margin = Math.round(Math.max(x1 - x0, y1 - y0) * 0.04) + 2;
+ x0 = Math.max(0, x0 - margin);
+ y0 = Math.max(0, y0 - margin);
+ x1 = Math.min(w - 1, x1 + margin);
+ y1 = Math.min(h - 1, y1 + margin);
+ const [out, octx] = canvasOf(x1 - x0 + 1, y1 - y0 + 1);
+ octx.drawImage(canvas, -x0, -y0);
+ return out;
+}
+
+/** The classic die-cut look: the cutout's white silhouette stamped at radial
+ * offsets beneath it, so a smooth outline hugs the shape. */
+export function withOutline(canvas: HTMLCanvasElement): HTMLCanvasElement {
+ const r = Math.max(3, Math.round(Math.max(canvas.width, canvas.height) * OUTLINE_SHARE));
+ const [out, ctx] = canvasOf(canvas.width + r * 2, canvas.height + r * 2);
+ // A white-filled silhouette of the cutout.
+ const [sil, sctx] = canvasOf(canvas.width, canvas.height);
+ sctx.drawImage(canvas, 0, 0);
+ sctx.globalCompositeOperation = "source-in";
+ sctx.fillStyle = "#FFFFFF";
+ sctx.fillRect(0, 0, sil.width, sil.height);
+ for (let k = 0; k < OUTLINE_STAMPS; k++) {
+ const a = (k / OUTLINE_STAMPS) * Math.PI * 2;
+ ctx.drawImage(sil, r + Math.cos(a) * r, r + Math.sin(a) * r);
+ }
+ ctx.drawImage(canvas, r, r);
+ return out;
+}
diff --git a/site/packages/effects-kit/src/effects.test.ts b/site/packages/effects-kit/src/effects.test.ts
new file mode 100644
index 00000000..c01841f4
--- /dev/null
+++ b/site/packages/effects-kit/src/effects.test.ts
@@ -0,0 +1,139 @@
+import { describe, expect, test } from "bun:test";
+import { EFFECT_IDS, effectFilterLines, effectPreviewState } from "./effects";
+
+// The bundled engine ffmpeg is LGPL: a recipe reaching for a GPL-only filter
+// renders on a dev machine (Homebrew ffmpeg) and fails in the shipped app.
+const LGPL_FILTERS = new Set([
+ "noise",
+ "rgbashift",
+ "eq",
+ "gblur",
+ "vignette",
+ "scale",
+ "crop",
+ "split",
+ "overlay",
+ // The graded looks are effects too, and their chains reach further.
+ "format",
+ "blend",
+ "lutyuv",
+ "colorchannelmixer",
+ "colorbalance",
+ "curves",
+ "hue",
+ "huesaturation",
+ "vibrance",
+ "unsharp",
+ "colortemperature",
+]);
+
+const filterNamesOf = (lines: string[]): string[] => {
+ const names: string[] = [];
+ for (const line of lines) {
+ // Strip labels and quoted expressions, then read each chain step's name.
+ const bare = line.replace(/\[[^\]]*\]/g, "").replace(/'[^']*'/g, "''");
+ for (const step of bare.split(",")) {
+ const name = step.split("=")[0].trim();
+ if (name) names.push(name);
+ }
+ }
+ return names;
+};
+
+describe("effect recipes", () => {
+ test("every effect exports through the LGPL-safe filter set", () => {
+ for (const id of EFFECT_IDS) {
+ const lines = effectFilterLines("in", "out", id, 0.7, 1, 4, 1080, 1920, "t");
+ expect(lines).not.toBe(null);
+ for (const name of filterNamesOf(lines!)) {
+ expect(LGPL_FILTERS.has(name)).toBe(true);
+ }
+ }
+ });
+
+ test("fragments are time-gated to their window", () => {
+ for (const id of EFFECT_IDS) {
+ const lines = effectFilterLines("in", "out", id, 0.5, 2.5, 6, 1080, 1920, "t")!;
+ const joined = lines.join("\n");
+ expect(joined.includes("gte(t,2.5)") || joined.includes("sin(t*")).toBe(true);
+ }
+ });
+
+ test("unknown ids return null instead of breaking the graph", () => {
+ expect(effectFilterLines("in", "out", "nope", 1, 0, 1, 1080, 1920, "t")).toBe(null);
+ });
+
+ test("time-varying expressions are evaluated per frame, not at init", () => {
+ // `eq` reads its expressions once at init by default, which pins a decay
+ // at its t=0 peak for the whole window.
+ const flash = effectFilterLines("in", "out", "flash", 1, 2.5, 6, 1920, 1080, "t")![0];
+ expect(flash.includes("exp(")).toBe(true);
+ expect(flash.includes("eval=frame")).toBe(true);
+ });
+
+ test("shake reads the same clock and travel as its canvas twin", () => {
+ const start = 2.5;
+ const lines = effectFilterLines("in", "out", "shake", 1, start, 6, 1920, 1080, "t")!;
+ const crop = lines.find((l) => l.includes("crop="))!;
+ // Element-local time, so the jitter lands on the frames the preview showed.
+ expect(crop.includes(`sin((t-${start})*33)`)).toBe(true);
+ expect(crop.includes("sin(t*33)")).toBe(false);
+ // …and the same travel: 22 design px at a 1080 short side.
+ const amp = Number(/\+([\d.]+)\*sin/.exec(crop)![1]);
+ const peak = Math.max(
+ ...Array.from({ length: 400 }, (_, i) => effectPreviewState("shake", 1, i / 400).dx ?? 0)
+ );
+ expect(Math.abs(amp - peak)).toBeLessThan(0.5);
+ });
+
+ test("a zoom lands the export on the point the preview holds", () => {
+ const focus = { x: 0.25, y: 0.8 };
+ const st = effectPreviewState("zoom", 1, 9, focus, 0);
+ expect(st.origin).toEqual(focus);
+ // Cut straight in: the copy is scaled once and cropped back to frame size.
+ const lines = effectFilterLines("in", "out", "zoom", 1, 0, 4, 1920, 1080, "t", focus, 0)!;
+ const line = lines.find((l) => l.includes("crop="))!;
+ const [zw, zh] = /scale=(\d+):(\d+)/.exec(line)!.slice(1).map(Number);
+ const [cx, cy] = /crop=1920:1080:(\d+):(\d+)/.exec(line)!.slice(1).map(Number);
+ expect(zw / 1920).toBeCloseTo(st.zoom!, 2);
+ // The crop sits the same fraction into the overscaled picture as the point
+ // sits into the frame, which is what holds it still.
+ expect(cx / (zw - 1920)).toBeCloseTo(focus.x, 2);
+ expect(cy / (zh - 1080)).toBeCloseTo(focus.y, 2);
+ });
+
+ test("a ramped zoom holds its focus while the picture grows into it", () => {
+ const focus = { x: 0.25, y: 0.8 };
+ const lines = effectFilterLines("in", "out", "zoom", 1, 2.5, 6, 1920, 1080, "t", focus, 0.8)!;
+ const over = lines.find((l) => l.includes("overlay="))!;
+ // The copy is placed by its live size, offset by the focus fractions, so
+ // the point stays put through the whole ramp.
+ expect(over.includes("x='-(w-W)*0.25'")).toBe(true);
+ expect(over.includes("y='-(h-H)*0.8'")).toBe(true);
+ // Element-local clock, so the ramp starts where the element does.
+ expect(lines.join("\n").includes("(t-2.5)/0.8")).toBe(true);
+ });
+
+ test("design px scale with the short side, so a vertical frame matches", () => {
+ const wide = effectFilterLines("in", "out", "blur", 1, 0, 4, 1920, 1080, "t")![0];
+ const tall = effectFilterLines("in", "out", "blur", 1, 0, 4, 1080, 1920, "t")![0];
+ expect(/sigma=([\d.]+)/.exec(wide)![1]).toBe(/sigma=([\d.]+)/.exec(tall)![1]);
+ });
+
+ test("preview state is deterministic in local time", () => {
+ for (const id of EFFECT_IDS) {
+ const a = effectPreviewState(id, 0.6, 1.234);
+ const b = effectPreviewState(id, 0.6, 1.234);
+ expect(a).toEqual(b);
+ }
+ });
+
+ test("flash decays from its start and shake stays bounded", () => {
+ const early = effectPreviewState("flash", 1, 0.05).flash!;
+ const late = effectPreviewState("flash", 1, 0.5).flash!;
+ expect(early).toBeGreaterThan(late);
+ const s = effectPreviewState("shake", 1, 0.37);
+ expect(Math.abs(s.dx ?? 0)).toBeLessThanOrEqual(22);
+ expect(Math.abs(s.dy ?? 0)).toBeLessThanOrEqual(22);
+ });
+});
diff --git a/site/packages/effects-kit/src/effects.ts b/site/packages/effects-kit/src/effects.ts
new file mode 100644
index 00000000..746c52e5
--- /dev/null
+++ b/site/packages/effects-kit/src/effects.ts
@@ -0,0 +1,561 @@
+/**
+ * Time-ranged visual effects as first-class overlay elements — the placeable
+ * cousins of looks: where a look grades one clip, an effect is a moment on
+ * the timeline that filters whatever video is under it. Each effect is one
+ * recipe with two renderers driven by the same `amount` knob: a canvas pass
+ * for the preview/in-tab render, and an ffmpeg fragment (LGPL-safe filters
+ * only: eq, gblur, noise, vignette, rgbashift, scale/crop, overlay) for the
+ * server pipeline.
+ */
+
+import type { OverlayBase } from "./types";
+import { LOOK_LABELS, lookCssFilter, lookFilterLines, lookPost, type LookStyle } from "./looks";
+
+export type EffectId =
+ | "zoom"
+ | "grain"
+ | "vhs"
+ | "glitch"
+ | "blur"
+ | "vignette"
+ | "lightleak"
+ | "flash"
+ | "shake"
+ /** The graded looks, placeable like any other effect. */
+ | "vintage"
+ | "horror"
+ | "halation"
+ | "tech"
+ | "noir"
+ | "pastel"
+ | "blockbuster"
+ | "dreamy";
+
+export interface EffectOverlay extends OverlayBase {
+ kind: "effect";
+ effect: EffectId;
+ /** Strength 0..1; absent = 0.5. */
+ amount?: number;
+ /** The point of the frame the zoom holds on, in frame fractions; absent =
+ * the middle. */
+ focus?: { x: number; y: number };
+ /** Seconds a zoom takes to reach its depth; absent = `ZOOM_RAMP_DEFAULT`,
+ * 0 = there on the first frame. */
+ ramp?: number;
+ /** Reserved: a sub-frame region (fractions); absent = full frame. */
+ region?: { w: number; h: number };
+}
+
+export const EFFECT_IDS: EffectId[] = [
+ "zoom",
+ "grain",
+ "vhs",
+ "glitch",
+ "blur",
+ "vignette",
+ "lightleak",
+ "flash",
+ "shake",
+ "vintage",
+ "horror",
+ "halation",
+ "tech",
+ "noir",
+ "pastel",
+ "blockbuster",
+ "dreamy",
+];
+
+/**
+ * The effects whose recipe is a graded look. A look was once a property of one
+ * clip; as an effect it is a stretch of the timeline like any other, so it can
+ * cover a clip, part of one, or a run of them.
+ *
+ * `vhs` and `grain` are not here — they are effects in their own right above,
+ * with tearing and moving grain the grades never had.
+ */
+const LOOK_EFFECTS: LookStyle[] = [
+ "vintage",
+ "horror",
+ "halation",
+ "tech",
+ "noir",
+ "pastel",
+ "blockbuster",
+ "dreamy",
+];
+
+const lookOf = (effect: string): LookStyle | null =>
+ (LOOK_EFFECTS as string[]).includes(effect) ? (effect as LookStyle) : null;
+
+export const EFFECT_LABELS: Record<EffectId, string> = {
+ zoom: "Zoom",
+ grain: "Film grain",
+ vhs: "VHS",
+ glitch: "Glitch",
+ blur: "Blur",
+ vignette: "Vignette",
+ lightleak: "Light leak",
+ flash: "Flash",
+ shake: "Shake",
+ vintage: LOOK_LABELS.vintage,
+ horror: LOOK_LABELS.horror,
+ halation: LOOK_LABELS.halation,
+ tech: LOOK_LABELS.tech,
+ noir: LOOK_LABELS.noir,
+ pastel: LOOK_LABELS.pastel,
+ blockbuster: LOOK_LABELS.blockbuster,
+ dreamy: LOOK_LABELS.dreamy,
+};
+
+const clampAmount = (k: number | undefined) => Math.max(0.05, Math.min(1, k ?? 0.5));
+const fmt = (n: number) => (Math.round(n * 1000) / 1000).toString();
+
+/** What one effect asks the preview canvas to do at `tLocal` seconds in. */
+export interface EffectPreviewState {
+ /** ctx.filter for a self-redraw of the frame; "" = none. */
+ cssFilter: string;
+ /** Animated noise-tile alpha 0..1. */
+ grain?: number;
+ /** Radial corner darkening 0..1. */
+ vignette?: number;
+ /** Flat washes over the picture. */
+ washes?: { color: string; alpha: number; mode: GlobalCompositeOperation }[];
+ /** Chroma ghosting: tinted copies offset by this frame-width fraction. */
+ ghostFrac?: number;
+ /** Whole-frame white flash alpha 0..1. */
+ flash?: number;
+ /** Frame offset in design px (1080 short side). */
+ dx?: number;
+ dy?: number;
+ /** Frame scale: a shake's slight overscale to keep its edges covered, a
+ * zoom's push in. */
+ zoom?: number;
+ /** The point `zoom` scales about, in frame fractions; absent = the middle. */
+ origin?: { x: number; y: number };
+ /** Self-copy glow: blur radius as a fraction of frame height, blended back
+ * over the picture; `bright` isolates highlights first (halation). */
+ glow?: { blurFrac: number; alpha: number; mode: "screen" | "lighten"; bright?: boolean };
+}
+
+/** A host-registered effect recipe: the same dual-renderer shape the
+ * built-ins use. */
+export interface EffectRecipe {
+ previewState(amount: number | undefined, tLocal: number): EffectPreviewState;
+ filterLines(
+ inLabel: string,
+ outLabel: string,
+ amount: number | undefined,
+ start: number,
+ end: number,
+ width: number,
+ height: number,
+ tag: string
+ ): string[] | null;
+}
+
+const customEffects = new Map<string, EffectRecipe>();
+
+/** Register a custom effect (or replace a built-in's recipe). */
+export function defineEffect(id: string, recipe: EffectRecipe): void {
+ customEffects.set(id, recipe);
+}
+
+/** Shake travel at full amount, in design px (1080 short side). Both recipes
+ * read it so the export matches the preview. */
+const SHAKE_AMP = 22;
+
+/** How far a zoom pushes in at full depth. The picture renders at this much
+ * more than frame size and is cropped back around the focus point. */
+const ZOOM_RANGE = 0.6;
+
+/**
+ * The depths a zoom comes in. A zoom is a choice of how close to get, so it is
+ * picked from named depths; the amount each stands for is the same 0..1 knob
+ * every other effect carries.
+ */
+export const ZOOM_LEVELS: { id: string; label: string; amount: number }[] = [
+ { id: "shallow", label: "Shallow", amount: 0.25 },
+ { id: "moderate", label: "Moderate", amount: 0.5 },
+ { id: "deep", label: "Deep", amount: 1 },
+];
+
+/** The frame scale a zoom of this amount renders at. */
+export const zoomScale = (amount: number | undefined) => 1 + ZOOM_RANGE * clampAmount(amount);
+
+/** How long a zoom may take to reach its depth, in seconds. Zero is a cut
+ * straight to the close shot. */
+export const ZOOM_RAMP_MAX = 4;
+/** Quick enough to read as a move, slow enough that the smoothstep eases show. */
+export const ZOOM_RAMP_DEFAULT = 0.5;
+
+export const zoomRampOf = (ramp: number | undefined) =>
+ Math.max(0, Math.min(ZOOM_RAMP_MAX, ramp ?? ZOOM_RAMP_DEFAULT));
+
+/**
+ * How far into its push in a zoom is at `tLocal`: the camera moves in over the
+ * ramp, holds, and pulls back out over the same ramp before the element ends.
+ * The curve eases at both ends, so the move starts and settles rather than
+ * snapping to a constant speed. Without `dur` the zoom only comes in.
+ */
+export const zoomProgress = (tLocal: number, ramp: number | undefined, dur?: number) => {
+ const r = zoomRampOf(ramp);
+ if (r <= 0) return 1;
+ const inP = Math.max(0, Math.min(1, tLocal / r));
+ // An element shorter than two ramps never reaches full depth — the in and
+ // out meet partway, which is the same in every renderer.
+ const outP = dur === undefined ? 1 : Math.max(0, Math.min(1, (dur - tLocal) / r));
+ const p = Math.min(inP, outP);
+ return p * p * (3 - 2 * p);
+};
+
+/** The focus a zoom holds on, defaulted to the middle of the frame. */
+const focusOf = (focus?: { x: number; y: number }) => ({
+ x: Math.max(0, Math.min(1, focus?.x ?? 0.5)),
+ y: Math.max(0, Math.min(1, focus?.y ?? 0.5)),
+});
+
+/** The preview recipe, sampled at a moment — deterministic in `tLocal`, so
+ * the in-tab export replays exactly what the live preview showed. */
+export function effectPreviewState(
+ effect: string,
+ amount: number | undefined,
+ tLocal: number,
+ focus?: { x: number; y: number },
+ ramp?: number,
+ /** The element's own length, so a zoom knows where to pull back out. */
+ dur?: number
+): EffectPreviewState {
+ const custom = customEffects.get(effect);
+ if (custom) return custom.previewState(amount, tLocal);
+ const k = clampAmount(amount);
+ const look = lookOf(effect);
+ if (look) {
+ const post = lookPost(look, k);
+ return {
+ cssFilter: lookCssFilter(look, k),
+ grain: post?.grain,
+ vignette: post?.vignette,
+ washes: post?.washes,
+ ghostFrac: post?.ghost?.shiftFrac,
+ glow: post?.glow,
+ };
+ }
+ switch (effect as EffectId) {
+ case "zoom":
+ // A push in on one part of the picture: the frame renders larger and is
+ // cropped back to size around the focus, which stays where it is. The
+ // scale rides in over the ramp, so the shot moves in rather than cuts.
+ return {
+ cssFilter: "",
+ zoom: 1 + (zoomScale(k) - 1) * zoomProgress(tLocal, ramp, dur),
+ origin: focusOf(focus),
+ };
+ case "grain":
+ return { cssFilter: "", grain: 0.06 + 0.18 * k };
+ case "vhs":
+ return {
+ cssFilter: `saturate(${fmt(1 - 0.35 * k)}) contrast(${fmt(1 - 0.05 * k)}) blur(${fmt(0.6 * k)}px)`,
+ grain: 0.1 * k,
+ ghostFrac: 0.004 * k,
+ };
+ case "glitch": {
+ // The shift jumps between quantized offsets a few times a second, so it
+ // reads as digital tearing rather than a steady fringe.
+ const step = Math.floor(tLocal * 9);
+ const jitter = ((step * 7919) % 5) - 2; // -2..2, deterministic
+ return { cssFilter: "", ghostFrac: (0.004 + 0.003 * jitter) * k, grain: 0.04 * k };
+ }
+ case "blur":
+ return { cssFilter: `blur(${fmt(10 * k)}px)` };
+ case "vignette":
+ return { cssFilter: "", vignette: 0.75 * k };
+ case "lightleak":
+ return {
+ cssFilter: `brightness(${fmt(1 + 0.1 * k)}) saturate(${fmt(1 + 0.15 * k)})`,
+ washes: [{ color: "#ff9a3c", alpha: 0.18 * k, mode: "soft-light" }],
+ vignette: -0.0, // leaks brighten; no darkening
+ };
+ case "flash": {
+ // One bright pop at the element start, decaying over ~0.4s.
+ const gain = Math.exp(-9 * Math.max(0, tLocal));
+ return { cssFilter: "", flash: Math.min(1, 0.85 * k * gain) };
+ }
+ case "shake": {
+ const amp = SHAKE_AMP * k; // design px
+ return {
+ cssFilter: "",
+ dx: amp * Math.sin(tLocal * 33),
+ dy: amp * 0.7 * Math.cos(tLocal * 47),
+ zoom: 1 + (amp * 2) / 1080,
+ };
+ }
+ default:
+ return { cssFilter: "" };
+ }
+}
+
+/**
+ * Apply one effect to the composited frame in place. `scratch` is a reusable
+ * canvas the caller owns (same size as `canvas`); `grainTile` supplies the
+ * shared noise tile (see looks.ts).
+ */
+export function applyEffectToCanvas(
+ canvas: HTMLCanvasElement | OffscreenCanvas,
+ scratch: HTMLCanvasElement | OffscreenCanvas,
+ effect: string,
+ amount: number | undefined,
+ tLocal: number,
+ grainTileFor: (tick: number) => CanvasImageSource | null,
+ focus?: { x: number; y: number },
+ ramp?: number,
+ dur?: number
+): void {
+ const state = effectPreviewState(effect, amount, tLocal, focus, ramp, dur);
+ const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
+ const sctx = scratch.getContext("2d") as CanvasRenderingContext2D | null;
+ if (!ctx || !sctx) return;
+ const W = canvas.width;
+ const H = canvas.height;
+ const scale = Math.min(W, H) / 1080;
+
+ const needsRedraw = state.cssFilter || state.dx || state.dy || (state.zoom && state.zoom !== 1);
+ if (needsRedraw) {
+ sctx.clearRect(0, 0, W, H);
+ sctx.drawImage(canvas, 0, 0);
+ ctx.save();
+ if (state.cssFilter) ctx.filter = state.cssFilter;
+ const zoom = state.zoom ?? 1;
+ const dx = (state.dx ?? 0) * scale;
+ const dy = (state.dy ?? 0) * scale;
+ // The scale holds its origin in place: the middle of the frame for a
+ // shake, the chosen point for a zoom.
+ const ox = state.origin?.x ?? 0.5;
+ const oy = state.origin?.y ?? 0.5;
+ ctx.drawImage(
+ scratch,
+ dx - W * (zoom - 1) * ox,
+ dy - H * (zoom - 1) * oy,
+ W * zoom,
+ H * zoom
+ );
+ ctx.restore();
+ }
+
+ if (state.glow) {
+ // A blurred copy of the frame laid back over itself — halation isolates
+ // the highlights first, dreamy blooms the whole picture.
+ const blurPx = Math.max(1, state.glow.blurFrac * H);
+ sctx.clearRect(0, 0, W, H);
+ sctx.filter = state.glow.bright
+ ? `contrast(2.5) brightness(0.55) saturate(1.4) sepia(0.35) blur(${blurPx}px)`
+ : `blur(${blurPx}px)`;
+ sctx.drawImage(canvas, 0, 0);
+ sctx.filter = "none";
+ ctx.save();
+ ctx.globalAlpha = state.glow.alpha;
+ ctx.globalCompositeOperation = state.glow.mode;
+ ctx.drawImage(scratch, 0, 0);
+ ctx.restore();
+ }
+
+ if (state.ghostFrac) {
+ // Tinted ghost copies, the same approximation the VHS look previews with.
+ const shift = state.ghostFrac * W;
+ sctx.clearRect(0, 0, W, H);
+ sctx.drawImage(canvas, 0, 0);
+ ctx.save();
+ ctx.globalAlpha = 0.35;
+ ctx.globalCompositeOperation = "screen";
+ ctx.drawImage(scratch, shift, 0);
+ ctx.drawImage(scratch, -shift, 0);
+ ctx.restore();
+ }
+
+ if (state.washes) {
+ ctx.save();
+ for (const w of state.washes) {
+ ctx.globalCompositeOperation = w.mode;
+ ctx.globalAlpha = w.alpha;
+ ctx.fillStyle = w.color;
+ ctx.fillRect(0, 0, W, H);
+ }
+ ctx.restore();
+ }
+
+ if (state.grain) {
+ const tile = grainTileFor(Math.floor(tLocal * 30));
+ if (tile) {
+ ctx.save();
+ ctx.globalAlpha = state.grain;
+ ctx.globalCompositeOperation = "overlay";
+ for (let y = 0; y < H; y += 256) {
+ for (let x = 0; x < W; x += 256) ctx.drawImage(tile, x, y);
+ }
+ ctx.restore();
+ }
+ }
+
+ if (state.vignette && state.vignette > 0) {
+ const g = ctx.createRadialGradient(
+ W / 2,
+ H / 2,
+ Math.min(W, H) * 0.35,
+ W / 2,
+ H / 2,
+ Math.hypot(W, H) / 2
+ );
+ g.addColorStop(0, "rgba(0,0,0,0)");
+ g.addColorStop(1, `rgba(0,0,0,${Math.min(0.85, state.vignette)})`);
+ ctx.save();
+ ctx.fillStyle = g;
+ ctx.fillRect(0, 0, W, H);
+ ctx.restore();
+ }
+
+ if (state.flash) {
+ ctx.save();
+ ctx.globalAlpha = state.flash;
+ ctx.fillStyle = "#FFFFFF";
+ ctx.fillRect(0, 0, W, H);
+ ctx.restore();
+ }
+}
+
+/* ---------------------------------------------------------------- ffmpeg */
+
+/** The timeline gate every fragment carries (the gte*lt spelling keeps commas
+ * out of quoted expressions). */
+const gate = (a: number, b: number) => `enable='gte(t,${fmt(a)})*lt(t,${fmt(b)})'`;
+
+/**
+ * ffmpeg filter_complex lines rendering one effect from `[inLabel]` into
+ * `[outLabel]`, active only inside [start, end). Every filter used is in the
+ * LGPL build the shipped engine bundles. Null for an unknown id — the export
+ * renders without the effect rather than failing.
+ */
+export function effectFilterLines(
+ inLabel: string,
+ outLabel: string,
+ effect: string,
+ amount: number | undefined,
+ start: number,
+ end: number,
+ width: number,
+ height: number,
+ tag: string,
+ focus?: { x: number; y: number },
+ ramp?: number
+): string[] | null {
+ const custom = customEffects.get(effect);
+ if (custom) return custom.filterLines(inLabel, outLabel, amount, start, end, width, height, tag);
+ const k = clampAmount(amount);
+ const en = gate(start, end);
+ // Design px are px at a 1080 short side, the same convention the canvas
+ // recipes use — scale by the short side so a vertical frame blurs by as
+ // much as its preview did.
+ const h = Math.min(width, height);
+ const look = lookOf(effect);
+ if (look) {
+ // The graded copy renders on its own branch and replaces the frame only
+ // inside the window, the same shape the shake recipe uses — a look chain
+ // is several filters deep and not all of them take a timeline gate.
+ const lines = lookFilterLines(`lkfi${tag}`, `lkfo${tag}`, look, k, height, "yuv420p", tag);
+ if (!lines) return null;
+ return [
+ `[${inLabel}]split[lkfb${tag}][lkfi${tag}]`,
+ ...lines,
+ `[lkfb${tag}][lkfo${tag}]overlay=0:0:${en}:eof_action=pass[${outLabel}]`,
+ ];
+ }
+ switch (effect as EffectId) {
+ case "zoom": {
+ // The pushed-in copy renders on its own branch — scaled up, then laid
+ // back over the frame so the focus point stays where it is — and
+ // replaces the picture only inside the window. Scale and placement
+ // mirror the canvas recipe, so the export lands on the same part of the
+ // frame the preview showed.
+ const z = zoomScale(k);
+ const f = focusOf(focus);
+ const r = zoomRampOf(ramp);
+ if (r <= 0) {
+ const zw = 2 * Math.ceil((width * z) / 2);
+ const zh = 2 * Math.ceil((height * z) / 2);
+ return [
+ `[${inLabel}]split[efb${tag}][efs${tag}]`,
+ `[efs${tag}]scale=${zw}:${zh},crop=${width}:${height}:` +
+ `${Math.round((zw - width) * f.x)}:${Math.round((zh - height) * f.y)}[efc${tag}]`,
+ `[efb${tag}][efc${tag}]overlay=0:0:${en}:eof_action=pass[${outLabel}]`,
+ ];
+ }
+ // Ramped: the branch is rescaled every frame along the same eased curve
+ // the canvas recipe walks, and the overlay reads the copy's live size,
+ // so the focus holds while the picture grows into it.
+ // In over the ramp, hold, back out over the same ramp before the end —
+ // the same curve `zoomProgress` walks, so preview and export agree.
+ const p =
+ `min(min(max((t-${fmt(start)})/${fmt(r)},0),1),` +
+ `min(max((${fmt(end)}-t)/${fmt(r)},0),1))`;
+ const eased = `(${p})*(${p})*(3-2*(${p}))`;
+ const grow = `(1+${fmt(z - 1)}*(${eased}))`;
+ return [
+ `[${inLabel}]split[efb${tag}][efs${tag}]`,
+ `[efs${tag}]scale=w='trunc(iw*${grow}/2)*2':h='trunc(ih*${grow}/2)*2':eval=frame[efc${tag}]`,
+ `[efb${tag}][efc${tag}]overlay=x='-(w-W)*${fmt(f.x)}':y='-(h-H)*${fmt(f.y)}':` +
+ `${en}:eof_action=pass[${outLabel}]`,
+ ];
+ }
+ case "grain":
+ return [`[${inLabel}]noise=alls=${Math.round(6 + 22 * k)}:allf=t+u:${en}[${outLabel}]`];
+ case "vhs":
+ return [
+ `[${inLabel}]rgbashift=rh=${Math.round(3 * k)}:bh=${-Math.round(3 * k)}:${en},` +
+ `eq=saturation=${fmt(1 - 0.35 * k)}:${en},` +
+ `gblur=sigma=${fmt((0.8 * k * h) / 1080)}:${en},` +
+ `noise=alls=${Math.round(14 * k)}:allf=t+u:${en}[${outLabel}]`,
+ ];
+ case "glitch":
+ return [
+ `[${inLabel}]rgbashift=rh=${Math.round(8 * k)}:bh=${-Math.round(8 * k)}:rv=${Math.round(2 * k)}:${en},` +
+ `noise=alls=${Math.round(8 * k)}:allf=t+u:${en}[${outLabel}]`,
+ ];
+ case "blur":
+ return [`[${inLabel}]gblur=sigma=${fmt((10 * k * h) / 1080)}:${en}[${outLabel}]`];
+ case "vignette":
+ return [`[${inLabel}]vignette=angle=${fmt((k * Math.PI) / 3.5)}:${en}[${outLabel}]`];
+ case "lightleak":
+ return [
+ `[${inLabel}]eq=brightness=${fmt(0.08 * k)}:saturation=${fmt(1 + 0.15 * k)}:gamma_r=${fmt(1 + 0.22 * k)}:gamma_b=${fmt(1 - 0.08 * k)}:${en}[${outLabel}]`,
+ ];
+ case "flash":
+ // A decaying brightness pop from the element start. `eq` reads its
+ // expressions once at init unless told otherwise, so the decay needs
+ // eval=frame — without it the whole window renders at the t=0 peak.
+ return [
+ `[${inLabel}]eq=brightness='${fmt(0.85 * k)}*exp(-9*(t-${fmt(start)}))':eval=frame:${en}[${outLabel}]`,
+ ];
+ case "shake": {
+ // The shaken copy renders on its own branch (overscaled, then cropped
+ // with time-jittered offsets) and replaces the frame only inside the
+ // window, so the rest of the video keeps its unscaled pixels. Zoom,
+ // amplitude and phase mirror the canvas recipe above so the export
+ // shakes exactly like the preview: design px scale with the short side,
+ // and the jitter clock starts at the element, not at the timeline.
+ const zoom = 1 + (2 * SHAKE_AMP * k) / 1080;
+ const zw = 2 * Math.ceil((width * zoom) / 2);
+ const zh = 2 * Math.ceil((height * zoom) / 2);
+ const px = (SHAKE_AMP * k * Math.min(width, height)) / 1080;
+ const ax = fmt(Math.min((zw - width) / 2, px));
+ const ay = fmt(Math.min((zh - height) / 2, px * 0.7));
+ const tl = `(t-${fmt(start)})`;
+ return [
+ `[${inLabel}]split[efb${tag}][efs${tag}]`,
+ `[efs${tag}]scale=${zw}:${zh},crop=${width}:${height}:` +
+ `x='(in_w-out_w)/2+${ax}*sin(${tl}*33)':y='(in_h-out_h)/2+${ay}*cos(${tl}*47)'[efc${tag}]`,
+ `[efb${tag}][efc${tag}]overlay=0:0:${en}:eof_action=pass[${outLabel}]`,
+ ];
+ }
+ default:
+ return null;
+ }
+}
diff --git a/site/packages/effects-kit/src/index.ts b/site/packages/effects-kit/src/index.ts
new file mode 100644
index 00000000..642c0b61
--- /dev/null
+++ b/site/packages/effects-kit/src/index.ts
@@ -0,0 +1,10 @@
+export * from "./types";
+export * from "./anim";
+export * from "./keys";
+export * from "./render";
+export * from "./registry";
+export * from "./cutout";
+export * from "./lottie";
+export * from "./looks";
+export * from "./colorGrade";
+export * from "./effects";
diff --git a/site/packages/effects-kit/src/keys.test.ts b/site/packages/effects-kit/src/keys.test.ts
new file mode 100644
index 00000000..89462f60
--- /dev/null
+++ b/site/packages/effects-kit/src/keys.test.ts
@@ -0,0 +1,128 @@
+import { describe, expect, test } from "bun:test";
+import {
+ evalOverlayFrame,
+ hasOverlayKeys,
+ isOverlayAnimated,
+ keyAt,
+ keyIndexAt,
+ poseAt,
+ poseExtent,
+ removeKeyAt,
+ upsertKey,
+ type OverlayKey,
+} from "./keys";
+
+const el = (over: Partial<Parameters<typeof poseAt>[0]> = {}) => ({
+ start: 2,
+ end: 6,
+ x: 0.5,
+ y: 0.5,
+ ...over,
+});
+const key = (t: number, over: Partial<OverlayKey> = {}): OverlayKey => ({
+ t,
+ x: 0.5,
+ y: 0.5,
+ scale: 1,
+ rotation: 0,
+ opacity: 1,
+ ...over,
+});
+
+describe("keyframe poses", () => {
+ test("no keys leaves the element at rest", () => {
+ const o = el({ rotation: 30, opacity: 0.5 });
+ expect(poseAt(o, 1)).toEqual({ x: 0.5, y: 0.5, scale: 1, rotation: 30, opacity: 0.5 });
+ expect(hasOverlayKeys(o)).toBe(false);
+ expect(isOverlayAnimated(o)).toBe(false);
+ });
+
+ test("the pose moves between keys and holds outside them", () => {
+ const o = el({ kf: [key(1, { x: 0.2 }), key(3, { x: 0.8, scale: 2 })] });
+ expect(poseAt(o, 0).x).toBeCloseTo(0.2); // held before the first
+ expect(poseAt(o, 2).x).toBeCloseTo(0.5); // halfway
+ expect(poseAt(o, 2).scale).toBeCloseTo(1.5);
+ expect(poseAt(o, 9).x).toBeCloseTo(0.8); // held after the last
+ });
+
+ test("keys handed over out of order still play in time order", () => {
+ const o = el({ kf: [key(3, { x: 0.8 }), key(1, { x: 0.2 })] });
+ expect(poseAt(o, 2).x).toBeCloseTo(0.5);
+ });
+
+ test("rotation takes the short way around", () => {
+ const o = el({ kf: [key(0, { rotation: -170 }), key(2, { rotation: 170 })] });
+ // Through -180, not forward through zero.
+ expect(poseAt(o, 1).rotation).toBeCloseTo(-180);
+ });
+
+ test("a new key captures the pose, so adding one changes nothing", () => {
+ const o = el({ kf: [key(0, { x: 0.2 }), key(4, { x: 0.6 })] });
+ const mid = keyAt(o, 2);
+ const after = { ...o, kf: upsertKey(o.kf, mid) };
+ for (const t of [0, 1, 2, 3, 4]) {
+ expect(poseAt(after, t).x).toBeCloseTo(poseAt(o, t).x);
+ }
+ });
+
+ test("a key dropped on an existing one replaces it", () => {
+ const kf = upsertKey([key(1, { x: 0.2 })], key(1.001, { x: 0.9 }));
+ expect(kf.length).toBe(1);
+ expect(kf[0].x).toBeCloseTo(0.9);
+ expect(keyIndexAt(kf, 1)).toBe(0);
+ expect(removeKeyAt(kf, 1).length).toBe(0);
+ });
+});
+
+describe("presets composing over a pose", () => {
+ test("a fade multiplies the keyed opacity instead of replacing it", () => {
+ const o = el({
+ kf: [key(0, { opacity: 0.6 })],
+ anim: { in: { style: "fade", seconds: 1 } },
+ });
+ expect(evalOverlayFrame(o, 0.5).opacity).toBeCloseTo(0.3); // 0.6 × half-faded
+ expect(evalOverlayFrame(o, 2).opacity).toBeCloseTo(0.6); // ramp done
+ });
+
+ test("a slide's travel rides on top of the keyed position", () => {
+ const o = el({ kf: [key(0, { x: 0.2 }), key(4, { x: 0.8 })] });
+ const withSlide = { ...o, anim: { in: { style: "slideleft" as const, seconds: 1 } } };
+ const plain = evalOverlayFrame(o, 0.5);
+ const slid = evalOverlayFrame(withSlide, 0.5);
+ expect(slid.x).toBeCloseTo(plain.x); // the pose is untouched…
+ expect(slid.dx).toBeGreaterThan(0); // …and the travel is separate
+ });
+
+ test("a pulse loop multiplies the keyed scale", () => {
+ const o = el({
+ kf: [key(0, { scale: 2 })],
+ anim: { loop: { style: "pulse" as const, speed: 1 } },
+ });
+ // The loop swings ±6% about the keyed size, never about 1.
+ const scales = Array.from({ length: 40 }, (_, i) => evalOverlayFrame(o, i / 20).scale);
+ expect(Math.min(...scales)).toBeGreaterThan(2 * 0.9);
+ expect(Math.max(...scales)).toBeLessThan(2 * 1.1);
+ });
+
+ test("keys alone count as animated", () => {
+ expect(isOverlayAnimated(el({ kf: [key(0)] }))).toBe(true);
+ expect(isOverlayAnimated(el({ anim: { loop: { style: "spin", speed: 1 } } }))).toBe(true);
+ });
+});
+
+describe("the region a keyed element sweeps", () => {
+ test("spans every pose and the largest scale", () => {
+ const o = el({ kf: [key(0, { x: 0.2, y: 0.4 }), key(3, { x: 0.9, y: 0.5, scale: 2.5 })] });
+ expect(poseExtent(o)).toEqual({ x0: 0.2, y0: 0.4, x1: 0.9, y1: 0.5, scale: 2.5 });
+ });
+
+ test("an unkeyed element sweeps only its own spot", () => {
+ expect(poseExtent(el({ x: 0.3, y: 0.7 }))).toEqual({
+ x0: 0.3,
+ y0: 0.7,
+ x1: 0.3,
+ y1: 0.7,
+ scale: 1,
+ });
+ });
+});
diff --git a/site/packages/effects-kit/src/keys.ts b/site/packages/effects-kit/src/keys.ts
new file mode 100644
index 00000000..c914177a
--- /dev/null
+++ b/site/packages/effects-kit/src/keys.ts
@@ -0,0 +1,193 @@
+/**
+ * Keyframes: the power tier over the In / Out / Loop presets.
+ *
+ * A key is a whole pose — position, scale, rotation, opacity — captured at a
+ * moment inside the element. Between keys the pose moves linearly; before the
+ * first and after the last it holds. An element with no keys poses at its own
+ * resting fields, so keyframes are strictly additive: nothing changes until a
+ * key exists.
+ *
+ * Presets still compose on top. The preset transform is read as an offset —
+ * travel, scale multiplier, extra rotation, alpha multiplier — so a title can
+ * fade in, drift along a keyframed path, and pulse, all at once. `evalOverlayFrame`
+ * is the one function that folds the two together, and every renderer calls
+ * it: the DOM preview, the in-tab canvas export, the behind-speaker pass, and
+ * the frame sampler that feeds ffmpeg.
+ */
+
+import { evalOverlayAnim, hasOverlayAnim, type OverlayAnim } from "./anim";
+
+/** A pose captured at `t` seconds into the element. Values are absolute, in
+ * the same units as the element's own fields — `scale` multiplies the
+ * element's intrinsic size, where 1 is the size it was drawn at. */
+export interface OverlayKey {
+ t: number;
+ x: number; // center, fraction of frame width
+ y: number; // center, fraction of frame height
+ scale: number;
+ rotation: number; // degrees clockwise
+ opacity: number; // 0..1
+}
+
+/** What an element looks like at one moment, before presets compose over it. */
+export interface OverlayPose {
+ x: number;
+ y: number;
+ scale: number;
+ rotation: number;
+ opacity: number;
+}
+
+/** The fields `evalOverlayFrame` reads — every overlay kind has them. */
+interface Posable {
+ start: number;
+ end: number;
+ x: number;
+ y: number;
+ rotation?: number;
+ opacity?: number;
+ anim?: OverlayAnim;
+ kf?: OverlayKey[];
+}
+
+/** Two keys closer than this are the same key. Half a frame at 30fps, so a
+ * key can be dropped on any frame and still replace the one already there. */
+export const KEY_EPSILON = 1 / 60;
+
+/** The element's pose with no keys in play. */
+export function restingPose(o: Posable): OverlayPose {
+ return {
+ x: o.x,
+ y: o.y,
+ scale: 1,
+ rotation: o.rotation ?? 0,
+ opacity: o.opacity ?? 1,
+ };
+}
+
+/** Whether the element carries keys worth evaluating. */
+export function hasOverlayKeys(o: Posable): boolean {
+ return !!o.kf && o.kf.length > 0;
+}
+
+/** Whether anything about the element moves — a preset, keys, or both. An
+ * element that does not move renders as one still picture. */
+export function isOverlayAnimated(o: Posable): boolean {
+ return hasOverlayAnim(o.anim) || hasOverlayKeys(o);
+}
+
+/** Keys in play order. Callers may hand them over unsorted (a key added at the
+ * playhead lands wherever the user was). */
+export function sortedKeys(keys: OverlayKey[]): OverlayKey[] {
+ return [...keys].sort((a, b) => a.t - b.t);
+}
+
+/** The pose at `tLocal` seconds into the element: interpolated between the
+ * surrounding keys, held flat outside them, resting when there are no keys. */
+export function poseAt(o: Posable, tLocal: number): OverlayPose {
+ const keys = o.kf;
+ if (!keys || keys.length === 0) return restingPose(o);
+ const ks = sortedKeys(keys);
+ if (tLocal <= ks[0].t) return poseOf(ks[0]);
+ const last = ks[ks.length - 1];
+ if (tLocal >= last.t) return poseOf(last);
+ let i = 0;
+ while (i < ks.length - 1 && ks[i + 1].t <= tLocal) i++;
+ const a = ks[i];
+ const b = ks[i + 1];
+ const span = b.t - a.t;
+ const p = span > 1e-6 ? (tLocal - a.t) / span : 0;
+ const mix = (u: number, v: number) => u + (v - u) * p;
+ return {
+ x: mix(a.x, b.x),
+ y: mix(a.y, b.y),
+ scale: mix(a.scale, b.scale),
+ // Rotation takes the short way around, so a key at 350° into one at 10°
+ // turns 20° forward instead of 340° back.
+ rotation: a.rotation + shortestTurn(a.rotation, b.rotation) * p,
+ opacity: mix(a.opacity, b.opacity),
+ };
+}
+
+function shortestTurn(from: number, to: number): number {
+ let d = (to - from) % 360;
+ if (d > 180) d -= 360;
+ if (d < -180) d += 360;
+ return d;
+}
+
+function poseOf(k: OverlayKey): OverlayPose {
+ return { x: k.x, y: k.y, scale: k.scale, rotation: k.rotation, opacity: k.opacity };
+}
+
+/** A key holding the element's pose at `t`, ready to be added. Capturing the
+ * live pose is what makes the first two keys a no-op: the element sits where
+ * it already sat until one of them is moved. */
+export function keyAt(o: Posable, t: number): OverlayKey {
+ return { t, ...poseAt(o, t) };
+}
+
+/** Insert or replace a key, keeping the list in play order. A key dropped on
+ * top of an existing one (within `KEY_EPSILON`) replaces it. */
+export function upsertKey(keys: OverlayKey[] | undefined, key: OverlayKey): OverlayKey[] {
+ const rest = (keys ?? []).filter((k) => Math.abs(k.t - key.t) > KEY_EPSILON);
+ return sortedKeys([...rest, key]);
+}
+
+/** Drop the key at `t`, or return the list unchanged when none sits there. */
+export function removeKeyAt(keys: OverlayKey[] | undefined, t: number): OverlayKey[] {
+ return (keys ?? []).filter((k) => Math.abs(k.t - t) > KEY_EPSILON);
+}
+
+/** The key sitting at `t`, if there is one. */
+export function keyIndexAt(keys: OverlayKey[] | undefined, t: number): number {
+ return (keys ?? []).findIndex((k) => Math.abs(k.t - t) <= KEY_EPSILON);
+}
+
+/** The element's full frame state: the keyframed pose with the preset
+ * animation composed over it. `dx`/`dy` are the preset's travel in design px
+ * (1080 short side); the pose's `x`/`y` stay fractions of the frame, because
+ * that is what keeps an element in the same place at any aspect.
+ */
+export interface OverlayFrameState extends OverlayPose {
+ dx: number;
+ dy: number;
+ /** 0..1 share of the text shown (typewriter); absent when not typing. */
+ textProgress?: number;
+}
+
+export function evalOverlayFrame(o: Posable, tLocal: number): OverlayFrameState {
+ const dur = Math.max(0.1, o.end - o.start);
+ const pose = poseAt(o, tLocal);
+ const ev = evalOverlayAnim(o.anim, tLocal, dur);
+ return {
+ x: pose.x,
+ y: pose.y,
+ dx: ev.dx,
+ dy: ev.dy,
+ scale: pose.scale * ev.scale,
+ rotation: pose.rotation + ev.rotate,
+ opacity: pose.opacity * ev.alpha,
+ ...(ev.textProgress !== undefined ? { textProgress: ev.textProgress } : {}),
+ };
+}
+
+/** The corners of the box the pose sweeps through, as fractions of the frame,
+ * plus the largest scale it reaches. The frame sampler crops to this, so a
+ * keyframed element's pictures stay small even when it crosses the screen. */
+export function poseExtent(o: Posable): {
+ x0: number;
+ y0: number;
+ x1: number;
+ y1: number;
+ scale: number;
+} {
+ const poses = hasOverlayKeys(o) ? sortedKeys(o.kf!).map(poseOf) : [restingPose(o)];
+ return {
+ x0: Math.min(...poses.map((p) => p.x)),
+ y0: Math.min(...poses.map((p) => p.y)),
+ x1: Math.max(...poses.map((p) => p.x)),
+ y1: Math.max(...poses.map((p) => p.y)),
+ scale: Math.max(...poses.map((p) => p.scale), 1),
+ };
+}
diff --git a/site/src/cut/lib/looks.test.ts b/site/packages/effects-kit/src/looks.test.ts
similarity index 98%
rename from site/src/cut/lib/looks.test.ts
rename to site/packages/effects-kit/src/looks.test.ts
index 3bbcad1c..305924bf 100644
--- a/site/src/cut/lib/looks.test.ts
+++ b/site/packages/effects-kit/src/looks.test.ts
@@ -1,6 +1,6 @@
import { describe, expect, test } from "bun:test";
import { lookCssFilter, lookFilterLines, lookPost } from "./looks";
-import { LOOK_IDS } from "./types";
+import { LOOK_IDS } from "./index";
describe("lookFilterLines", () => {
test("every look renders lines from [in] to [out] ending in the format", () => {
diff --git a/site/src/cut/lib/looks.ts b/site/packages/effects-kit/src/looks.ts
similarity index 92%
rename from site/src/cut/lib/looks.ts
rename to site/packages/effects-kit/src/looks.ts
index bcfe554a..2d0b516d 100644
--- a/site/src/cut/lib/looks.ts
+++ b/site/packages/effects-kit/src/looks.ts
@@ -1,4 +1,42 @@
-import type { LookStyle } from "./types";
+/** Preset filter looks: a named color/effect treatment baked over a clip's
+ * picture in both preview and export. */
+export type LookStyle =
+ | "vintage"
+ | "vhs"
+ | "horror"
+ | "halation"
+ | "tech"
+ | "noir"
+ | "grain"
+ | "pastel"
+ | "blockbuster"
+ | "dreamy";
+
+export const LOOK_IDS: LookStyle[] = [
+ "vintage",
+ "vhs",
+ "horror",
+ "halation",
+ "tech",
+ "noir",
+ "grain",
+ "pastel",
+ "blockbuster",
+ "dreamy",
+];
+
+export const LOOK_LABELS: Record<LookStyle, string> = {
+ vintage: "Vintage",
+ vhs: "VHS",
+ horror: "Analogue horror",
+ halation: "Halation",
+ tech: "Modern tech",
+ noir: "Noir",
+ grain: "Film grain",
+ pastel: "Pastel",
+ blockbuster: "Blockbuster",
+ dreamy: "Dreamy",
+};
/**
* Preset filter looks, one recipe per style with two renderers — the preview
diff --git a/site/packages/effects-kit/src/lottie.ts b/site/packages/effects-kit/src/lottie.ts
new file mode 100644
index 00000000..bf58aad2
--- /dev/null
+++ b/site/packages/effects-kit/src/lottie.ts
@@ -0,0 +1,83 @@
+/**
+ * Lottie sticker playback: a deterministic frame-seek wrapper over
+ * lottie-web's canvas renderer. Preview and export both drive it with
+ * goToAndStop, so the same timeline second always draws the same frame.
+ * lottie-web loads lazily — hosts that never touch a Lottie sticker never
+ * ship or parse it.
+ */
+
+export interface LottieHandle {
+ width: number;
+ height: number;
+ /** One play-through, seconds. */
+ duration: number;
+ /** Draw the frame at `tSec` (wrapped into the duration) and return the
+ * backing canvas, sized to the animation's own pixels. */
+ seek(tSec: number): HTMLCanvasElement;
+ destroy(): void;
+}
+
+interface LottieData {
+ w?: number;
+ h?: number;
+ ip?: number;
+ op?: number;
+ fr?: number;
+ layers?: unknown[];
+ v?: string;
+}
+
+/** Whether parsed JSON looks like a Lottie animation document. */
+export function isLottieData(data: unknown): data is LottieData {
+ const d = data as LottieData | null;
+ return (
+ !!d &&
+ typeof d === "object" &&
+ Array.isArray(d.layers) &&
+ typeof d.w === "number" &&
+ typeof d.h === "number" &&
+ typeof d.fr === "number" &&
+ typeof d.op === "number"
+ );
+}
+
+/** Instantiate a seekable player for a Lottie JSON document. Each handle owns
+ * its own backing canvas, so two elements can sit on different frames. */
+export async function createLottieHandle(data: unknown): Promise<LottieHandle | null> {
+ if (!isLottieData(data) || typeof document === "undefined") return null;
+ const lottie = (await import("lottie-web")).default;
+ const width = Math.max(2, Math.round(data.w!));
+ const height = Math.max(2, Math.round(data.h!));
+ const canvas = document.createElement("canvas");
+ canvas.width = width;
+ canvas.height = height;
+ const ctx = canvas.getContext("2d")!;
+ const anim = lottie.loadAnimation<"canvas">({
+ // Unused when a context is provided, but the config type requires it.
+ container: document.createElement("div"),
+ renderer: "canvas",
+ loop: false,
+ autoplay: false,
+ // A fresh copy per instance: lottie mutates the document as it plays.
+ animationData: JSON.parse(JSON.stringify(data)),
+ rendererSettings: { context: ctx, clearCanvas: true },
+ });
+ const fr = data.fr!;
+ const frames = Math.max(1, data.op! - (data.ip ?? 0));
+ const duration = frames / fr;
+ return {
+ width,
+ height,
+ duration,
+ seek(tSec: number) {
+ const wrapped = ((tSec % duration) + duration) % duration;
+ // goToAndStop takes a frame index when isFrame is true and renders it
+ // synchronously into the provided context.
+ anim.goToAndStop(Math.min(frames - 0.001, wrapped * fr), true);
+ return canvas;
+ },
+ destroy() {
+ anim.destroy();
+ },
+ };
+}
diff --git a/site/packages/effects-kit/src/registry.ts b/site/packages/effects-kit/src/registry.ts
new file mode 100644
index 00000000..2b928ad3
--- /dev/null
+++ b/site/packages/effects-kit/src/registry.ts
@@ -0,0 +1,28 @@
+/**
+ * Element plugin registry: one registration per element kind. The built-in
+ * kinds (text, shape, sticker) paint without registering; a host registers a
+ * plugin to add a custom kind or replace a built-in painter.
+ */
+
+import type { PaintFrame, RenderEnv } from "./render";
+import type { Overlay } from "./types";
+
+export interface ElementPlugin<T extends Overlay = Overlay> {
+ kind: string;
+ paint(
+ ctx: CanvasRenderingContext2D,
+ element: T,
+ frame: PaintFrame,
+ env: RenderEnv
+ ): void | Promise<void>;
+}
+
+const plugins = new Map<string, ElementPlugin>();
+
+export function defineElement<T extends Overlay>(plugin: ElementPlugin<T>): void {
+ plugins.set(plugin.kind, plugin as unknown as ElementPlugin);
+}
+
+export function elementPlugin(kind: string): ElementPlugin | undefined {
+ return plugins.get(kind);
+}
diff --git a/site/packages/effects-kit/src/render.ts b/site/packages/effects-kit/src/render.ts
new file mode 100644
index 00000000..0543ef9e
--- /dev/null
+++ b/site/packages/effects-kit/src/render.ts
@@ -0,0 +1,674 @@
+/**
+ * Canvas painters: rasterize any overlay element to a transparent full-frame
+ * PNG at the export resolution, with the same metrics the host's DOM preview
+ * uses. The host supplies a RenderEnv for everything it owns — font stacks and
+ * sticker image bytes — so painters stay host-agnostic.
+ */
+
+import {
+ FLOAT_TRAVEL,
+ loopPeriod,
+ SLIDE_TRAVEL,
+ type OverlayAnim,
+ type OverlayAnimStyle,
+} from "./anim";
+import { evalOverlayFrame, hasOverlayKeys, poseAt, poseExtent, sortedKeys } from "./keys";
+import type { LottieHandle } from "./lottie";
+import { elementPlugin } from "./registry";
+import type { Overlay, ShapeOverlay, StickerOverlay, TextOverlay, TextShadowSpec } from "./types";
+
+// Text metrics shared by DOM preview and canvas burn-in.
+export const LINE_HEIGHT = 1.25;
+export const PLATE_PAD_X = 0.55; // em
+export const PLATE_PAD_Y = 0.3; // em
+export const PLATE_RADIUS = 0.32; // em
+export const PLATE_COLOR = "#000000";
+export const PLATE_OPACITY = 0.55;
+export const PLATE_FILL = "rgba(0, 0, 0, 0.55)";
+export const SHADOW = { color: "rgba(0, 0, 0, 0.65)", blur: 14, offsetY: 2 };
+
+/** A hex color with an opacity folded in, as rgba(); non-hex input falls back
+ * to black at that opacity. */
+function rgba(hex: string, opacity: number): string {
+ const m = /^#?([0-9a-fA-F]{6})$/.exec(hex.trim());
+ if (!m) return `rgba(0, 0, 0, ${opacity})`;
+ const n = parseInt(m[1], 16);
+ return `rgba(${(n >> 16) & 255}, ${(n >> 8) & 255}, ${n & 255}, ${opacity})`;
+}
+
+/** A text element's plate fill as an rgba() string, defaulting to translucent
+ * black when color/opacity are unset. Shared by preview and export. */
+export function plateFill(o: { plateColor?: string; plateOpacity?: number }): string {
+ return rgba(o.plateColor ?? PLATE_COLOR, o.plateOpacity ?? PLATE_OPACITY);
+}
+
+/** Resolve a text element's shadow to concrete design-px values, or null when
+ * it is off. `true` and `{}` both mean the legacy default look. */
+export function resolveShadow(
+ shadow: boolean | TextShadowSpec
+): { color: string; blur: number; offsetY: number } | null {
+ if (shadow === false) return null;
+ const spec: TextShadowSpec = shadow === true ? {} : shadow;
+ return {
+ color: rgba(spec.color ?? "#000000", spec.opacity ?? 0.65),
+ blur: spec.blur ?? SHADOW.blur,
+ offsetY: spec.offsetY ?? SHADOW.offsetY,
+ };
+}
+
+/** Await the faces a text element needs before measuring or painting, so a
+ * rasterize never silently measures a fallback font. No-op outside the DOM. */
+export async function ensureFontLoaded(cssFont: string, text?: string): Promise<void> {
+ if (typeof document === "undefined" || !document.fonts?.load) return;
+ try {
+ await document.fonts.load(cssFont, text);
+ } catch {
+ // An unknown family or a blocked face falls back; painting proceeds.
+ }
+}
+
+/** A decoded sticker image with its natural size (for aspect). */
+export interface StickerImage {
+ source: CanvasImageSource;
+ width: number;
+ height: number;
+}
+
+/** What the host supplies so painters can resolve host-owned resources. */
+export interface RenderEnv {
+ /** CSS font-family stack for a host font id. */
+ fontStack(font: string): string;
+ /** Decoded image for a sticker asset id; null when the asset is gone. */
+ resolveAsset?(assetId: string): Promise<StickerImage | null>;
+ /** Seekable player for a Lottie sticker asset id; null when the asset is
+ * gone or isn't a Lottie document. */
+ resolveLottie?(assetId: string): Promise<LottieHandle | null>;
+}
+
+/** The frame geometry handed to painters. `scale` converts design px (1080
+ * short side) to output px. `t` is seconds since the element's start, for
+ * painters whose pixels are time-dependent (Lottie stickers). */
+export interface PaintFrame {
+ width: number;
+ height: number;
+ scale: number;
+ t?: number;
+}
+
+/**
+ * A shape's pixel geometry in the output frame — the one place the fractions
+ * become pixels, shared by the canvas painter and testable on its own.
+ * Line/arrow thickness rides the shape's `h`; the arrow head scales with the
+ * thickness but never eats more than half the length.
+ */
+export function shapeMetrics(o: ShapeOverlay, frame: PaintFrame) {
+ const cx = o.x * frame.width;
+ const cy = o.y * frame.height;
+ const w = Math.max(1, o.w * frame.width);
+ const h = Math.max(1, o.h * frame.height);
+ const thickness = Math.max(2 * frame.scale, h);
+ const headLen = Math.min(thickness * 3, w / 2);
+ return {
+ cx,
+ cy,
+ w,
+ h,
+ radius: (o.radius ?? 0) * frame.scale,
+ strokeWidth: (o.stroke?.width ?? 0) * frame.scale,
+ thickness,
+ headLen,
+ headHalf: thickness * 1.6,
+ };
+}
+
+function paintShape(ctx: CanvasRenderingContext2D, o: ShapeOverlay, frame: PaintFrame) {
+ const m = shapeMetrics(o, frame);
+ const alpha = ctx.globalAlpha;
+ ctx.fillStyle = o.fill;
+ if (o.shape === "rect" || o.shape === "ellipse") {
+ ctx.beginPath();
+ if (o.shape === "rect") {
+ ctx.roundRect(m.cx - m.w / 2, m.cy - m.h / 2, m.w, m.h, m.radius);
+ } else {
+ ctx.ellipse(m.cx, m.cy, m.w / 2, m.h / 2, 0, 0, Math.PI * 2);
+ }
+ ctx.globalAlpha = alpha * (o.fillOpacity ?? 1);
+ ctx.fill();
+ ctx.globalAlpha = alpha;
+ if (o.stroke && m.strokeWidth > 0) {
+ ctx.strokeStyle = o.stroke.color;
+ ctx.lineWidth = m.strokeWidth;
+ ctx.stroke();
+ }
+ return;
+ }
+ // Line / arrow: a horizontal stroke across the element box; rotation (the
+ // generic element transform) gives it any direction.
+ const x0 = m.cx - m.w / 2;
+ const x1 = m.cx + m.w / 2;
+ ctx.strokeStyle = o.fill;
+ ctx.lineWidth = m.thickness;
+ ctx.lineCap = "round";
+ ctx.beginPath();
+ ctx.moveTo(x0, m.cy);
+ ctx.lineTo(o.shape === "arrow" ? x1 - m.headLen : x1, m.cy);
+ ctx.stroke();
+ if (o.shape === "arrow") {
+ ctx.fillStyle = o.fill;
+ ctx.beginPath();
+ ctx.moveTo(x1, m.cy);
+ ctx.lineTo(x1 - m.headLen, m.cy - m.headHalf);
+ ctx.lineTo(x1 - m.headLen, m.cy + m.headHalf);
+ ctx.closePath();
+ ctx.fill();
+ }
+}
+
+async function paintSticker(
+ ctx: CanvasRenderingContext2D,
+ o: StickerOverlay,
+ frame: PaintFrame,
+ env: RenderEnv
+) {
+ const cx = o.x * frame.width;
+ const cy = o.y * frame.height;
+ const w = Math.max(1, o.w * frame.width);
+ if (!o.assetId) return;
+ if (o.lottie) {
+ const handle = await env.resolveLottie?.(o.assetId);
+ if (!handle) return;
+ const aspect = handle.width > 0 && handle.height > 0 ? handle.width / handle.height : 1;
+ const h = w / aspect;
+ ctx.drawImage(handle.seek(frame.t ?? 0), cx - w / 2, cy - h / 2, w, h);
+ return;
+ }
+ if (!env.resolveAsset) return;
+ const img = await env.resolveAsset(o.assetId);
+ if (!img) return;
+ const aspect = img.width > 0 && img.height > 0 ? img.width / img.height : 1;
+ const h = w / aspect;
+ ctx.drawImage(img.source, cx - w / 2, cy - h / 2, w, h);
+}
+
+/** The CSS font shorthand a text element paints (and loads faces) with. */
+export function textCssFont(overlay: TextOverlay, fpx: number, env: RenderEnv): string {
+ return `${overlay.italic ? "italic " : ""}${overlay.weight} ${fpx}px ${env.fontStack(overlay.font)}`;
+}
+
+async function paintText(
+ ctx: CanvasRenderingContext2D,
+ overlay: TextOverlay,
+ frame: PaintFrame,
+ env: RenderEnv
+) {
+ const { width, scale } = frame;
+ const fpx = overlay.size * scale;
+ const cssFont = textCssFont(overlay, fpx, env);
+ // Faces must be resident before measureText, or the plate and alignment are
+ // measured against a fallback font and the burn-in drifts off the preview.
+ await ensureFontLoaded(cssFont, overlay.text);
+ ctx.font = cssFont;
+ if ("letterSpacing" in ctx) {
+ ctx.letterSpacing = `${(overlay.letterSpacing ?? 0) * fpx}px`;
+ }
+ ctx.textAlign = "center";
+ ctx.textBaseline = "middle";
+
+ const lines = overlay.text.split("\n");
+ const lineH = fpx * (overlay.lineHeight ?? LINE_HEIGHT);
+ const totalH = lines.length * lineH;
+ const cx = overlay.x * width;
+ const cy = overlay.y * frame.height;
+ const widthsOf = lines.map((l) => ctx.measureText(l).width);
+ const maxW = Math.max(...widthsOf, 1);
+
+ if (overlay.plate) {
+ const padX = PLATE_PAD_X * fpx;
+ const padY = PLATE_PAD_Y * fpx;
+ const r = (overlay.plateRadius ?? PLATE_RADIUS) * fpx;
+ const w = maxW + padX * 2;
+ const h = totalH + padY * 2;
+ ctx.fillStyle = plateFill(overlay);
+ ctx.beginPath();
+ ctx.roundRect(cx - w / 2, cy - h / 2, w, h, r);
+ ctx.fill();
+ }
+
+ const shadow = resolveShadow(overlay.shadow);
+ if (shadow) {
+ ctx.shadowColor = shadow.color;
+ ctx.shadowBlur = shadow.blur * scale;
+ ctx.shadowOffsetY = shadow.offsetY * scale;
+ }
+
+ // Outline behind the fill: stroke first, then fill over it — the same order
+ // the DOM pair (-webkit-text-stroke + paint-order: stroke fill) paints.
+ const strokeW = (overlay.stroke?.width ?? 0) * fpx;
+ const drawText = (text: string, x: number, y: number, fill: string = overlay.color) => {
+ if (overlay.stroke && strokeW > 0) {
+ ctx.lineJoin = "round";
+ ctx.lineWidth = strokeW * 2; // half of a centered stroke shows behind the fill
+ ctx.strokeStyle = overlay.stroke.color;
+ ctx.strokeText(text, x, y);
+ }
+ ctx.fillStyle = fill;
+ ctx.fillText(text, x, y);
+ };
+
+ // Line alignment inside the block; the block stays centered on (x, y).
+ const align = overlay.align ?? "center";
+ const lineX = (i: number) =>
+ align === "center"
+ ? cx
+ : align === "left"
+ ? cx - maxW / 2 + widthsOf[i] / 2
+ : cx + maxW / 2 - widthsOf[i] / 2;
+
+ // Karaoke: draw word by word so the spoken word gets the accent color and an
+ // underline; the word index counts across all lines.
+ let k = 0;
+ lines.forEach((line, i) => {
+ const y = cy - totalH / 2 + lineH * (i + 0.5);
+ if (overlay.highlightWord === undefined) {
+ drawText(line, lineX(i), y);
+ return;
+ }
+ const words = line.split(" ").filter(Boolean);
+ const spaceW = ctx.measureText(" ").width;
+ const widths = words.map((w) => ctx.measureText(w).width);
+ const lineW = widths.reduce((a, b) => a + b, 0) + spaceW * (words.length - 1);
+ let x = cx - lineW / 2;
+ ctx.textAlign = "left";
+ words.forEach((w, wi) => {
+ const active = k === overlay.highlightWord;
+ if (active && overlay.highlightMode === "box") {
+ // Accent box behind the word, contrast text on top — drawn with the
+ // shadow off so the box and its word stay crisp.
+ const pad = 0.12 * fpx;
+ const prevShadow = ctx.shadowColor;
+ ctx.shadowColor = "transparent";
+ ctx.fillStyle = overlay.highlightColor ?? "#FFE94A";
+ ctx.beginPath();
+ ctx.roundRect(x - pad, y - fpx * 0.5 - pad, widths[wi] + pad * 2, fpx + pad * 2, 0.18 * fpx);
+ ctx.fill();
+ drawText(w, x, y, overlay.highlightText ?? "#111114");
+ ctx.shadowColor = prevShadow;
+ } else if (active) {
+ drawText(w, x, y, overlay.highlightColor ?? "#FFE94A");
+ if (overlay.highlightMode !== "color") {
+ ctx.fillRect(x, y + fpx * 0.42, widths[wi], Math.max(2 * scale, fpx * 0.07));
+ }
+ } else {
+ drawText(w, x, y);
+ }
+ x += widths[wi] + spaceW;
+ k++;
+ });
+ ctx.textAlign = "center";
+ });
+}
+
+/** Paint one element in frame coordinates with no element-level transform —
+ * the raw painter dispatch. Callers own rotation/opacity (renderElementPng)
+ * or the full animation transform (renderOverlayFrames). */
+export async function paintElement(
+ ctx: CanvasRenderingContext2D,
+ overlay: Overlay,
+ frame: PaintFrame,
+ env: RenderEnv
+): Promise<void> {
+ const kind = overlay.kind ?? "text";
+ if (kind === "effect") return; // effects filter the video; nothing to paint
+ const plugin = elementPlugin(kind);
+ if (plugin) {
+ await plugin.paint(ctx, overlay, frame, env);
+ } else if (kind === "shape") {
+ paintShape(ctx, overlay as ShapeOverlay, frame);
+ } else if (kind === "sticker") {
+ await paintSticker(ctx, overlay as StickerOverlay, frame, env);
+ } else {
+ await paintText(ctx, overlay as TextOverlay, frame, env);
+ }
+}
+
+function pngBlob(canvas: HTMLCanvasElement): Promise<Blob> {
+ return new Promise((resolve, reject) =>
+ canvas.toBlob(
+ (b) => (b ? resolve(b) : reject(new Error("Could not render the overlay."))),
+ "image/png"
+ )
+ );
+}
+
+/**
+ * Rasterize one element to a transparent full-frame PNG at the given output
+ * size. Rotation and opacity apply here, around the element center, so every
+ * painter draws unrotated in frame coordinates.
+ */
+export async function renderElementPng(
+ overlay: Overlay,
+ width: number,
+ height: number,
+ env: RenderEnv
+): Promise<Blob> {
+ // Element sizes are design pixels with a 1080 short side, so scaling by the
+ // short side keeps them the same visual size in any aspect and resolution.
+ const scale = Math.min(width, height) / 1080;
+ const canvas = document.createElement("canvas");
+ canvas.width = width;
+ canvas.height = height;
+ const ctx = canvas.getContext("2d")!;
+ const frame: PaintFrame = { width, height, scale };
+
+ ctx.globalAlpha = overlay.opacity ?? 1;
+ if (overlay.rotation) {
+ const cx = overlay.x * width;
+ const cy = overlay.y * height;
+ ctx.translate(cx, cy);
+ ctx.rotate((overlay.rotation * Math.PI) / 180);
+ ctx.translate(-cx, -cy);
+ }
+
+ await paintElement(ctx, overlay, frame, env);
+ return pngBlob(canvas);
+}
+
+/** An element's resting box in output pixels (center + size), rotation left
+ * to the caller. Conservative on purpose: it feeds the animated-region crop,
+ * where a few spare pixels beat a clipped shadow. */
+export async function measureElementBounds(
+ overlay: Overlay,
+ frame: PaintFrame,
+ env: RenderEnv
+): Promise<{ cx: number; cy: number; w: number; h: number }> {
+ const cx = overlay.x * frame.width;
+ const cy = overlay.y * frame.height;
+ const kind = overlay.kind ?? "text";
+ if (kind === "effect") return { cx, cy, w: 2, h: 2 };
+ if (kind === "shape") {
+ const s = overlay as ShapeOverlay;
+ const m = shapeMetrics(s, frame);
+ const lineLike = s.shape === "line" || s.shape === "arrow";
+ return {
+ cx,
+ cy,
+ w: m.w + m.strokeWidth * 2 + (lineLike ? m.thickness : 0),
+ h: (lineLike ? Math.max(m.thickness, m.headHalf * 2) : m.h) + m.strokeWidth * 2,
+ };
+ }
+ if (kind === "sticker") {
+ const s = overlay as StickerOverlay;
+ const w = Math.max(1, s.w * frame.width);
+ if (s.lottie) {
+ const handle = s.assetId ? await env.resolveLottie?.(s.assetId) : null;
+ const aspect = handle && handle.width > 0 ? handle.width / handle.height : 1;
+ return { cx, cy, w, h: w / aspect };
+ }
+ const img = s.assetId && env.resolveAsset ? await env.resolveAsset(s.assetId) : null;
+ const aspect = img && img.width > 0 && img.height > 0 ? img.width / img.height : 1;
+ return { cx, cy, w, h: w / aspect };
+ }
+ const o = overlay as TextOverlay;
+ const fpx = o.size * frame.scale;
+ const cssFont = textCssFont(o, fpx, env);
+ await ensureFontLoaded(cssFont, o.text);
+ const scratch = document.createElement("canvas");
+ const ctx = scratch.getContext("2d")!;
+ ctx.font = cssFont;
+ if ("letterSpacing" in ctx) ctx.letterSpacing = `${(o.letterSpacing ?? 0) * fpx}px`;
+ const lines = o.text.split("\n");
+ const maxW = Math.max(...lines.map((l) => ctx.measureText(l).width), 1);
+ const lineH = fpx * (o.lineHeight ?? LINE_HEIGHT);
+ let w = maxW;
+ let h = lines.length * lineH;
+ if (o.plate) {
+ w += PLATE_PAD_X * fpx * 2;
+ h += PLATE_PAD_Y * fpx * 2;
+ }
+ const strokePad = (o.stroke?.width ?? 0) * fpx * 2;
+ const shadow = resolveShadow(o.shadow);
+ const shadowPad = shadow ? (shadow.blur + Math.abs(shadow.offsetY)) * frame.scale : 0;
+ // Ascenders/descenders overhang the em-box line grid a little.
+ return { cx, cy, w: w + strokePad * 2 + shadowPad * 2, h: h + fpx * 0.3 + strokePad * 2 + shadowPad * 2 };
+}
+
+/** One window of an animated element, ready to be drawn under a per-frame
+ * transform. `overlay` is neutral — rotation and opacity belong to the pose,
+ * so a picture that baked them in would apply each twice — and a typewriter
+ * window carries the characters revealed so far. */
+export interface AnimatedLayer {
+ overlay: Overlay;
+ start: number; // timeline seconds
+ end: number;
+ /** The slots still in play for this window (a typed-out one has spent its
+ * ramp, so that slot is gone). */
+ anim: OverlayAnim;
+}
+
+/** Typewriter slices per second: one per output frame, matching the rate the
+ * frame sequences sample at. */
+const TYPE_SLICE_FPS = 30;
+
+/**
+ * Split an animated element into the windows a canvas renderer draws.
+ *
+ * Continuous transforms — slides, fades, loops, a keyframed pose — stay one
+ * window whose picture is drawn under a per-frame transform. A typewriter
+ * changes the pixels themselves, so its ramp becomes one window per revealed
+ * slice. This is the canvas-side twin of `renderOverlayFrames`: same split,
+ * one baked into pictures, the other handed to a compositor.
+ */
+export function planAnimatedLayers(o: Overlay, end: number): AnimatedLayer[] {
+ const anim: OverlayAnim = o.anim ?? {};
+ const dur = Math.max(0.1, o.end - o.start);
+ const isTw = (slot: "in" | "out") =>
+ anim[slot]?.style === "typewriter" && (o.kind ?? "text") === "text";
+ const inS = anim.in ? Math.min(anim.in.seconds, dur) : 0;
+ const outS = anim.out ? Math.min(anim.out.seconds, Math.max(0, dur - inS)) : 0;
+ const out: AnimatedLayer[] = [];
+ const push = (overlay: Overlay, from: number, to: number, animPart: OverlayAnim) => {
+ const start = Math.max(o.start, from);
+ const stop = Math.min(end, to);
+ if (stop - start < 1e-3) return;
+ out.push({
+ overlay: { ...overlay, rotation: undefined, opacity: undefined } as Overlay,
+ start,
+ end: stop,
+ anim: animPart,
+ });
+ };
+ const typeSlices = (from: number, secs: number, reverse: boolean) => {
+ const text = (o as TextOverlay).text;
+ const n = Math.max(1, Math.min(text.length, Math.ceil(secs * TYPE_SLICE_FPS)));
+ for (let i = 1; i <= n; i++) {
+ const chars = Math.ceil((text.length * (reverse ? n - i + 1 : i)) / n);
+ push(
+ { ...o, text: text.slice(0, chars) } as Overlay,
+ from + ((i - 1) / n) * secs,
+ from + (i / n) * secs,
+ { ...anim, [reverse ? "out" : "in"]: undefined }
+ );
+ }
+ };
+
+ if (inS > 1e-3) {
+ if (isTw("in")) typeSlices(o.start, inS, false);
+ else push(o, o.start, o.start + inS, anim);
+ }
+ push(o, o.start + inS, o.start + dur - outS, anim);
+ if (outS > 1e-3) {
+ if (isTw("out")) typeSlices(o.start + dur - outS, outS, true);
+ else push(o, o.start + dur - outS, o.start + dur, anim);
+ }
+ return out;
+}
+
+/** One animated element as a region-cropped frame sequence: unique pictures
+ * plus play entries (repeats reference the same picture, so a loop costs one
+ * cycle of pixels however long the element runs). Entries cover the
+ * element's [0, dur] exactly; the caller pads the timeline around it. */
+export interface OverlayFrameSet {
+ /** Region top-left in output px, and its size. */
+ x: number;
+ y: number;
+ w: number;
+ h: number;
+ /** Unique region-sized pictures. */
+ images: Blob[];
+ /** Region-sized transparent filler for the ffconcat gaps. */
+ blank: Blob;
+ /** Play order: index into `images` + seconds on screen. */
+ entries: { image: number; duration: number }[];
+}
+
+/**
+ * Sample an animated element into frames at `fps`: the In and Out windows
+ * frame by frame, the middle as one still (or one loop cycle, repeated by
+ * reference). Samples the same evalOverlayAnim the live preview applies.
+ */
+export async function renderOverlayFrames(
+ overlay: Overlay,
+ width: number,
+ height: number,
+ fps: number,
+ env: RenderEnv
+): Promise<OverlayFrameSet> {
+ const scale = Math.min(width, height) / 1080;
+ const frame: PaintFrame = { width, height, scale };
+ const anim = overlay.anim;
+ const dur = Math.max(0.1, overlay.end - overlay.start);
+ const inS = anim?.in ? Math.min(anim.in.seconds, dur) : 0;
+ const outS = anim?.out ? Math.min(anim.out.seconds, Math.max(0, dur - inS)) : 0;
+ // A Lottie sticker's pixels change every frame on their own; its play-through
+ // acts as the middle segment's cycle even with no transform loop set.
+ const sticker = (overlay.kind ?? "text") === "sticker" ? (overlay as StickerOverlay) : null;
+ const lottieDur =
+ sticker?.lottie && sticker.assetId
+ ? ((await env.resolveLottie?.(sticker.assetId))?.duration ?? null)
+ : null;
+
+ // The crop region: the resting box, grown for travel, scale overshoot,
+ // shadow spill, and — when anything rotates — the circumscribed square.
+ const base = await measureElementBounds(overlay, frame, env);
+ const styles: OverlayAnimStyle[] = [anim?.in?.style, anim?.out?.style].filter(
+ (s): s is OverlayAnimStyle => !!s
+ );
+ const slides = styles.some((s) => s.startsWith("slide"));
+ const travel =
+ (slides ? SLIDE_TRAVEL : 0) * scale + (anim?.loop?.style === "float" ? FLOAT_TRAVEL * scale : 0);
+ // A keyframed element carries its own travel and its own zoom: the region
+ // spans every pose the track visits, at the largest scale it reaches.
+ const keyed = hasOverlayKeys(overlay);
+ const extent = poseExtent(overlay);
+ const maxScale = 1.15 * extent.scale;
+ const rotates =
+ !!overlay.rotation ||
+ anim?.loop?.style === "spin" ||
+ anim?.loop?.style === "wiggle" ||
+ (keyed && sortedKeys(overlay.kf!).some((k) => !!k.rotation));
+ let halfW = (base.w * maxScale) / 2 + travel + 4;
+ let halfH = (base.h * maxScale) / 2 + travel + 4;
+ if (rotates) {
+ const r = (Math.hypot(base.w, base.h) * maxScale) / 2 + travel + 4;
+ halfW = r;
+ halfH = r;
+ }
+ const x0 = Math.max(0, Math.floor(extent.x0 * width - halfW));
+ const y0 = Math.max(0, Math.floor(extent.y0 * height - halfH));
+ const x1 = Math.min(width, Math.ceil(extent.x1 * width + halfW));
+ const y1 = Math.min(height, Math.ceil(extent.y1 * height + halfH));
+ const rw = Math.max(2, x1 - x0);
+ const rh = Math.max(2, y1 - y0);
+
+ const canvas = document.createElement("canvas");
+ canvas.width = rw;
+ canvas.height = rh;
+ const ctx = canvas.getContext("2d")!;
+
+ const drawAt = async (tLocal: number): Promise<Blob> => {
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.clearRect(0, 0, rw, rh);
+ const ev = evalOverlayFrame(overlay, tLocal);
+ if (ev.opacity <= 0.001) return pngBlob(canvas); // fully transparent frame
+ ctx.globalAlpha = ev.opacity;
+ ctx.translate(-x0, -y0);
+ // The pose places the element; the preset's travel rides on top of it.
+ ctx.translate(ev.x * width + ev.dx * scale, ev.y * height + ev.dy * scale);
+ ctx.rotate((ev.rotation * Math.PI) / 180);
+ ctx.scale(ev.scale, ev.scale);
+ // Back to the origin the painters draw around, so they stay unaware of
+ // any of this.
+ ctx.translate(-overlay.x * width, -overlay.y * height);
+ let el: Overlay = { ...overlay, rotation: undefined, opacity: undefined };
+ if (ev.textProgress !== undefined && (overlay.kind ?? "text") === "text") {
+ const text = (overlay as TextOverlay).text;
+ const chars = Math.max(0, Math.min(text.length, Math.ceil(ev.textProgress * text.length)));
+ el = { ...el, text: text.slice(0, chars) } as Overlay;
+ // Typing reveals characters; it must not also fade them.
+ ctx.globalAlpha = poseAt(overlay, tLocal).opacity;
+ }
+ await paintElement(ctx, el, { ...frame, t: tLocal }, env);
+ return pngBlob(canvas);
+ };
+
+ const images: Blob[] = [];
+ const entries: OverlayFrameSet["entries"] = [];
+ const push = async (t: number, duration: number) => {
+ images.push(await drawAt(t));
+ entries.push({ image: images.length - 1, duration });
+ };
+
+ const step = 1 / fps;
+ // A keyframed pose changes on its own schedule, so there is no still middle
+ // and no cycle to repeat: sample the whole element frame by frame.
+ if (keyed) {
+ const n = Math.max(1, Math.round(dur * fps));
+ for (let i = 0; i < n; i++) {
+ await push(i * step, i === n - 1 ? dur - (n - 1) * step : step);
+ }
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.clearRect(0, 0, rw, rh);
+ return { x: x0, y: y0, w: rw, h: rh, images, blank: await pngBlob(canvas), entries };
+ }
+ // Head ramp, frame by frame; the last frame absorbs the rounding so the
+ // segment lengths sum exactly.
+ if (inS > 1e-3) {
+ const n = Math.max(1, Math.round(inS * fps));
+ for (let i = 0; i < n; i++) {
+ await push(i * step, i === n - 1 ? inS - (n - 1) * step : step);
+ }
+ }
+ const middle = dur - inS - outS;
+ if (middle > 1e-3) {
+ const period = loopPeriod(anim) ?? lottieDur;
+ if (period) {
+ // One cycle of pictures, repeated by reference to cover the middle.
+ const n = Math.max(2, Math.min(Math.round(period * fps), 180));
+ const cycleStep = period / n;
+ const cycleStart = images.length;
+ for (let j = 0; j < n; j++) images.push(await drawAt(inS + j * cycleStep));
+ let remaining = middle;
+ let j = 0;
+ while (remaining > 1e-4) {
+ const d = Math.min(cycleStep, remaining);
+ entries.push({ image: cycleStart + (j % n), duration: d });
+ remaining -= d;
+ j++;
+ }
+ } else {
+ await push(inS + middle / 2, middle);
+ }
+ }
+ if (outS > 1e-3) {
+ const from = dur - outS;
+ const n = Math.max(1, Math.round(outS * fps));
+ for (let i = 0; i < n; i++) {
+ await push(from + i * step, i === n - 1 ? outS - (n - 1) * step : step);
+ }
+ }
+
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.clearRect(0, 0, rw, rh);
+ const blank = await pngBlob(canvas);
+ return { x: x0, y: y0, w: rw, h: rh, images, blank, entries };
+}
diff --git a/site/packages/effects-kit/src/textStyle.test.ts b/site/packages/effects-kit/src/textStyle.test.ts
new file mode 100644
index 00000000..d037ce41
--- /dev/null
+++ b/site/packages/effects-kit/src/textStyle.test.ts
@@ -0,0 +1,52 @@
+import { describe, expect, test } from "bun:test";
+import { pickTextStyle, TEXT_STYLE_FIELDS, type TextOverlay } from "./types";
+
+const title = (over: Partial<TextOverlay> = {}): TextOverlay => ({
+ id: "t1",
+ kind: "text",
+ text: "Hello",
+ start: 1,
+ end: 4,
+ x: 0.25,
+ y: 0.75,
+ size: 88,
+ font: "anton",
+ weight: 700,
+ color: "#fff",
+ shadow: true,
+ plate: false,
+ rotation: 20,
+ opacity: 0.5,
+ ...over,
+});
+
+describe("text style", () => {
+ test("carries every style field and nothing that identifies the title", () => {
+ const style = pickTextStyle(title({ italic: true, letterSpacing: 0.1 }));
+ expect(Object.keys(style).sort()).toEqual([...TEXT_STYLE_FIELDS].sort());
+ // What makes it that particular title stays behind.
+ for (const own of ["id", "text", "start", "end", "x", "y", "rotation", "opacity"]) {
+ expect(own in style).toBe(false);
+ }
+ expect(style.size).toBe(88);
+ expect(style.font).toBe("anton");
+ expect(style.italic).toBe(true);
+ expect(style.letterSpacing).toBe(0.1);
+ });
+
+ test("applying a style over a title changes its look and leaves it in place", () => {
+ const original = title();
+ const styled = { ...original, ...pickTextStyle(title({ size: 40, color: "#0f0" })) };
+ expect(styled.size).toBe(40);
+ expect(styled.color).toBe("#0f0");
+ expect(styled.x).toBeCloseTo(0.25); // still where it was
+ expect(styled.text).toBe("Hello");
+ });
+
+ test("an unset field is carried as unset, so a saved look does not invent one", () => {
+ const style = pickTextStyle(title());
+ expect("italic" in style).toBe(true);
+ expect(style.italic).toBe(undefined);
+ expect(JSON.parse(JSON.stringify(style)).italic).toBe(undefined);
+ });
+});
diff --git a/site/packages/effects-kit/src/types.ts b/site/packages/effects-kit/src/types.ts
new file mode 100644
index 00000000..8fd1c9d0
--- /dev/null
+++ b/site/packages/effects-kit/src/types.ts
@@ -0,0 +1,223 @@
+/**
+ * The overlay element model: one discriminated union for everything that
+ * composites over the video frame. Positions are center fractions of the
+ * output frame (0..1); pixel sizes are design pixels at a 1080 short side,
+ * so an element reads the same at any aspect or export resolution.
+ */
+
+import type { OverlayAnim } from "./anim";
+import type { EffectOverlay } from "./effects";
+import type { OverlayKey } from "./keys";
+
+/** Fields every overlay element carries, whatever its kind. */
+export interface OverlayBase {
+ id: string;
+ start: number; // timeline seconds
+ end: number;
+ x: number; // center, fraction of frame width 0..1
+ y: number; // center, fraction of frame height 0..1
+ /** Which overlay track (row) this sits on, 0-based. */
+ lane?: number;
+ /** Rotation about the element center, degrees clockwise; absent = 0. */
+ rotation?: number;
+ /** Whole-element opacity 0..1; absent = 1. */
+ opacity?: number;
+ /** Hidden elements stay on the timeline (grayed) but are excluded from the
+ * played/exported picture. */
+ hidden?: boolean;
+ /** Group membership (select/move as one); assigned by the host's grouping. */
+ groupId?: string;
+ /** Preset In / Out / Loop animation (see anim.ts); absent = static. */
+ anim?: OverlayAnim;
+ /** Keyframed pose track, seconds from the element's start (see keys.ts).
+ * Absent or empty = the element holds its resting pose; presets compose
+ * over whichever of the two is in play. */
+ kf?: OverlayKey[];
+}
+
+/** How a highlighted word lights up in karaoke text: accent color only,
+ * accent color plus underline, or an accent box behind the word. */
+export type WordAccentMode = "color" | "underline" | "box";
+
+/** A custom drop shadow. Every field is optional — absent ones take the
+ * legacy defaults, so `shadow: true` and `shadow: {}` render identically. */
+export interface TextShadowSpec {
+ color?: string; // hex; opacity folds in separately
+ blur?: number; // px at the 1080 design short side
+ offsetY?: number; // px at the 1080 design short side
+ opacity?: number; // 0..1
+}
+
+export type TextAlign = "left" | "center" | "right";
+
+/** A text element. `kind` may be absent — documents written before the union
+ * existed stored bare title objects, and absence still means text. `font` is a
+ * host font id, resolved to a CSS stack through the render env. */
+export interface TextOverlay extends OverlayBase {
+ kind?: "text";
+ text: string;
+ size: number; // px at a 1080-wide design frame
+ font: string;
+ weight: 400 | 700;
+ italic?: boolean;
+ color: string;
+ /** Outline drawn behind the fill (stroke-before-fill; the DOM pair is
+ * -webkit-text-stroke with paint-order: stroke fill). Width is in em so the
+ * outline scales with the text. */
+ stroke?: { color: string; width: number };
+ /** Extra tracking between glyphs, in em. */
+ letterSpacing?: number;
+ /** Line height as a multiplier; absent = the shared LINE_HEIGHT. */
+ lineHeight?: number;
+ /** How lines align inside the text block; absent = center. The block
+ * itself stays centered on (x, y). */
+ align?: TextAlign;
+ /** Drop shadow: `true` = the legacy default look, an object customizes it. */
+ shadow: boolean | TextShadowSpec;
+ plate: boolean; // rounded plate behind the text
+ plateRadius?: number; // plate corner radius in em
+ plateColor?: string; // plate fill color
+ plateOpacity?: number; // plate fill opacity 0..1
+ /** Draw this text behind the person in the shot (host-composited: video →
+ * text → segmented person back on top). Falls back to a normal front title
+ * when no person is detectable. */
+ behindSubject?: boolean;
+ /** Karaoke burn-in: index of the display word (whitespace-split across all
+ * lines) drawn per the accent treatment. */
+ highlightWord?: number;
+ highlightColor?: string;
+ highlightMode?: WordAccentMode;
+ highlightText?: string;
+}
+
+export type ShapeKind = "rect" | "ellipse" | "line" | "arrow";
+
+export interface ShapeOverlay extends OverlayBase {
+ kind: "shape";
+ shape: ShapeKind;
+ w: number; // width, fraction of frame width
+ h: number; // height, fraction of frame height (line/arrow: stroke thickness)
+ fill: string;
+ fillOpacity?: number; // 0..1, absent = 1 (composes with `opacity`)
+ radius?: number; // rect corner radius, px at 1080 short side
+ stroke?: { color: string; width: number }; // outline, width px at 1080 short side
+}
+
+/** An image sticker. Height follows the source's own aspect, so only the
+ * width is stored. */
+export interface StickerOverlay extends OverlayBase {
+ kind: "sticker";
+ /** Host media asset holding the sticker image (PNG/JPG/SVG, or a Lottie
+ * JSON document when `lottie` is set). */
+ assetId?: string;
+ w: number; // width, fraction of frame width
+ /** The asset is a Lottie animation; it plays on a loop for the element's
+ * whole duration, frame-seeked from timeline time. */
+ lottie?: boolean;
+}
+
+export type Overlay = TextOverlay | ShapeOverlay | StickerOverlay | EffectOverlay;
+
+export type OverlayKind = "text" | "shape" | "sticker" | "effect";
+
+/** A patch that may touch any kind's fields (never the discriminant). */
+export type OverlayPatch = Partial<
+ Omit<TextOverlay, "kind"> &
+ Omit<ShapeOverlay, "kind"> &
+ Omit<StickerOverlay, "kind"> &
+ Omit<EffectOverlay, "kind">
+>;
+
+export function overlayKind(o: Overlay): OverlayKind {
+ return o.kind ?? "text";
+}
+
+export function isTextOverlay(o: Overlay): o is TextOverlay {
+ return (o.kind ?? "text") === "text";
+}
+
+export function isShapeOverlay(o: Overlay): o is ShapeOverlay {
+ return o.kind === "shape";
+}
+
+export function isStickerOverlay(o: Overlay): o is StickerOverlay {
+ return o.kind === "sticker";
+}
+
+export function isEffectOverlay(o: Overlay): o is EffectOverlay {
+ return o.kind === "effect";
+}
+
+/** Tolerant-load stamp: overlays saved before the union carry no `kind`;
+ * stamping `"text"` on load gives every in-memory element a discriminant. */
+export function stampOverlayKinds<T extends Overlay>(overlays: T[]): T[] {
+ return overlays.map((o) => (o.kind ? o : { ...o, kind: "text" as const }));
+}
+
+/** Serialize counterpart: drop the `kind` a text element only carries because
+ * the loader stamped it, so title-only documents write byte-identical to the
+ * pre-union shape. */
+export function stripDefaultOverlayKinds<T extends Overlay>(overlays: T[]): T[] {
+ // Same array back when there is nothing to strip: hosts compare documents by
+ // identity to tell an edit from a re-read, so a projection that always
+ // allocates would read as a change every time.
+ if (!overlays.some((o) => o.kind === "text")) return overlays;
+ return overlays.map((o) => {
+ if (o.kind !== "text") return o;
+ const rest = { ...o };
+ delete rest.kind;
+ return rest;
+ });
+}
+
+/**
+ * A group renamer for one copying operation — a paste, a template applied.
+ * Members copied together land in one new group; the copies never join the
+ * group they were taken from, which is what keeps a pasted pair from welding
+ * itself to the pair it came from. Returns a patch to spread over each copy,
+ * empty for an ungrouped element. The host supplies `newId` so id format
+ * stays its business.
+ */
+export function groupRemap(newId: () => string): (o: Overlay) => { groupId?: string } {
+ const seen = new Map<string, string>();
+ return (o) => {
+ if (!o.groupId) return {};
+ let next = seen.get(o.groupId);
+ if (!next) {
+ next = newId();
+ seen.set(o.groupId, next);
+ }
+ return { groupId: next };
+ };
+}
+
+/** What a title's look is made of, apart from its words, timing and place:
+ * the one list, which both the type and the reader below derive from. A new
+ * style field is added here and nowhere else. */
+export const TEXT_STYLE_FIELDS = [
+ "size",
+ "font",
+ "weight",
+ "italic",
+ "color",
+ "stroke",
+ "letterSpacing",
+ "lineHeight",
+ "align",
+ "shadow",
+ "plate",
+ "plateColor",
+ "plateOpacity",
+ "plateRadius",
+] as const;
+
+/** A saved look, and what one title copies from another. */
+export type TextStyle = Pick<TextOverlay, (typeof TEXT_STYLE_FIELDS)[number]>;
+
+/** Read a text element's style, leaving everything that makes it that
+ * particular title behind. */
+export function pickTextStyle(o: TextOverlay): TextStyle {
+ const style: Record<string, unknown> = {};
+ for (const field of TEXT_STYLE_FIELDS) style[field] = o[field];
+ return style as TextStyle;
+}
diff --git a/site/public/mediapipe/selfie_segmenter.tflite b/site/public/mediapipe/selfie_segmenter.tflite
new file mode 100644
index 00000000..a4ebd477
Binary files /dev/null and b/site/public/mediapipe/selfie_segmenter.tflite differ
diff --git a/site/scripts/copy-mediapipe-assets.mjs b/site/scripts/copy-mediapipe-assets.mjs
new file mode 100644
index 00000000..6174d0b4
--- /dev/null
+++ b/site/scripts/copy-mediapipe-assets.mjs
@@ -0,0 +1,21 @@
+// Stage the MediaPipe vision runtime into public/ so segmentation runs
+// self-hosted (no runtime Google fetch). The wasm pair is ~12MB, so it is
+// copied from node_modules on install rather than committed; the tiny
+// person-segmentation model is committed beside it. Runs from postinstall.
+import { copyFileSync, existsSync, mkdirSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+const src = path.join(root, "node_modules", "@mediapipe", "tasks-vision", "wasm");
+const dest = path.join(root, "public", "mediapipe", "wasm");
+
+// Modern browsers all take the SIMD build; the nosimd/module variants stay out.
+const files = ["vision_wasm_internal.js", "vision_wasm_internal.wasm"];
+
+if (!existsSync(src)) {
+ console.warn("copy-mediapipe-assets: @mediapipe/tasks-vision not installed; skipping");
+ process.exit(0);
+}
+mkdirSync(dest, { recursive: true });
+for (const f of files) copyFileSync(path.join(src, f), path.join(dest, f));
diff --git a/site/src/cut/components/OverlayLayer.tsx b/site/src/cut/components/OverlayLayer.tsx
index 76114b32..c8df19d2 100644
--- a/site/src/cut/components/OverlayLayer.tsx
+++ b/site/src/cut/components/OverlayLayer.tsx
@@ -1,6 +1,6 @@
"use client";
-import { useCallback, useEffect, useRef, useState, type CSSProperties } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState, type CSSProperties } from "react";
import { startDrag } from "@/cut/lib/drag";
import { useEditor } from "@/cut/lib/store";
import {
@@ -14,6 +14,7 @@ import {
subtitleLaneCount,
trackPos,
} from "@/cut/lib/subtitles";
+import { evalOverlayFrame, hasOverlayKeys, isOverlayAnimated, resolveShadow, shapeMetrics, type LottieHandle } from "@donkeycut/effects-kit";
import {
LINE_HEIGHT,
PLATE_PAD_X,
@@ -22,13 +23,62 @@ import {
plateFill,
SHADOW,
} from "@/cut/lib/textRender";
-import { frameOf, fontStack, type TextOverlay } from "@/cut/lib/types";
+import {
+ clampOverlayPos,
+ frameOf,
+ fontStack,
+ isTextOverlay,
+ laneOf,
+ type Overlay,
+ type OverlayPatch,
+ type ShapeOverlay,
+ type StickerOverlay,
+} from "@/cut/lib/types";
import { cn } from "@/lib/utils";
// Plate geometry as CSS, kept in lockstep with the export burn-in metrics.
const PLATE_PADDING = `${PLATE_PAD_Y}em ${PLATE_PAD_X}em`;
const PLATE_RADIUS_EM = `${PLATE_RADIUS}em`;
+/**
+ * The rotate handle's cursor. CSS has no rotate cursor, so the affordance is
+ * drawn here — and because the glyph is an arc with a head at each end rather
+ * than a full circle, which way it faces carries meaning: it is turned to sit
+ * tangent to the element, so the heads always point the way a drag will take
+ * it. White over a dark rim keeps it legible on any footage.
+ */
+const ROTATE_GLYPH_W = 59;
+const ROTATE_GLYPH_H = 90;
+const ROTATE_GLYPH =
+ "M58.202 82.4995C57.967 81.6525 57.086 81.1635 56.239 81.3945L40.985 85.6485C46.122 77.9215 48.929 68.7815 48.929 59.4355C48.929 34.8625 30.107 14.6195 6.125 12.3295L18.995 2.88146C19.705 2.35846 19.857 1.36246 19.336 0.652464C18.815 -0.0575359 17.82 -0.213536 17.108 0.310464L0.65 12.3935C0.601 12.4295 0.576001 12.4825 0.533001 12.5225C0.480001 12.5705 0.437001 12.6245 0.391001 12.6795C0.306001 12.7815 0.230999 12.8825 0.174999 12.9995C0.138999 13.0715 0.118 13.1475 0.0939999 13.2255C0.0589999 13.3415 0.0320002 13.4545 0.0240002 13.5755C0.0210002 13.6165 0 13.6525 0 13.6945C0 13.7585 0.0289993 13.8125 0.0359993 13.8735C0.0479993 13.9695 0.0630004 14.0605 0.0930004 14.1545C0.133 14.2855 0.19 14.4015 0.261 14.5155C0.282 14.5495 0.285999 14.5885 0.309999 14.6215L12.391 31.0785C12.702 31.5055 13.186 31.7295 13.677 31.7295C14.004 31.7295 14.334 31.6295 14.619 31.4215C15.329 30.8985 15.481 29.9025 14.96 29.1925L4.873 15.4545C27.685 17.1435 45.741 36.1995 45.741 59.4355C45.741 68.5495 42.879 77.4515 37.663 84.8545L33.051 68.3205C32.814 67.4735 31.932 66.9815 31.088 67.2155C30.24 67.4525 29.745 68.3305 29.981 69.1805L35.467 88.8445C35.474 88.8695 35.494 88.8895 35.502 88.9135C35.551 89.0595 35.618 89.1925 35.709 89.3215C35.745 89.3735 35.785 89.4185 35.827 89.4665C35.88 89.5255 35.912 89.5965 35.975 89.6485C36.03 89.6935 36.096 89.7135 36.155 89.7495C36.179 89.7645 36.194 89.7895 36.218 89.8035C36.237 89.8145 36.261 89.8095 36.281 89.8205C36.505 89.9325 36.742 90.0095 36.984 90.0095C36.987 90.0095 36.99 90.0075 36.993 90.0075C36.996 90.0075 36.998 90.0095 37.001 90.0095C37.144 90.0095 37.289 89.9905 37.429 89.9505L57.095 84.4645C57.944 84.2275 58.439 83.3495 58.202 82.4995Z";
+/** A square the glyph fits inside at any angle, so turning it never clips. */
+const ROTATE_BOX = Math.ceil(Math.hypot(ROTATE_GLYPH_W, ROTATE_GLYPH_H));
+/** The glyph's own arrows run top-to-bottom, so a quarter turn puts them
+ * left-to-right — the way an unrotated element's top handle drags. */
+const ROTATE_BASE_DEG = -90;
+
+const rotateCursorCache = new Map<number, string>();
+
+function rotateCursor(rotationDeg: number): string {
+ // Quantized: a drag would otherwise mint a new cursor image every frame, and
+ // five degrees is below what the eye reads on a glyph this size.
+ const deg = Math.round((rotationDeg + ROTATE_BASE_DEG) / 5) * 5;
+ const hit = rotateCursorCache.get(deg);
+ if (hit) return hit;
+ const c = ROTATE_BOX / 2;
+ const dx = (ROTATE_BOX - ROTATE_GLYPH_W) / 2;
+ const dy = (ROTATE_BOX - ROTATE_GLYPH_H) / 2;
+ const layer = (attrs: string) =>
+ `<g transform='rotate(${deg} ${c} ${c}) translate(${dx} ${dy})'><path d='${ROTATE_GLYPH}' ${attrs}/></g>`;
+ const svg =
+ `<svg xmlns='http://www.w3.org/2000/svg' width='32' height='32' viewBox='0 0 ${ROTATE_BOX} ${ROTATE_BOX}'>` +
+ `${layer("fill='none' stroke='rgba(0,0,0,0.55)' stroke-width='10' stroke-linejoin='round'")}` +
+ `${layer("fill='%23ffffff'")}</svg>`;
+ const css = `url("data:image/svg+xml;utf8,${svg}") 16 16, grab`;
+ rotateCursorCache.set(deg, css);
+ return css;
+}
+
/** Snap when a title edge/center lands within this many stage px of a line. */
const SNAP_PX = 6;
/** Safe-area inset (fraction of the frame) offered as margin snap lines. */
@@ -44,8 +94,33 @@ interface Guides {
* uids, so these can't collide. */
const subtitleBoxId = (lane: number) => `subtitle-caption-${lane}`;
-export function OverlayLayer({ stageWidth }: { stageWidth: number }) {
- const overlays = useEditor((s) => s.overlays);
+export function OverlayLayer({
+ stageWidth,
+ transform,
+ filter,
+ from = 0,
+ to = Infinity,
+ captions = true,
+}: {
+ stageWidth: number;
+ /** How a live effect moves the frame (shake, tearing) — the elements travel
+ * with the picture they sit on. */
+ transform?: string;
+ /** The grade of every effect above these elements. An effect filters what
+ * plays under it, so the ones below it wear its look and the ones above it
+ * do not (see StageEffects). */
+ filter?: string;
+ /** The lanes this slice of the stack draws: [from, to). */
+ from?: number;
+ to?: number;
+ /** Captions ride the topmost slice; nothing grades them. */
+ captions?: boolean;
+}) {
+ const allOverlays = useEditor((s) => s.overlays);
+ const overlays = useMemo(
+ () => allOverlays.filter((o) => laneOf(o) >= from && laneOf(o) < to),
+ [allOverlays, from, to]
+ );
const currentTime = useEditor((s) => s.currentTime);
const skimTime = useEditor((s) => s.skimTime);
const playing = useEditor((s) => s.playing);
@@ -67,7 +142,7 @@ export function OverlayLayer({ stageWidth }: { stageWidth: number }) {
const frame = frameOf(aspect);
const stageHeight = (stageWidth * frame.h) / frame.w;
- // Figma-style smart snapping: while dragging an item, pull its left/center/
+ // Smart snapping: while dragging an item, pull its left/center/
// right edges to the frame edges, safe margins, center line, and the edges
// and centers of the other on-screen items — independently per axis — and
// paint the matched guide lines. Hold ⌘/Ctrl to bypass.
@@ -135,20 +210,31 @@ export function OverlayLayer({ stageWidth }: { stageWidth: number }) {
// isolation: it shows alone so it never stacks over whatever title is live.
// Not while scrubbing — the skimmer must still show the exact frame's titles.
const scrubbing = !playing && skimTime !== null;
- const sel = selection?.kind === "text" ? overlays.find((o) => o.id === selection.id) : undefined;
+ const sel =
+ selection?.kind === "overlay" ? overlays.find((o) => o.id === selection.id) : undefined;
const isolate = !!sel && !scrubbing && !(t >= sel.start && t <= sel.end);
return (
- <div ref={rootRef} className="pointer-events-none absolute inset-0">
- <SubtitleCaptions
- stageWidth={stageWidth}
- stageHeight={stageHeight}
- registerBox={registerBox}
- snap={snap}
- onSnapEnd={clearGuides}
- />
+ <div
+ ref={rootRef}
+ className="pointer-events-none absolute inset-0"
+ style={{ transform, filter }}
+ >
+ {captions && (
+ <SubtitleCaptions
+ stageWidth={stageWidth}
+ stageHeight={stageHeight}
+ registerBox={registerBox}
+ snap={snap}
+ onSnapEnd={clearGuides}
+ />
+ )}
{overlays.map((o) => {
if (o.hidden) return null;
+ // An effect has no place in the frame — it grades what plays under it
+ // (see StageEffects), so it draws nothing here and takes no box,
+ // handles or drag. The timeline bar is the thing you grab.
+ if (o.kind === "effect") return null;
const selected = sel?.id === o.id;
const inRange = t >= o.start && t <= o.end;
// While hover-scrubbing (paused, skimmer active) the preview must show the
@@ -164,6 +250,7 @@ export function OverlayLayer({ stageWidth }: { stageWidth: number }) {
// selection chrome (outline, resize handle) does not.
selected={selected && !scrubbing}
ghost={!inRange && !selected}
+ t={t}
stageWidth={stageWidth}
registerBox={registerBox}
snap={snap}
@@ -342,14 +429,17 @@ function OverlayItem({
overlay: o,
selected,
ghost,
+ t,
stageWidth,
registerBox,
snap,
onSnapEnd,
}: {
- overlay: TextOverlay;
+ overlay: Overlay;
selected: boolean;
ghost: boolean;
+ /** The previewed timeline moment (playhead or paused skimmer). */
+ t: number;
stageWidth: number;
registerBox: (id: string, el: HTMLElement | null) => void;
snap: (id: string, x: number, y: number, ev: PointerEvent) => { x: number; y: number };
@@ -361,8 +451,9 @@ function OverlayItem({
const frame = frameOf(useEditor((s) => s.aspect));
const scale = stageWidth / frame.w;
const stageHeight = (stageWidth * frame.h) / frame.w;
+ const isText = isTextOverlay(o);
- // Publish this title's box so a sibling drag can align to it.
+ // Publish this element's box so a sibling drag can align to it.
useEffect(() => {
registerBox(o.id, boxRef.current);
return () => registerBox(o.id, null);
@@ -379,24 +470,72 @@ function OverlayItem({
}
}, [editing]);
+ // The element's animation state at this exact moment — the same evaluator
+ // the export samples, applied here as a live CSS transform. Ghosts (shown
+ // out of range for editing) render at rest.
+ const live = !ghost && isOverlayAnimated(o) ? evalOverlayFrame(o, Math.max(0, t - o.start)) : null;
+ const animTransform = live
+ ? ` translate(${live.dx * scale}px, ${live.dy * scale}px)` +
+ (live.rotation ? ` rotate(${live.rotation}deg)` : "") +
+ (live.scale !== 1 ? ` scale(${live.scale})` : "")
+ : o.rotation
+ ? ` rotate(${o.rotation}deg)`
+ : "";
+ // Typewriter: the visible slice of the text (display only, never while
+ // the box is being edited).
+ const shownText =
+ isText && live?.textProgress !== undefined && !editing
+ ? o.text.slice(0, Math.ceil(live.textProgress * o.text.length))
+ : isText
+ ? o.text
+ : "";
+ // Behind-speaker titles draw inside the canvas compositor; the DOM keeps an
+ // invisible hit target (and the selection chrome) so editing still works.
+ const behindHidden = isText && !!o.behindSubject && !editing;
+
+ // Every kind shares position, rotation, and opacity; text carries its type
+ // styles on the same box so the edit caret inherits them.
const style: CSSProperties = {
- left: `${o.x * 100}%`,
- top: `${o.y * 100}%`,
- fontSize: o.size * scale,
- fontFamily: fontStack(o.font),
- fontWeight: o.weight,
- lineHeight: LINE_HEIGHT,
- color: o.color,
- textShadow: o.shadow
- ? `0 ${SHADOW.offsetY * scale}px ${SHADOW.blur * scale}px ${SHADOW.color}`
- : undefined,
- background: o.plate ? plateFill(o) : undefined,
- padding: o.plate ? PLATE_PADDING : undefined,
- borderRadius: o.plate ? `${o.plateRadius ?? PLATE_RADIUS}em` : undefined,
- opacity: ghost ? 0.35 : 1,
+ // A keyframed element is placed by its pose, not by its resting x/y.
+ left: `${(live?.x ?? o.x) * 100}%`,
+ top: `${(live?.y ?? o.y) * 100}%`,
+ transform: `translate(-50%, -50%)${animTransform}`,
+ opacity: (live ? live.opacity : (o.opacity ?? 1)) * (ghost ? 0.35 : 1),
+ ...(isText
+ ? (() => {
+ const shadow = resolveShadow(o.shadow);
+ return {
+ fontSize: o.size * scale,
+ fontFamily: fontStack(o.font),
+ fontWeight: o.weight,
+ fontStyle: o.italic ? "italic" : undefined,
+ lineHeight: o.lineHeight ?? LINE_HEIGHT,
+ letterSpacing: o.letterSpacing ? `${o.letterSpacing}em` : undefined,
+ textAlign: o.align ?? "center",
+ color: o.color,
+ // The canvas pair strokes before filling; paint-order keeps the
+ // DOM stroke behind the fill the same way. `width` is the visible
+ // outline outside the glyph, so the centered CSS stroke doubles it
+ // (the canvas painter doubles its lineWidth identically).
+ WebkitTextStroke: o.stroke
+ ? `${o.stroke.width * 2}em ${o.stroke.color}`
+ : undefined,
+ paintOrder: o.stroke ? ("stroke fill" as const) : undefined,
+ textShadow: shadow
+ ? `0 ${shadow.offsetY * scale}px ${shadow.blur * scale}px ${shadow.color}`
+ : undefined,
+ // A behind-speaker title paints on the canvas; the DOM box keeps
+ // its footprint but no visible plate.
+ background: o.plate && !(o.behindSubject && !editing) ? plateFill(o) : undefined,
+ padding: o.plate ? PLATE_PADDING : undefined,
+ borderRadius: o.plate ? `${o.plateRadius ?? PLATE_RADIUS}em` : undefined,
+ };
+ })()
+ : {}),
};
const commitText = () => {
+ if (!isText) return;
const text = (editRef.current?.innerText ?? "").replace(/\n+$/, "");
setEditing(false);
if (text !== o.text) {
@@ -419,11 +558,159 @@ function OverlayItem({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [editing]);
+ /** The whole group's live members (or just this element), snapshotted at a
+ * gesture's start so deltas apply member-relative. */
+ const groupSnapshot = () => {
+ const all = useEditor.getState().overlays;
+ return o.groupId ? all.filter((x) => x.groupId === o.groupId) : [o];
+ };
+ const clampPos = clampOverlayPos;
+
+ /**
+ * Commit a live transform. Position and rotation are pose: on a keyframed
+ * element they land in the key at the playhead — which is what makes
+ * dragging in the preview the way you author motion — while everything else
+ * (a font size, a shape's box) is the element's own and always patches it.
+ */
+ const writeTransform = (
+ patches: { id: string; patch: { x?: number; y?: number; rotation?: number } }[]
+ ) => {
+ const s = useEditor.getState();
+ const direct: { id: string; patch: OverlayPatch }[] = [];
+ for (const { id, patch } of patches) {
+ const el = s.overlays.find((x) => x.id === id);
+ const { x, y, rotation, ...rest } = patch;
+ const posed = x !== undefined || y !== undefined || "rotation" in patch;
+ if (el && posed && hasOverlayKeys(el)) {
+ s.setOverlayKey(
+ id,
+ Math.max(0, Math.min(t - el.start, Math.max(0.1, el.end - el.start))),
+ {
+ ...(x !== undefined ? { x } : {}),
+ ...(y !== undefined ? { y } : {}),
+ // A key stores its angle outright; absence means zero only on the
+ // element itself.
+ ...("rotation" in patch ? { rotation: rotation ?? 0 } : {}),
+ },
+ { transient: true }
+ );
+ if (Object.keys(rest).length) direct.push({ id, patch: rest as OverlayPatch });
+ continue;
+ }
+ direct.push({ id, patch: patch as OverlayPatch });
+ }
+ if (direct.length) s.updateOverlaysTransient(direct);
+ };
+
+ // The corner handle resizes what the kind actually stores: a title's font
+ // size, a shape's box (both axes by the drag ratio), a sticker's width.
+ // A group scales as one: every member's size and its offset from the group
+ // center ride the same factor.
+ const resizeFrom = (e: React.PointerEvent) => {
+ const s = useEditor.getState();
+ s.pushHistory();
+ const box = boxRef.current!.getBoundingClientRect();
+ const cx = box.left + box.width / 2;
+ const cy = box.top + box.height / 2;
+ const d0 = Math.max(8, Math.hypot(e.clientX - cx, e.clientY - cy));
+ const members = groupSnapshot();
+ const gx = members.reduce((sum, m) => sum + m.x, 0) / members.length;
+ const gy = members.reduce((sum, m) => sum + m.y, 0) / members.length;
+ const scaled = (m: Overlay, k: number): Partial<Overlay> => {
+ if (isTextOverlay(m)) {
+ return { size: Math.round(Math.min(320, Math.max(16, m.size * k))) };
+ }
+ if (m.kind === "shape") {
+ return {
+ w: Math.min(2, Math.max(0.01, m.w * k)),
+ h: Math.min(2, Math.max(0.002, m.h * k)),
+ };
+ }
+ if (m.kind === "sticker") return { w: Math.min(1.5, Math.max(0.02, m.w * k)) };
+ return {};
+ };
+ startDrag(e, {
+ onMove: (_dx, _dy, ev) => {
+ const k = Math.hypot(ev.clientX - cx, ev.clientY - cy) / d0;
+ writeTransform(
+ members.map((m) => ({
+ id: m.id,
+ patch: {
+ ...scaled(m, k),
+ ...(members.length > 1
+ ? { x: clampPos(gx + (m.x - gx) * k), y: clampPos(gy + (m.y - gy) * k) }
+ : {}),
+ },
+ }))
+ );
+ },
+ });
+ };
+
+ // The lollipop above the box rotates around the center; plain angles within
+ // 3° of a quarter turn snap to it, and 0 stores as absence. A group orbits
+ // its shared center: positions revolve and each member's rotation shifts by
+ // the same delta.
+ const rotateFrom = (e: React.PointerEvent) => {
+ const s = useEditor.getState();
+ s.pushHistory();
+ const box = boxRef.current!.getBoundingClientRect();
+ const cx = box.left + box.width / 2;
+ const cy = box.top + box.height / 2;
+ const members = groupSnapshot();
+ const gx = members.reduce((sum, m) => sum + m.x, 0) / members.length;
+ const gy = members.reduce((sum, m) => sum + m.y, 0) / members.length;
+ const angleAt = (ev: { clientX: number; clientY: number }) =>
+ (Math.atan2(ev.clientY - cy, ev.clientX - cx) * 180) / Math.PI + 90;
+ const start0 = angleAt(e);
+ const norm = (deg: number) => ((((deg + 180) % 360) + 360) % 360) - 180;
+ // The cursor turns with the element for the whole drag, so its heads keep
+ // pointing the way the next bit of travel will take it.
+ let liveRotation = o.rotation ?? 0;
+ startDrag(e, {
+ cursor: () => rotateCursor(liveRotation),
+ onMove: (_dx, _dy, ev) => {
+ if (members.length === 1) {
+ let deg = norm(angleAt(ev));
+ for (const q of [-180, -90, 0, 90, 180]) {
+ if (Math.abs(deg - q) < 3) deg = q;
+ }
+ const rotation = Math.round(deg);
+ liveRotation = rotation;
+ writeTransform([{ id: o.id, patch: { rotation: rotation === 0 ? undefined : rotation } }]);
+ return;
+ }
+ const delta = norm(angleAt(ev) - start0);
+ liveRotation = norm((o.rotation ?? 0) + delta);
+ const rad = (delta * Math.PI) / 180;
+ // Positions are frame fractions with unequal axes; orbit in a square
+ // space keyed to width so the group turns rigidly on screen.
+ const ax = 1;
+ const ay = stageHeight / stageWidth;
+ writeTransform(
+ members.map((m) => {
+ const ox = (m.x - gx) * ax;
+ const oy = (m.y - gy) * ay;
+ const rotation = Math.round(norm((m.rotation ?? 0) + delta));
+ return {
+ id: m.id,
+ patch: {
+ x: clampPos(gx + (ox * Math.cos(rad) - oy * Math.sin(rad)) / ax),
+ y: clampPos(gy + (ox * Math.sin(rad) + oy * Math.cos(rad)) / ay),
+ rotation: rotation === 0 ? undefined : rotation,
+ },
+ };
+ })
+ );
+ },
+ });
+ };
+
return (
<div
ref={boxRef}
className={cn(
- "overlay-item pointer-events-auto absolute -translate-x-1/2 -translate-y-1/2 cursor-grab rounded-xs text-center whitespace-pre active:cursor-grabbing",
+ "overlay-item pointer-events-auto absolute cursor-grab rounded-xs text-center whitespace-pre active:cursor-grabbing",
selected && "outline-[1.5px] outline-offset-[3px] outline-[#0a84ff]",
editing && "cursor-text"
)}
@@ -431,64 +718,242 @@ function OverlayItem({
onPointerDown={(e) => {
if (editing) return;
const s = useEditor.getState();
- s.select({ kind: "text", id: o.id });
+ s.select({ kind: "overlay", id: o.id });
s.pushHistory();
- const { x, y } = o;
+ // A grouped element drags its whole group: the grabbed one snaps, the
+ // rest follow by the same delta.
+ const members = groupSnapshot().map((m) => ({ id: m.id, x: m.x, y: m.y }));
+ const self = members.find((m) => m.id === o.id) ?? { id: o.id, x: o.x, y: o.y };
startDrag(e, {
onMove: (dx, dy, ev) => {
- const p = snap(o.id, x + dx / stageWidth, y + dy / stageHeight, ev);
- s.updateOverlayTransient(o.id, {
- x: Math.min(0.98, Math.max(0.02, p.x)),
- y: Math.min(0.98, Math.max(0.02, p.y)),
- });
+ const p = snap(o.id, self.x + dx / stageWidth, self.y + dy / stageHeight, ev);
+ const ddx = clampPos(p.x) - self.x;
+ const ddy = clampPos(p.y) - self.y;
+ writeTransform(
+ members.map((m) => ({
+ id: m.id,
+ patch: { x: clampPos(m.x + ddx), y: clampPos(m.y + ddy) },
+ }))
+ );
},
onUp: onSnapEnd,
});
}}
- onDoubleClick={() => setEditing(true)}
+ onDoubleClick={isText ? () => setEditing(true) : undefined}
>
- {editing ? (
- <div
- ref={editRef}
- className="min-w-2 outline-none select-text"
- contentEditable
- suppressContentEditableWarning
- onBlur={commitText}
- onKeyDown={(e) => {
- if (e.key === "Escape") {
- e.preventDefault();
- commitText();
- }
- e.stopPropagation();
- }}
- >
- {o.text}
- </div>
- ) : (
- <span>{o.text}</span>
- )}
+ {behindHidden ? (
+ <span className="opacity-0">{shownText}</span>
+ ) : isText ? (
+ editing ? (
+ <div
+ ref={editRef}
+ className="min-w-2 outline-none select-text"
+ contentEditable
+ suppressContentEditableWarning
+ onBlur={commitText}
+ onKeyDown={(e) => {
+ if (e.key === "Escape") {
+ e.preventDefault();
+ commitText();
+ }
+ e.stopPropagation();
+ }}
+ >
+ {o.text}
+ </div>
+ ) : (
+ <span>{shownText}</span>
+ )
+ ) : o.kind === "shape" ? (
+ <ShapeView shape={o} stageWidth={stageWidth} stageHeight={stageHeight} scale={scale} />
+ ) : o.kind === "sticker" ? (
+ <StickerView sticker={o} stageWidth={stageWidth} t={t} />
+ ) : null}
{selected && !editing && (
- <span
- title="Drag to resize"
- className="overlay-resize absolute -right-2 -bottom-2 size-[13px] cursor-nwse-resize rounded-full border-[2.5px] border-[#0a84ff] bg-white shadow-[0_1px_4px_rgba(0,0,0,0.4)]"
- onPointerDown={(e) => {
- const s = useEditor.getState();
- s.pushHistory();
- const box = boxRef.current!.getBoundingClientRect();
- const cx = box.left + box.width / 2;
- const cy = box.top + box.height / 2;
- const d0 = Math.max(8, Math.hypot(e.clientX - cx, e.clientY - cy));
- const size0 = o.size;
- startDrag(e, {
- onMove: (_dx, _dy, ev) => {
- const d = Math.hypot(ev.clientX - cx, ev.clientY - cy);
- const size = Math.round(Math.min(320, Math.max(16, (size0 * d) / d0)));
- s.updateOverlayTransient(o.id, { size });
- },
- });
- }}
- />
+ <>
+ {/* The grab zone is wider than the dot, so the rotate cursor shows
+ as the pointer approaches rather than only dead on it. It clears
+ the element's own top edge, so moving is never caught by it. */}
+ <span
+ title="Drag to rotate"
+ className="overlay-rotate absolute -top-8 left-1/2 grid size-7 -translate-x-1/2 place-items-center"
+ // The handle rides the element's rotation, so the cursor turns
+ // with it and its heads keep pointing along the drag.
+ style={{ cursor: rotateCursor(live?.rotation ?? o.rotation ?? 0) }}
+ onPointerDown={rotateFrom}
+ >
+ <span className="size-[13px] rounded-full border-[2.5px] border-[#0a84ff] bg-white shadow-[0_1px_4px_rgba(0,0,0,0.4)]" />
+ </span>
+ <span
+ title="Drag to resize"
+ className="overlay-resize absolute -right-2 -bottom-2 size-[13px] cursor-nwse-resize rounded-full border-[2.5px] border-[#0a84ff] bg-white shadow-[0_1px_4px_rgba(0,0,0,0.4)]"
+ onPointerDown={resizeFrom}
+ />
+ </>
)}
</div>
);
}
+
+/** A shape drawn as inline SVG with the painter's own pixel geometry
+ * (`shapeMetrics` in stage space), so preview and export burn-in match. */
+function ShapeView({
+ shape: o,
+ stageWidth,
+ stageHeight,
+ scale,
+}: {
+ shape: ShapeOverlay;
+ stageWidth: number;
+ stageHeight: number;
+ scale: number;
+}) {
+ const m = shapeMetrics(o, { width: stageWidth, height: stageHeight, scale });
+ if (o.shape === "line" || o.shape === "arrow") {
+ const h = Math.max(m.thickness, m.headHalf * 2);
+ const mid = h / 2;
+ return (
+ <svg
+ width={m.w}
+ height={h}
+ className="block overflow-visible"
+ style={{ pointerEvents: "none" }}
+ >
+ <line
+ x1={m.thickness / 2}
+ y1={mid}
+ x2={o.shape === "arrow" ? m.w - m.headLen : m.w - m.thickness / 2}
+ y2={mid}
+ stroke={o.fill}
+ strokeWidth={m.thickness}
+ strokeLinecap="round"
+ />
+ {o.shape === "arrow" && (
+ <polygon
+ points={`${m.w},${mid} ${m.w - m.headLen},${mid - m.headHalf} ${m.w - m.headLen},${mid + m.headHalf}`}
+ fill={o.fill}
+ />
+ )}
+ </svg>
+ );
+ }
+ return (
+ <svg width={m.w} height={m.h} className="block overflow-visible" style={{ pointerEvents: "none" }}>
+ {o.shape === "rect" ? (
+ <rect
+ x={0}
+ y={0}
+ width={m.w}
+ height={m.h}
+ rx={m.radius}
+ fill={o.fill}
+ fillOpacity={o.fillOpacity ?? 1}
+ stroke={o.stroke?.color}
+ strokeWidth={m.strokeWidth || undefined}
+ />
+ ) : (
+ <ellipse
+ cx={m.w / 2}
+ cy={m.h / 2}
+ rx={m.w / 2}
+ ry={m.h / 2}
+ fill={o.fill}
+ fillOpacity={o.fillOpacity ?? 1}
+ stroke={o.stroke?.color}
+ strokeWidth={m.strokeWidth || undefined}
+ />
+ )}
+ </svg>
+ );
+}
+
+/** A sticker: the project image at its stored width (height from the asset's
+ * own aspect), or a Lottie animation frame-seeked from timeline time. */
+function StickerView({
+ sticker: o,
+ stageWidth,
+ t,
+}: {
+ sticker: StickerOverlay;
+ stageWidth: number;
+ t: number;
+}) {
+ const asset = useEditor((s) =>
+ o.assetId ? s.assets.find((a) => a.id === o.assetId) : undefined
+ );
+ if (!asset) return <span className="block size-8 rounded bg-white/20" />;
+ if (o.lottie) {
+ return <LottieView asset={asset} width={o.w * stageWidth} tLocal={Math.max(0, t - o.start)} />;
+ }
+ return (
+ // eslint-disable-next-line @next/next/no-img-element -- project media blob/engine URL
+ <img
+ src={asset.url}
+ alt={asset.name}
+ draggable={false}
+ className="block max-w-none select-none"
+ style={{ width: o.w * stageWidth, height: "auto" }}
+ />
+ );
+}
+
+/** A Lottie sticker in the preview: its own player instance, hard-seeked to
+ * the timeline moment (goToAndStop — never free-running), destroyed on
+ * unmount. Out-of-range elements don't mount, which caps live instances. */
+function LottieView({
+ asset,
+ width,
+ tLocal,
+}: {
+ asset: { id: string; url: string; fileName: string; name: string };
+ width: number;
+ tLocal: number;
+}) {
+ const canvasRef = useRef<HTMLCanvasElement>(null);
+ const handleRef = useRef<LottieHandle | null>(null);
+ const [ready, setReady] = useState(0);
+ const [aspect, setAspect] = useState(1);
+
+ useEffect(() => {
+ let live = true;
+ void import("@/cut/lib/lottieAssets").then(async (m) => {
+ const full = useEditor.getState().assets.find((a) => a.id === asset.id);
+ const handle = full ? await m.newLottieInstance(full) : null;
+ if (!live) {
+ handle?.destroy();
+ return;
+ }
+ handleRef.current = handle;
+ if (handle) setAspect(handle.width / Math.max(1, handle.height));
+ setReady((n) => n + 1);
+ });
+ return () => {
+ live = false;
+ handleRef.current?.destroy();
+ handleRef.current = null;
+ };
+ }, [asset.id]);
+
+ useEffect(() => {
+ const handle = handleRef.current;
+ const canvas = canvasRef.current;
+ if (!handle || !canvas) return;
+ const frame = handle.seek(tLocal);
+ if (canvas.width !== frame.width || canvas.height !== frame.height) {
+ canvas.width = frame.width;
+ canvas.height = frame.height;
+ }
+ const ctx = canvas.getContext("2d");
+ if (!ctx) return;
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ ctx.drawImage(frame, 0, 0);
+ }, [tLocal, ready]);
+
+ return (
+ <canvas
+ ref={canvasRef}
+ className="block max-w-none select-none"
+ style={{ width, height: width / aspect }}
+ />
+ );
+}
diff --git a/site/src/cut/components/Preview.tsx b/site/src/cut/components/Preview.tsx
index ad423a3a..963f8ca6 100644
--- a/site/src/cut/components/Preview.tsx
+++ b/site/src/cut/components/Preview.tsx
@@ -15,6 +15,7 @@ import { setPreviewCanvas } from "@/cut/lib/previewCanvas";
import { frameOf, isFullRect, rectOf, type Aspect, type ClipSpan, type FrameRect, type MediaAsset, type VideoClip } from "@/cut/lib/types";
import { cn } from "@/lib/utils";
import { OverlayLayer } from "./OverlayLayer";
+import { StageEffectPaint, stageSlices, useEffectLanes, useStageEffects } from "./StageEffects";
/** The clip under the playhead, when it overflows the frame in fill mode. */
function pannableSpan(s: {
@@ -77,6 +78,12 @@ export function Preview() {
const frame = frameOf(aspect);
usePlayback(canvasRef);
+ // An effect grades what plays under it, so the stage is built in slices:
+ // the picture, then the elements of each lane band with the look of the
+ // effects above them, and each effect's paints sitting where the effect does.
+ const stageFx = useStageEffects();
+ const effectLanes = useEffectLanes();
+ const { picture, slices } = stageSlices(stageFx, effectLanes);
useEffect(() => {
setPreviewCanvas(canvasRef.current);
@@ -193,6 +200,7 @@ export function Preview() {
width={frame.w}
height={frame.h}
className="block size-full"
+ style={{ transform: picture.transform, filter: picture.filter }}
// Drag the viewport to reference what's on screen: the clip under
// the playhead travels as a media drag (timeline placement, chat
// attachment, generation reference). Pan on a fill clip wins —
@@ -211,7 +219,21 @@ export function Preview() {
onDragEnd={clearAssetDrag}
/>
<OverlayPipHandle stage={stage} />
- <OverlayLayer stageWidth={stage.w} />
+ {slices.map((slice) =>
+ slice.kind === "elements" ? (
+ <OverlayLayer
+ key={slice.key}
+ stageWidth={stage.w}
+ transform={slice.transform}
+ filter={slice.filter}
+ from={slice.from}
+ to={slice.to}
+ captions={slice.captions}
+ />
+ ) : (
+ <StageEffectPaint key={slice.key} states={slice.states} />
+ )
+ )}
</div>
</div>
</section>
diff --git a/site/src/cut/components/StageEffects.tsx b/site/src/cut/components/StageEffects.tsx
new file mode 100644
index 00000000..6008371a
--- /dev/null
+++ b/site/src/cut/components/StageEffects.tsx
@@ -0,0 +1,97 @@
+"use client";
+
+import { useMemo } from "react";
+import { effectPreviewState, type EffectPreviewState } from "@donkeycut/effects-kit";
+import { useEditor } from "@/cut/lib/store";
+import { isEffectOverlay, laneOf } from "@/cut/lib/types";
+import type { LiveEffect } from "@/cut/lib/effectStack";
+
+/**
+ * Effect elements in the preview's stack.
+ *
+ * An effect filters whatever plays under it: the footage and every element on
+ * a lane below its own. What that means for the stage — which slice wears
+ * which grade — is `effectStack.ts`; this file reads the document for it and
+ * paints the frame-wide passes (grain, vignette, leak wash, flash) that sit
+ * where the effect does.
+ *
+ * Both exports walk the same stack, so what plays here is what renders.
+ */
+
+export { stageEffectTransform, stageSlices, type LiveEffect, type StageSlice } from "@/cut/lib/effectStack";
+
+/** Every effect live at `t`, deepest lane last. Skimming previews them too,
+ * the same as the elements beside them. */
+export function useStageEffects(): LiveEffect[] {
+ const overlays = useEditor((s) => s.overlays);
+ const currentTime = useEditor((s) => s.currentTime);
+ const skimTime = useEditor((s) => s.skimTime);
+ const playing = useEditor((s) => s.playing);
+ const t = !playing && skimTime !== null ? skimTime : currentTime;
+ return useMemo(
+ () =>
+ overlays
+ .filter(isEffectOverlay)
+ .filter((o) => !o.hidden && t >= o.start && t <= o.end)
+ .map((o) => ({
+ lane: laneOf(o),
+ state: effectPreviewState(o.effect, o.amount, t - o.start, o.focus, o.ramp, o.end - o.start),
+ }))
+ .sort((a, b) => a.lane - b.lane),
+ [overlays, t]
+ );
+}
+
+/** The lanes effects sit on, whether or not one is live right now. The stage
+ * splits on these, so its slices stay put while the playhead moves and only
+ * change when a row does. */
+export function useEffectLanes(): number[] {
+ const overlays = useEditor((s) => s.overlays);
+ return useMemo(
+ () => [...new Set(overlays.filter(isEffectOverlay).map(laneOf))].sort((a, b) => a - b),
+ [overlays]
+ );
+}
+
+/** The frame-wide paints of the effects on one lane, over everything under
+ * them and under everything above. */
+export function StageEffectPaint({ states }: { states: EffectPreviewState[] }) {
+ if (states.length === 0) return null;
+ const grain = Math.max(0, ...states.map((s) => s.grain ?? 0));
+ const vignette = Math.max(0, ...states.map((s) => s.vignette ?? 0));
+ const flash = Math.max(0, ...states.map((s) => s.flash ?? 0));
+ const washes = states.flatMap((s) => s.washes ?? []);
+ if (!grain && !vignette && !flash && washes.length === 0) return null;
+ return (
+ <div className="pointer-events-none absolute inset-0">
+ {grain > 0 && (
+ <div
+ className="absolute inset-0 bg-[radial-gradient(circle_at_1px_1px,#fff_0.6px,transparent_0)] bg-[length:3px_3px]"
+ style={{ opacity: grain }}
+ />
+ )}
+ {vignette > 0 && (
+ <div
+ className="absolute inset-0"
+ style={{
+ background: `radial-gradient(circle at 50% 50%, transparent 35%, rgba(0,0,0,${vignette.toFixed(3)}) 100%)`,
+ }}
+ />
+ )}
+ {washes.map((w, i) => (
+ <div
+ key={i}
+ className="absolute inset-0"
+ style={{
+ background: w.color,
+ opacity: w.alpha,
+ // The recipe names its wash with a canvas composite op; the ones
+ // effects use are CSS blend modes by the same name.
+ mixBlendMode: w.mode as React.CSSProperties["mixBlendMode"],
+ }}
+ />
+ ))}
+ {flash > 0 && <div className="absolute inset-0 bg-white" style={{ opacity: flash }} />}
+ </div>
+ );
+}
diff --git a/site/src/cut/components/Stickers.tsx b/site/src/cut/components/Stickers.tsx
new file mode 100644
index 00000000..70bf62c3
--- /dev/null
+++ b/site/src/cut/components/Stickers.tsx
@@ -0,0 +1,289 @@
+"use client";
+
+import { useCallback, useMemo, useState } from "react";
+import { Copy, Loader2, Plus, Sparkles, Trash2 } from "lucide-react";
+import { Button } from "@/components/ui/button";
+import { DropdownMenuItem } from "@/components/ui/dropdown-menu";
+import { clearAssetDrag, setAssetDragData } from "@/cut/lib/assetDrag";
+import { PICKED_RING, useAssetPick } from "@/cut/lib/assetPick";
+import {
+ addRefOnce,
+ collectRefs,
+ mentionToken,
+ sameRef,
+ upsertRef,
+ useAssetDrop,
+ useRefCandidates,
+ type AssetRef,
+} from "@/cut/lib/assetRef";
+import { signInUrl, useSignedIn } from "@/cut/lib/generate";
+import { isLottieAsset } from "@/cut/lib/lottieAssets";
+import { refsFromDroppedFiles } from "@/cut/lib/refMedia";
+import { createCustomSticker, type StickerStage } from "@/cut/lib/stickerCreate";
+import { useEditor } from "@/cut/lib/store";
+import type { MediaAsset } from "@/cut/lib/types";
+import { cn } from "@/lib/utils";
+import { CopyHandlePill, MentionTextarea, RefChips } from "./AssetRefs";
+import { GeneratedAssetMenu } from "./GeneratedAssetMenu";
+import { DictationControl } from "./MicDictation";
+import { HostedErrorText } from "./hostedError";
+
+/**
+ * The project's sticker images — generated cutouts, all tagged origin
+ * "sticker" so they live in the Elements panel and never in Media. The pieces
+ * here are what that panel's Stickers category is built from: the list, the
+ * tile, and the generate composer.
+ */
+export function useProjectStickers(): {
+ stickers: MediaAsset[];
+ /** The @name a prompt would reach a sticker by, for its tile's copy pill. */
+ handleOf: (id: string) => string | undefined;
+} {
+ // Select the array itself and narrow outside the selector: a `filter` inside
+ // returns a fresh array every read, which the store's snapshot check reads
+ // as a change on every render.
+ const assets = useEditor((s) => s.assets);
+ const stickers = useMemo(() => assets.filter((a) => a.origin === "sticker"), [assets]);
+ const candidates = useRefCandidates();
+ return {
+ stickers,
+ handleOf: (id) => candidates.find((c) => c.scope === "project" && c.id === id)?.handle,
+ };
+}
+
+/** A hover button in the tile's top-right corner, as the generated-image tiles
+ * wear them. */
+const ROUND_ACTION =
+ "grid size-5 place-items-center rounded-full bg-black/45 text-white opacity-0 transition-opacity group-hover:opacity-100 hover:bg-black/65";
+
+/** One sticker: click picks it, the hover buttons act on it, and it drags into
+ * any composer as a reference. The picking, the @tag and the drag are the same
+ * as the Image and Video tabs — see `assetPick.ts`. */
+export function StickerTile({
+ asset: a,
+ projectId,
+ handle,
+ readOnly,
+}: {
+ asset: MediaAsset;
+ projectId: string;
+ handle?: string;
+ readOnly: boolean;
+}) {
+ const { picked, pick } = useAssetPick(a.id);
+ const lottie = isLottieAsset(a);
+ return (
+ <div className={cn("group relative rounded-lg", picked && PICKED_RING)}>
+ <button
+ title={a.name}
+ className="block w-full overflow-hidden rounded-lg bg-muted outline-none focus-visible:ring-2 focus-visible:ring-ring"
+ draggable
+ onDragStart={(e) => {
+ setAssetDragData(e, a.id);
+ // The tile is the ghost, so it keeps its corner radius instead of
+ // the browser's square white frame.
+ const r = e.currentTarget.getBoundingClientRect();
+ e.dataTransfer.setDragImage(e.currentTarget, e.clientX - r.left, e.clientY - r.top);
+ }}
+ onDragEnd={clearAssetDrag}
+ onClick={pick}
+ >
+ {lottie ? (
+ <span className="grid aspect-square w-full place-items-center p-1 text-center">
+ <span className="min-w-0">
+ <span className="block text-[10px] font-semibold text-muted-foreground">Lottie</span>
+ <span className="block truncate text-[10px]" title={a.name}>
+ {a.name.replace(/\.json$/i, "")}
+ </span>
+ </span>
+ </span>
+ ) : (
+ // eslint-disable-next-line @next/next/no-img-element -- project media URL
+ <img
+ src={a.url}
+ alt={a.name}
+ loading="lazy"
+ className="aspect-square w-full object-contain p-1 transition-transform group-hover:scale-[1.06]"
+ />
+ )}
+ </button>
+ {/* The handle and the actions share one row, so the pill's taller box
+ can't sit off the buttons' center across the tile. */}
+ <div className="pointer-events-none absolute inset-x-1 top-1 flex items-center justify-between gap-1">
+ <CopyHandlePill handle={handle} name={a.name} className="pointer-events-auto static min-w-0" />
+ {!readOnly && (
+ <div className="pointer-events-auto flex shrink-0 items-center gap-1">
+ <button
+ title="Add to timeline"
+ aria-label={`Add ${a.name}`}
+ className={ROUND_ACTION}
+ onClick={() =>
+ useEditor
+ .getState()
+ .addSticker(lottie ? { assetId: a.id, lottie: true } : { assetId: a.id })
+ }
+ >
+ <Plus className="size-3" />
+ </button>
+ <GeneratedAssetMenu
+ asset={a}
+ projectId={projectId}
+ triggerClassName={cn(ROUND_ACTION, "data-popup-open:opacity-100")}
+ before={
+ <DropdownMenuItem
+ onClick={() =>
+ void navigator.clipboard
+ .writeText(handle ? `@${handle}` : mentionToken(a.name))
+ .catch(() => {})
+ }
+ >
+ <Copy /> Copy reference
+ </DropdownMenuItem>
+ }
+ after={
+ <DropdownMenuItem
+ variant="destructive"
+ onClick={() => useEditor.getState().removeAsset(a.id)}
+ >
+ <Trash2 /> Delete
+ </DropdownMenuItem>
+ }
+ />
+ </div>
+ )}
+ </div>
+ </div>
+ );
+}
+
+const STAGE_LABEL: Record<StickerStage, string> = {
+ generating: "Drawing…",
+ cutting: "Cutting out…",
+ saving: "Saving…",
+};
+
+/**
+ * "Create a sticker": prompt → hosted image generation → background removal
+ * (the plain-backdrop knockout first, then matting) → white die-cut outline →
+ * saved as an origin-"sticker" asset and dropped on the timeline. The composer
+ * is the generate-image one: same box, same dictation, same full-width button,
+ * so the two read as one way of asking for a picture.
+ */
+export function CreateSticker({ projectId }: { projectId: string }) {
+ const [prompt, setPrompt] = useState("");
+ const [refs, setRefs] = useState<AssetRef[]>([]);
+ const [stage, setStage] = useState<StickerStage | null>(null);
+ const [error, setError] = useState<string | null>(null);
+ const signedIn = useSignedIn();
+ const candidates = useRefCandidates();
+ const busy = stage !== null;
+
+ const addRef = useCallback((ref: AssetRef) => setRefs((rs) => addRefOnce(rs, ref)), []);
+ const updateRef = useCallback((ref: AssetRef) => setRefs((rs) => upsertRef(rs, ref)), []);
+ const removeRef = useCallback(
+ (ref: AssetRef) => setRefs((rs) => rs.filter((r) => !sameRef(r, ref))),
+ []
+ );
+ // OS files handed to the composer — dropped on it or pasted into the prompt —
+ // attach as references; media files import into the project on the way.
+ const attachFiles = useCallback(
+ (files: File[]) =>
+ void refsFromDroppedFiles(projectId, files).then((next) => next.forEach(addRef)),
+ [projectId, addRef]
+ );
+ const { active: dropActive, attachTarget, targetProps } = useAssetDrop(addRef, attachFiles);
+
+ const create = async () => {
+ const { text, refs: all } = collectRefs(prompt.trim(), refs, candidates);
+ if (!text || busy) return;
+ setError(null);
+ try {
+ const asset = await createCustomSticker(projectId, text, setStage, { refs: all });
+ useEditor.getState().addSticker({ assetId: asset.id });
+ setPrompt("");
+ setRefs([]);
+ } catch (err) {
+ setError(err instanceof Error ? err.message : "Could not create the sticker.");
+ } finally {
+ setStage(null);
+ }
+ };
+
+ return (
+ <div
+ ref={attachTarget}
+ {...targetProps}
+ className="relative flex shrink-0 flex-col gap-3"
+ >
+ {/* The generate-image composer: references ride as thumbnails inside the
+ box, above the prompt, and @name reaches anything in the project. */}
+ <div
+ className={cn(
+ "relative flex shrink-0 flex-col rounded-lg border border-input focus-within:border-ring",
+ dropActive && "border-[#0a84ff] ring-2 ring-[#0a84ff]/30 ring-inset"
+ )}
+ >
+ <RefChips
+ refs={refs}
+ onRemove={removeRef}
+ onUpdate={updateRef}
+ className="p-2 pb-0"
+ peekSide="bottom"
+ />
+ <MentionTextarea
+ className="min-h-[100px] w-full resize-y bg-transparent px-2.5 py-2 pr-9 text-[12.5px] leading-relaxed outline-none"
+ placeholder="A corgi in sunglasses… Drop an image in or type @ to reference one."
+ value={prompt}
+ onChange={setPrompt}
+ candidates={candidates}
+ submitKey="mod-enter"
+ menuSide="bottom"
+ onSubmit={() => void create()}
+ attachedRefs={refs}
+ onUpsertRef={updateRef}
+ onPasteFiles={attachFiles}
+ onRemoveLastRef={() => {
+ const last = refs[refs.length - 1];
+ if (last) removeRef(last);
+ }}
+ />
+ <DictationControl
+ text={prompt}
+ onResult={setPrompt}
+ buttonClassName="absolute right-2 bottom-2 bg-background/70 backdrop-blur-sm"
+ />
+ </div>
+
+ <div className="flex shrink-0 items-center gap-1.5">
+ <Button
+ className="min-w-0 flex-1"
+ disabled={!prompt.trim() || busy || signedIn === false}
+ onClick={() => void create()}
+ >
+ {stage ? (
+ <Loader2 data-icon="inline-start" className="animate-spin" />
+ ) : (
+ <Sparkles data-icon="inline-start" />
+ )}
+ {stage ? STAGE_LABEL[stage] : "Create sticker"}
+ </Button>
+ </div>
+
+ {signedIn === false && (
+ <p className="shrink-0 text-[11px] leading-relaxed text-muted-foreground">
+ Generating runs on your Donkey account.{" "}
+ <a className="font-medium text-blue-600 hover:underline dark:text-blue-400" href={signInUrl()}>
+ Sign in
+ </a>{" "}
+ to continue.
+ </p>
+ )}
+
+ {error && (
+ <div className="shrink-0 text-[10.5px] leading-snug break-words text-red-600">
+ <HostedErrorText error={error} />
+ </div>
+ )}
+ </div>
+ );
+}
diff --git a/site/src/cut/lib/behindPass.ts b/site/src/cut/lib/behindPass.ts
new file mode 100644
index 00000000..d0cfcea2
--- /dev/null
+++ b/site/src/cut/lib/behindPass.ts
@@ -0,0 +1,179 @@
+"use client";
+
+/**
+ * The text-behind-speaker pass, shared by the live preview and the in-tab
+ * export: with the video layers already on the canvas, snapshot them, draw
+ * the behind-tagged title rasters, then segment the snapshot and draw the
+ * person back on top. When no person registers the text simply stays on top —
+ * the effect degrades to a normal front title instead of hiding words.
+ */
+
+import { evalOverlayFrame } from "@donkeycut/effects-kit";
+import { personSegmenter, segmentSubjectAlpha } from "./cutout";
+import { renderElementPng } from "./textRender";
+import { isTextOverlay, type MediaAsset, type Overlay, type TextOverlay } from "./types";
+
+type Segmenter = import("@mediapipe/tasks-vision").ImageSegmenter;
+
+/** Segmentation input width — small on purpose; this runs per frame. */
+const SEG_WIDTH = 256;
+
+/** The behind-tagged titles live at `t`. */
+export function behindOverlaysAt(overlays: Overlay[], t: number): TextOverlay[] {
+ return overlays.filter(
+ (o): o is TextOverlay =>
+ isTextOverlay(o) &&
+ !!o.behindSubject &&
+ !o.hidden &&
+ !!o.text.trim() &&
+ t >= o.start &&
+ t <= o.end
+ );
+}
+
+/** Whether any element in the document wants the behind pass at all. */
+export function hasBehindOverlays(overlays: Overlay[]): boolean {
+ return overlays.some((o) => isTextOverlay(o) && !!o.behindSubject && !o.hidden);
+}
+
+export class BehindCompositor {
+ private segmenter: Segmenter | null = null;
+ private segKicked = false;
+ private rasters = new WeakMap<TextOverlay, ImageBitmap>();
+ private pending = new WeakSet<TextOverlay>();
+ private person: HTMLCanvasElement | null = null;
+ private small: HTMLCanvasElement | null = null;
+ private mask: { at: number; alpha: HTMLCanvasElement | null } = { at: -1e9, alpha: null };
+
+ /** Kick the (shared) segmenter load; safe to call every frame. */
+ private ensureSegmenter() {
+ if (this.segKicked) return;
+ this.segKicked = true;
+ void personSegmenter().then((s) => {
+ this.segmenter = s;
+ });
+ }
+
+ private rasterFor(
+ o: TextOverlay,
+ w: number,
+ h: number,
+ assets: MediaAsset[]
+ ): ImageBitmap | null {
+ const hit = this.rasters.get(o);
+ if (hit) return hit;
+ if (!this.pending.has(o)) {
+ this.pending.add(o);
+ // Neutral picture: position aside, the per-frame pose owns rotation and
+ // opacity, so baking them here would apply each of them twice.
+ void renderElementPng({ ...o, rotation: undefined, opacity: undefined }, w, h, assets)
+ .then((png) => createImageBitmap(png))
+ .then((bmp) => this.rasters.set(o, bmp))
+ .catch(() => {});
+ }
+ return null;
+ }
+
+ /** Export path: everything resident before the first frame draws. */
+ async prepare(overlays: Overlay[], w: number, h: number, assets: MediaAsset[]): Promise<void> {
+ this.ensureSegmenter();
+ await personSegmenter().then((s) => {
+ this.segmenter = s;
+ });
+ const behind = overlays.filter(
+ (o): o is TextOverlay => isTextOverlay(o) && !!o.behindSubject && !o.hidden
+ );
+ await Promise.all(
+ behind.map(async (o) => {
+ if (this.rasters.get(o)) return;
+ try {
+ const png = await renderElementPng(
+ { ...o, rotation: undefined, opacity: undefined },
+ w,
+ h,
+ assets
+ );
+ this.rasters.set(o, await createImageBitmap(png));
+ } catch {
+ // The overlay just draws in front when its raster is missing.
+ }
+ })
+ );
+ }
+
+ /**
+ * Run the pass on `canvas` (video layers already drawn). `minMaskInterval`
+ * throttles segmentation for the live preview; pass 0 for exports.
+ */
+ draw(
+ canvas: HTMLCanvasElement | OffscreenCanvas,
+ overlays: Overlay[],
+ assets: MediaAsset[],
+ t: number,
+ opts: { minMaskInterval?: number } = {}
+ ): void {
+ const active = behindOverlaysAt(overlays, t);
+ if (active.length === 0) return;
+ this.ensureSegmenter();
+ const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
+ if (!ctx) return;
+ const W = canvas.width;
+ const H = canvas.height;
+
+ // The person source: the video pixels before any text lands on them.
+ if (!this.person || this.person.width !== W || this.person.height !== H) {
+ this.person = document.createElement("canvas");
+ this.person.width = W;
+ this.person.height = H;
+ }
+ const pctx = this.person.getContext("2d")!;
+ pctx.globalCompositeOperation = "source-over";
+ pctx.clearRect(0, 0, W, H);
+ pctx.drawImage(canvas, 0, 0);
+
+ // The behind text, drawn straight onto the composite: a neutral raster
+ // under the element's pose for this moment, exactly like the export's
+ // stamped layers.
+ const scale = Math.min(W, H) / 1080;
+ for (const o of active) {
+ const bmp = this.rasterFor(o, W, H, assets);
+ if (!bmp) continue;
+ const ev = evalOverlayFrame(o, Math.max(0, t - o.start));
+ if (ev.opacity <= 0.001) continue;
+ const cx = o.x * W;
+ const cy = o.y * H;
+ ctx.save();
+ ctx.globalAlpha = ev.opacity;
+ ctx.translate(ev.x * W + ev.dx * scale, ev.y * H + ev.dy * scale);
+ ctx.rotate((ev.rotation * Math.PI) / 180);
+ ctx.scale(ev.scale, ev.scale);
+ ctx.translate(-cx, -cy);
+ ctx.drawImage(bmp, 0, 0, W, H);
+ ctx.restore();
+ }
+
+ // The person back on top. Segmentation is throttled: between refreshes
+ // the previous mask rides the new frame (the subject moves little in
+ // 1/15s, and a lagging mask beats a stuttering preview).
+ if (!this.segmenter) return;
+ const minInterval = opts.minMaskInterval ?? 1 / 15;
+ if (!this.mask.alpha || Math.abs(t - this.mask.at) >= minInterval) {
+ if (!this.small) this.small = document.createElement("canvas");
+ const sw = SEG_WIDTH;
+ const sh = Math.max(2, Math.round((SEG_WIDTH * H) / W));
+ if (this.small.width !== sw || this.small.height !== sh) {
+ this.small.width = sw;
+ this.small.height = sh;
+ }
+ const sctx = this.small.getContext("2d", { willReadFrequently: true })!;
+ sctx.drawImage(this.person, 0, 0, sw, sh);
+ this.mask = { at: t, alpha: segmentSubjectAlpha(this.segmenter, this.small) };
+ }
+ if (!this.mask.alpha) return; // no person in shot: the text stays in front
+ pctx.globalCompositeOperation = "destination-in";
+ pctx.imageSmoothingEnabled = true;
+ pctx.drawImage(this.mask.alpha, 0, 0, W, H);
+ pctx.globalCompositeOperation = "source-over";
+ ctx.drawImage(this.person, 0, 0);
+ }
+}
diff --git a/site/src/cut/lib/cutout.ts b/site/src/cut/lib/cutout.ts
new file mode 100644
index 00000000..48083f8f
--- /dev/null
+++ b/site/src/cut/lib/cutout.ts
@@ -0,0 +1,298 @@
+"use client";
+
+/**
+ * Sticker cutouts: turn a picture into a transparent-background sticker.
+ *
+ * Two matting paths share one shape. People run through MediaPipe's
+ * self-hosted person segmenter — free, on-device, offline (the same model
+ * text-behind-speaker uses). Everything else goes to the hosted inference
+ * surface: the vision model returns segmentation masks (box + base64 mask
+ * PNG per subject) and the alpha is applied here. Both end in the same
+ * finishing pass — crop to content, then an optional white die-cut outline —
+ * which is the kit's, since shaping a matted picture needs no host.
+ */
+
+import {
+ applyAlpha,
+ canvasOf,
+ cropToContent,
+ keepLargestSubject,
+ removeFlatBackdrop,
+ withOutline,
+} from "@donkeycut/effects-kit";
+import { hostedPost } from "./hosted";
+import { geminiModelRoles } from "@/lib/inference/gemini-models";
+
+const WASM_BASE = "/mediapipe/wasm";
+const PERSON_MODEL = "/mediapipe/selfie_segmenter.tflite";
+/** Confidence above which a pixel counts as subject. */
+const PERSON_THRESHOLD = 0.35;
+type Segmenter = import("@mediapipe/tasks-vision").ImageSegmenter;
+
+let segmenterOnce: Promise<Segmenter | null> | null = null;
+
+/**
+ * Give the wasm module somewhere quiet to log.
+ *
+ * TFLite writes its start-up notes ("INFO: Created TensorFlow Lite XNNPACK
+ * delegate for CPU.", the GL and feedback-manager warnings) to the module's
+ * stderr, and Emscripten binds stderr to `console.error` as the glue script
+ * evaluates. Next's dev overlay classifies by channel rather than severity, so
+ * an INFO line arrives on screen as a page error, pinned to whichever frame
+ * happened to be running — in practice the behind-speaker pass, mid-playback.
+ *
+ * Emscripten reads `print`/`printErr` off the module object, and the task
+ * runner passes a pre-set `self.Module` through to the factory (copying its own
+ * `locateFile` onto it and clearing the global afterwards). Pointing those at
+ * `console.debug` keeps the notes readable under verbose logging and off the
+ * error channel. Patching `console.error` around the call cannot work: the glue
+ * captured the original binding before we could reach it.
+ */
+type EmscriptenScope = typeof globalThis & { Module?: Record<string, unknown> };
+
+async function withQuietWasmLogs<T>(create: () => Promise<T>): Promise<T> {
+ const scope = globalThis as EmscriptenScope;
+ const prior = scope.Module;
+ const note = (...args: unknown[]) => console.debug("[mediapipe]", ...args);
+ scope.Module = { print: note, printErr: note };
+ try {
+ return await create();
+ } finally {
+ if (prior === undefined) delete scope.Module;
+ else scope.Module = prior;
+ }
+}
+
+/** Build the inference delegate at load time. TFLite defers it to the first
+ * inference, which would otherwise land on the first playback frame that needs
+ * a matte. */
+const WARM_PX = 16;
+
+function warmSegmenter(segmenter: Segmenter): void {
+ const [canvas] = canvasOf(WARM_PX, WARM_PX);
+ try {
+ segmenter.segment(canvas).close();
+ } catch {
+ // A warm-up that fails is not a reason to withhold a working segmenter.
+ }
+}
+
+/** The shared IMAGE-mode person segmenter, created on first use. Null when
+ * the runtime assets are missing (segmentation quietly unavailable). */
+export function personSegmenter(): Promise<Segmenter | null> {
+ segmenterOnce ??= (async () => {
+ try {
+ const { FilesetResolver, ImageSegmenter } = await import("@mediapipe/tasks-vision");
+ const fileset = await FilesetResolver.forVisionTasks(WASM_BASE);
+ const segmenter = await withQuietWasmLogs(() =>
+ ImageSegmenter.createFromOptions(fileset, {
+ baseOptions: { modelAssetPath: PERSON_MODEL },
+ runningMode: "IMAGE",
+ outputCategoryMask: false,
+ outputConfidenceMasks: true,
+ })
+ );
+ warmSegmenter(segmenter);
+ return segmenter;
+ } catch {
+ segmenterOnce = null;
+ return null;
+ }
+ })();
+ return segmenterOnce;
+}
+
+async function decode(blob: Blob): Promise<HTMLImageElement | null> {
+ return new Promise((resolve) => {
+ const url = URL.createObjectURL(blob);
+ const img = new Image();
+ img.onload = () => {
+ URL.revokeObjectURL(url);
+ resolve(img);
+ };
+ img.onerror = () => {
+ URL.revokeObjectURL(url);
+ resolve(null);
+ };
+ img.src = url;
+ });
+}
+
+function pngOf(canvas: HTMLCanvasElement): Promise<Blob | null> {
+ return new Promise((resolve) => canvas.toBlob(resolve, "image/png"));
+}
+
+/**
+ * Segment one frame and return the subject as an alpha canvas at the mask's
+ * own (small) resolution — white where the person is, transparent elsewhere.
+ * Scaling it over a frame with `destination-in` cuts the person out; drawing
+ * it into a mask video encodes the same information as luma. Null when no
+ * segmenter or no clear person. The `source` should be small (≤ ~512px) —
+ * the model resamples anyway, and segmentation runs per preview frame.
+ */
+export function segmentSubjectAlpha(
+ segmenter: Segmenter,
+ source: HTMLCanvasElement
+): HTMLCanvasElement | null {
+ const result = segmenter.segment(source);
+ try {
+ const mask = result.confidenceMasks?.[0];
+ if (!mask) return null;
+ const conf = mask.getAsFloat32Array();
+ const mw = mask.width;
+ const mh = mask.height;
+ const [out, ctx] = canvasOf(mw, mh);
+ const img = ctx.createImageData(mw, mh);
+ let kept = 0;
+ for (let i = 0; i < mw * mh; i++) {
+ // The selfie model's first confidence mask is background; subject = 1-bg.
+ const subject = 1 - conf[i];
+ const a =
+ subject > PERSON_THRESHOLD
+ ? Math.min(255, Math.round((subject - PERSON_THRESHOLD) * 4 * 255))
+ : 0;
+ img.data[i * 4] = 255;
+ img.data[i * 4 + 1] = 255;
+ img.data[i * 4 + 2] = 255;
+ img.data[i * 4 + 3] = a;
+ if (a > 128) kept++;
+ }
+ if (kept < mw * mh * 0.01) return null; // no meaningful person
+ ctx.putImageData(img, 0, 0);
+ return out;
+ } finally {
+ result.close();
+ }
+}
+
+/** Person matting, fully on-device. Null when no person registers (callers
+ * fall through to the hosted path). */
+export async function personCutout(blob: Blob): Promise<HTMLCanvasElement | null> {
+ const [img, segmenter] = await Promise.all([decode(blob), personSegmenter()]);
+ if (!img || !segmenter) return null;
+ const w = img.naturalWidth;
+ const h = img.naturalHeight;
+ const [canvas, ctx] = canvasOf(w, h);
+ ctx.drawImage(img, 0, 0);
+ const result = segmenter.segment(canvas);
+ try {
+ const mask = result.confidenceMasks?.[0];
+ if (!mask) return null;
+ const conf = mask.getAsFloat32Array();
+ const mw = mask.width;
+ const mh = mask.height;
+ // The selfie model's first confidence mask is background; subject = 1-bg.
+ const ok = applyAlpha(ctx, w, h, (i) => {
+ const x = Math.min(mw - 1, Math.round(((i % w) * mw) / w));
+ const y = Math.min(mh - 1, Math.round((Math.floor(i / w) * mh) / h));
+ const subject = 1 - conf[y * mw + x];
+ return subject > PERSON_THRESHOLD ? Math.min(1, (subject - PERSON_THRESHOLD) * 4) : 0;
+ });
+ return ok ? canvas : null;
+ } finally {
+ result.close();
+ }
+}
+
+/** Hosted matting for general subjects: the vision model returns segmentation
+ * masks; the largest subject's mask becomes the alpha. */
+export async function hostedCutout(blob: Blob): Promise<HTMLCanvasElement | null> {
+ const img = await decode(blob);
+ if (!img) return null;
+ const w = img.naturalWidth;
+ const h = img.naturalHeight;
+ const [canvas, ctx] = canvasOf(w, h);
+ ctx.drawImage(img, 0, 0);
+ // Fit the upload to the inline budget; masks come back box-relative, so a
+ // downscaled request still masks the full-size picture.
+ const [small, sctx] = canvasOf(Math.min(1024, w), Math.round((Math.min(1024, w) * h) / w));
+ sctx.drawImage(img, 0, 0, small.width, small.height);
+ const dataUrl = small.toDataURL("image/jpeg", 0.85);
+ const res = await hostedPost("/api/inference/responses", {
+ donkeyProvider: "gemini",
+ model: geminiModelRoles.chat,
+ instructions:
+ "You segment images. Output a JSON object {\"masks\": [...]} where each entry has \"box_2d\" ([y0, x0, y1, x1], normalized 0-1000), \"mask\" (a data: URI PNG probability mask for that box), and \"label\". Segment only the single main foreground subject.",
+ response_format: { type: "json_object" },
+ input: [
+ {
+ role: "user",
+ content: [
+ { type: "input_image", dataBase64: dataUrl.split(",")[1], mimeType: "image/jpeg" },
+ { text: "Give the segmentation mask for the main foreground subject." },
+ ],
+ },
+ ],
+ });
+ if (!res.ok) return null;
+ const body = (await res.json()) as { output_text?: string };
+ let masks: { box_2d?: number[]; mask?: string }[] = [];
+ try {
+ const parsed = JSON.parse(body.output_text ?? "") as
+ | { masks?: typeof masks }
+ | typeof masks;
+ masks = Array.isArray(parsed) ? parsed : (parsed.masks ?? []);
+ } catch {
+ return null;
+ }
+ const best = masks
+ .filter((m) => Array.isArray(m.box_2d) && m.box_2d.length === 4 && typeof m.mask === "string")
+ .sort((a, b) => {
+ const area = (box: number[]) => Math.max(0, box[2] - box[0]) * Math.max(0, box[3] - box[1]);
+ return area(b.box_2d!) - area(a.box_2d!);
+ })[0];
+ if (!best) return null;
+ const maskImg = await decode(await (await fetch(best.mask!)).blob().catch(() => new Blob()));
+ if (!maskImg) return null;
+ const [y0, x0, y1, x1] = best.box_2d!.map((v) => v / 1000);
+ const [, mctx] = canvasOf(w, h);
+ mctx.drawImage(maskImg, x0 * w, y0 * h, Math.max(1, (x1 - x0) * w), Math.max(1, (y1 - y0) * h));
+ const maskData = mctx.getImageData(0, 0, w, h).data;
+ const ok = applyAlpha(ctx, w, h, (i) => {
+ const v = maskData[i * 4] / 255; // grayscale probability
+ return v > 0.5 ? 1 : v > 0.25 ? (v - 0.25) * 4 : 0;
+ });
+ return ok ? canvas : null;
+}
+
+/** Knock out a flat studio backdrop — the shape a generated sticker arrives
+ * in. Null when the picture has no such backdrop. */
+export async function flatBackdropCutout(blob: Blob): Promise<HTMLCanvasElement | null> {
+ const img = await decode(blob);
+ if (!img) return null;
+ const [canvas, ctx] = canvasOf(img.naturalWidth, img.naturalHeight);
+ ctx.drawImage(img, 0, 0);
+ return removeFlatBackdrop(canvas) ? canvas : null;
+}
+
+/**
+ * The whole cutout pass, cheapest answer first: on-device person matting, then
+ * the flat-backdrop knockout that generated stickers are made for, and only
+ * then the hosted matte — the one that costs a round trip and credits. Ends in
+ * crop-to-content and the optional die-cut outline. Null when no path finds a
+ * subject, which is the caller's cue to keep the picture as it came.
+ */
+export async function makeStickerCutout(
+ blob: Blob,
+ opts: { outline?: boolean; plainBackdrop?: boolean } = {}
+): Promise<Blob | null> {
+ // A sticker we generated was asked for on a plain sweep, so the knockout is
+ // both the cheapest answer and the one that knows the most. It goes first
+ // there; a cat on a studio grey reads as a person often enough that letting
+ // the segmenter answer first returns a mangled matte.
+ const order = opts.plainBackdrop
+ ? [flatBackdropCutout, personCutout, hostedCutout]
+ : [personCutout, flatBackdropCutout, hostedCutout];
+ let cut: HTMLCanvasElement | null = null;
+ for (const attempt of order) {
+ cut = await attempt(blob);
+ if (cut) break;
+ }
+ if (!cut) return null;
+ // A sticker is one thing. Strokes and specks the generator scattered around
+ // it survive any matte — they are not backdrop — so the body is picked out
+ // before the crop, which would otherwise frame the strays too.
+ keepLargestSubject(cut);
+ const cropped = cropToContent(cut);
+ return pngOf(opts.outline === false ? cropped : withOutline(cropped));
+}
diff --git a/site/src/cut/lib/effectStack.ts b/site/src/cut/lib/effectStack.ts
new file mode 100644
index 00000000..bab4acf8
--- /dev/null
+++ b/site/src/cut/lib/effectStack.ts
@@ -0,0 +1,118 @@
+/**
+ * Where effects stand in the element stack.
+ *
+ * Lane 0 is the top row and a bigger lane number sits further under it, so an
+ * effect grades the lanes below its own and leaves the ones above alone. The
+ * stage is therefore built in slices, bottom to top: the picture, the elements
+ * under the deepest effect, that effect's paints, the elements above it, and
+ * so on up. A slice wears the grade of every effect above it as a plain CSS
+ * filter and their frame moves as a transform.
+ *
+ * The math is here, apart from React, because both exports walk the same stack.
+ */
+
+import type { EffectPreviewState } from "@donkeycut/effects-kit";
+
+/** One effect live at a moment, with the lane that places it in the stack. */
+export interface LiveEffect {
+ lane: number;
+ state: EffectPreviewState;
+}
+
+/** How the content under an effect moves: a zoom's push in, a shake, and the
+ * horizontal jump that reads as tearing. Empty for the effects that only
+ * filter. */
+export function stageEffectTransform(states: EffectPreviewState[]): string | undefined {
+ let dx = 0;
+ let dy = 0;
+ let zoom = 1;
+ let origin = { x: 0.5, y: 0.5 };
+ for (const st of states) {
+ dx += st.dx ?? 0;
+ dy += st.dy ?? 0;
+ // Tearing shows as a jump of the frame itself; the fraction is of frame
+ // width, so it scales with whatever size the stage is.
+ if (st.ghostFrac) dx += st.ghostFrac * 1080 * 3;
+ zoom *= st.zoom ?? 1;
+ if (st.origin) origin = st.origin;
+ }
+ if (!dx && !dy && zoom === 1) return undefined;
+ // The offsets are design pixels against a 1080 short side.
+ const pct = (px: number) => ((px / 1080) * 100).toFixed(3);
+ // Percentage translates are of the stage's own box, so the scale can be sat
+ // on its origin without a transform-origin the callers would have to carry.
+ // They are measured from the box's middle, which is where `transform-origin`
+ // already stands: taken from the corner instead, the point held still lands
+ // half a frame off and the push in walks the picture out of frame.
+ const ox = (origin.x - 0.5) * 100;
+ const oy = (origin.y - 0.5) * 100;
+ return (
+ `translate(${pct(dx)}%, ${pct(dy)}%) ` +
+ `translate(${ox.toFixed(3)}%, ${oy.toFixed(3)}%) scale(${zoom}) ` +
+ `translate(${(-ox).toFixed(3)}%, ${(-oy).toFixed(3)}%)`
+ );
+}
+
+/** The grade of a run of effects, as one CSS filter. */
+const stageEffectFilter = (states: EffectPreviewState[]) =>
+ states
+ .map((s) => s.cssFilter)
+ .filter(Boolean)
+ .join(" ") || undefined;
+
+/** One slice of the stage, bottom of the stack first. */
+export type StageSlice =
+ | {
+ kind: "elements";
+ key: string;
+ /** Elements on lanes in [from, to) — `from` counts down from the top. */
+ from: number;
+ to: number;
+ filter?: string;
+ transform?: string;
+ /** The topmost slice carries the captions, which nothing grades. */
+ captions: boolean;
+ }
+ | { kind: "paint"; key: string; states: EffectPreviewState[] };
+
+/**
+ * The stage bottom to top: what the picture itself wears, then alternating
+ * element slices and the paints of the effect that sits over each.
+ */
+export function stageSlices(
+ live: LiveEffect[],
+ lanes: number[]
+): { picture: { filter?: string; transform?: string }; slices: StageSlice[] } {
+ const above = (lane: number) => live.filter((e) => e.lane < lane).map((e) => e.state);
+ const all = live.map((e) => e.state);
+ const slices: StageSlice[] = [];
+ // Deepest first: elements under that effect, then its paints over them.
+ for (let i = lanes.length - 1; i >= 0; i--) {
+ const lane = lanes[i];
+ slices.push({
+ kind: "elements",
+ key: `band-${lane}`,
+ from: lane + 1,
+ to: i + 1 < lanes.length ? lanes[i + 1] : Infinity,
+ filter: stageEffectFilter(above(lane + 1)),
+ transform: stageEffectTransform(above(lane + 1)),
+ captions: false,
+ });
+ slices.push({
+ kind: "paint",
+ key: `paint-${lane}`,
+ states: live.filter((e) => e.lane === lane).map((e) => e.state),
+ });
+ }
+ slices.push({
+ kind: "elements",
+ key: "band-top",
+ from: 0,
+ to: lanes.length ? lanes[0] : Infinity,
+ captions: true,
+ });
+ return {
+ picture: { filter: stageEffectFilter(all), transform: stageEffectTransform(all) },
+ slices,
+ };
+}
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index df482342..049df3b0 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -2,20 +2,20 @@
import { apiFetch, apiJson, getBackend, type CutBackend } from "./backend";
import { quotaErrorMessage } from "./backend/cloud";
-import { normalizeGrade } from "./colorGrade";
import { renderProjectToMp4 } from "./exportRender";
import { putSigned } from "./media";
import { clipSpeed, getClipSpans, overlayLayers, projectDuration, spanSequence, useEditor } from "./store";
import { captionStyle, cueOverlay, cueWordWindows, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
-import { renderOverlayPng } from "./textRender";
-import { frameOf, overlayAnimStyle } from "./types";
+import { isOverlayAnimated, normalizeGrade } from "@donkeycut/effects-kit";
+import { renderElementFrames, renderElementPng } from "./textRender";
+import { frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle } from "./types";
import type {
Aspect,
AudioClip,
ClipAnim,
MediaAsset,
+ Overlay,
SubtitlesBlock,
- TextOverlay,
VideoClip,
} from "./types";
@@ -171,7 +171,7 @@ export interface ExportDoc {
/** Every video clip, any track (track 0 folds sequentially, others composite). */
clips: VideoClip[];
audioClips: AudioClip[];
- overlays: TextOverlay[];
+ overlays: Overlay[];
subtitles: SubtitlesBlock;
/** Whole-video fades (seconds): in from black / out to black on the final
* composite. */
@@ -343,14 +343,90 @@ async function buildExportPayload(
duck: a.duck,
}));
- const overlays: { file: string; start: number; end: number }[] = [];
+ // Text-behind-speaker: the mask video renders first so the loop below tags
+ // behind elements only when the effect will actually composite (no person
+ // segmenter, or no person → the text stays a normal front title).
+ let behindMask: { file: string; from: number } | undefined;
+ if (doc.overlays.some((o) => isTextOverlay(o) && !!o.behindSubject && !o.hidden)) {
+ const mask = await import("./maskVideo")
+ .then((m) => m.renderBehindMask(doc, duration))
+ .catch(() => null);
+ if (mask) {
+ pngs.push({ name: "behind_mask.mp4", blob: mask.blob });
+ behindMask = { file: "behind_mask.mp4", from: mask.from };
+ }
+ }
+
+ const overlays: {
+ file?: string;
+ start: number;
+ end: number;
+ x?: number;
+ y?: number;
+ blank?: string;
+ frames?: { file: string; duration: number }[];
+ behind?: boolean;
+ /** Its row: lane 0 is the top of the stack, and the effects interleave
+ * with these by lane. */
+ lane?: number;
+ }[] = [];
+ // Effect elements never rasterize: they ship as time-gated filter recipes
+ // the server builds into the graph (the ids ride, never filter text).
+ const effects = doc.overlays
+ .filter((o) => o.kind === "effect" && !o.hidden && o.start < duration)
+ .map((o) => ({
+ effect: (o as { effect: string }).effect,
+ amount: (o as { amount?: number }).amount,
+ focus: (o as { focus?: { x: number; y: number } }).focus,
+ ramp: (o as { ramp?: number }).ramp,
+ lane: laneOf(o),
+ start: o.start,
+ end: Math.min(o.end, duration),
+ }));
+
for (let i = 0; i < doc.overlays.length; i++) {
const o = doc.overlays[i];
- if (o.hidden || o.start >= duration || !o.text.trim()) continue;
- const png = await renderOverlayPng(o, settings.width, settings.height);
+ if (o.hidden || o.start >= duration) continue;
+ if (o.kind === "effect") continue;
+ // A blank title has no pixels to burn; shapes and stickers always render.
+ if (isTextOverlay(o) && !o.text.trim()) continue;
+ // A Lottie sticker's pixels move on their own, so it exports as frames
+ // even with no transform animation set.
+ if (isOverlayAnimated(o) || (isStickerOverlay(o) && o.lottie)) {
+ // Animated: a region-cropped 30fps frame sequence that the server plays
+ // as a concat-demuxer slideshow overlaid at the region. Presets sample
+ // their heads and tails and reuse the middle; a keyframed pose changes
+ // on its own schedule, so its whole span is sampled frame by frame.
+ const set = await renderElementFrames(o, settings.width, settings.height, 30, doc.assets);
+ const names = set.images.map((blob, j) => {
+ const name = `overlay_${i}_f${j}.png`;
+ pngs.push({ name, blob });
+ return name;
+ });
+ const blank = `overlay_${i}_blank.png`;
+ pngs.push({ name: blank, blob: set.blank });
+ overlays.push({
+ start: o.start,
+ end: Math.min(o.end, duration),
+ x: set.x,
+ y: set.y,
+ blank,
+ frames: set.entries.map((e) => ({ file: names[e.image], duration: e.duration })),
+ lane: laneOf(o),
+ ...(behindMask && isTextOverlay(o) && o.behindSubject ? { behind: true } : {}),
+ });
+ continue;
+ }
+ const png = await renderElementPng(o, settings.width, settings.height, doc.assets);
const key = `overlay_${i}.png`;
pngs.push({ name: key, blob: png });
- overlays.push({ file: key, start: o.start, end: Math.min(o.end, duration) });
+ overlays.push({
+ file: key,
+ start: o.start,
+ end: Math.min(o.end, duration),
+ lane: laneOf(o),
+ ...(behindMask && isTextOverlay(o) && o.behindSubject ? { behind: true } : {}),
+ });
}
// Subtitle stills travel in their own spec lane: the server plays each
@@ -376,7 +452,7 @@ async function buildExportPayload(
for (let wi = 0; wi < windows.length; wi++) {
const win = windows[wi];
if (win.start >= duration) break;
- const png = await renderOverlayPng(
+ const png = await renderElementPng(
cueOverlay(
cue,
capStyle,
@@ -425,6 +501,8 @@ async function buildExportPayload(
overlayVideos,
overlays,
captions,
+ ...(effects.length ? { effects } : {}),
+ ...(behindMask ? { behindMask } : {}),
},
pngs,
};
@@ -474,7 +552,7 @@ async function postExport(
payload.pngs.map(async (p) => {
const target = byName.get(p.name);
if (!target) throw new Error("Export failed to start.");
- await putSigned(target.url, p.blob, "image/png");
+ await putSigned(target.url, p.blob, p.blob.type || "image/png");
})
);
for (const p of payload.pngs) overlays.push({ name: p.name, key: byName.get(p.name)!.key });
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index a725e385..41ae72fe 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -32,9 +32,11 @@ import { overlayPlan, trackZeroPlan } from "./framePlan";
import { frameSink, openMedia, videoTrackOf } from "./mediaRead";
import { getClipSpans, overlayLayers, projectDuration, spanSequence } from "./store";
import { captionStyle, cueOverlay, cueWordWindows, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
-import { renderOverlayPng } from "./textRender";
-import { frameOf, isFullRect, overlayAnimStyle, projectFadeSeconds, rectOf } from "./types";
-import type { ClipSpan, MediaAsset, TextOverlay } from "./types";
+import { applyEffectToCanvas, evalOverlayFrame, grainTile, isOverlayAnimated, planAnimatedLayers, type LottieHandle, type OverlayAnim } from "@donkeycut/effects-kit";
+import { BehindCompositor, hasBehindOverlays } from "./behindPass";
+import { renderElementPng } from "./textRender";
+import { frameOf, isEffectOverlay, isFullRect, isTextOverlay, laneOf, overlayAnimStyle, projectFadeSeconds, rectOf } from "./types";
+import type { ClipSpan, EffectOverlay, MediaAsset, Overlay, StickerOverlay } from "./types";
import type { ExportDoc, ExportSettings } from "./exportClient";
/** Audio is written at the rate and width a delivery file wants, rather than
@@ -43,7 +45,7 @@ const AUDIO_RATE = 48000;
const AUDIO_CHANNELS = 2;
/** Video codecs to try, best first. */
-const VIDEO_CODECS: VideoCodec[] = ["avc", "hevc", "vp9", "av1"];
+export const VIDEO_CODECS: VideoCodec[] = ["avc", "hevc", "vp9", "av1"];
export interface RenderProgress {
/** 0..1 across the whole render. */
@@ -134,7 +136,7 @@ function sourceTimeAt(span: ClipSpan, t: number): number {
* rather than seeking back to the start for every frame. A still decodes once
* and is handed back for as long as its clip is on screen.
*/
-class ClipReader {
+export class ClipReader {
private input: ReturnType<typeof openMedia> | null = null;
private sink: ClipSink | null = null;
private still: Frame | null = null;
@@ -223,10 +225,39 @@ class ClipReader {
/** A text layer and the window it is on screen for. The picture is drawn when
* the render reaches it, not up front. */
+/** The effect elements live at `t`, deepest lane first — the order the stack
+ * is walked in. */
+function liveEffectsAt(overlays: Overlay[], t: number): EffectOverlay[] {
+ return overlays
+ .filter(isEffectOverlay)
+ .filter((o) => !o.hidden && t >= o.start && t < o.end)
+ .sort((a, b) => laneOf(b) - laneOf(a));
+}
+
+/** Captions sit above every element and every effect; lane 0 is the topmost
+ * element row, so they need a place above that. */
+const CAPTION_LANE = -1;
+
interface StampedLayer {
- overlay: TextOverlay;
+ overlay: Overlay;
start: number;
end: number;
+ /** Where this layer sits in the stack: the element's lane, or -1 for a
+ * caption, which rides above every element and every effect. */
+ stackLane: number;
+ /** Animated element: drawStamps evaluates this at (t − animStart) over
+ * animDur and applies the result as a canvas transform about the element
+ * center. Typewriter windows are pre-sliced into per-char layers, so their
+ * anim carries the remaining slots only. */
+ anim?: OverlayAnim;
+ animStart?: number;
+ animDur?: number;
+ /** The element this layer came from, before the picture was neutralized or
+ * a typewriter slice trimmed its text. The per-frame pose is evaluated from
+ * it, so keys and the element's own rotation/opacity survive. */
+ source?: Overlay;
+ /** Lottie sticker: seek this per frame instead of caching one bitmap. */
+ lottie?: LottieHandle;
}
/**
@@ -242,13 +273,40 @@ interface StampedLayer {
* The layout itself is the same rendering the ffmpeg path burns in, so the
* words land in the same place whichever renderer produced the file.
*/
-function stampText(doc: ExportDoc): StampedLayer[] {
+async function stampText(doc: ExportDoc): Promise<StampedLayer[]> {
const duration = projectDuration(doc);
const layers: StampedLayer[] = [];
for (const o of doc.overlays) {
- if (o.hidden || o.start >= duration || !o.text.trim()) continue;
- layers.push({ overlay: o, start: o.start, end: Math.min(o.end, duration) });
+ if (o.hidden || o.start >= duration) continue;
+ // A blank title has no pixels to draw; shapes and stickers always render.
+ if (isTextOverlay(o) && !o.text.trim()) continue;
+ // Behind-speaker titles composite in the behind pass, under the person.
+ if (isTextOverlay(o) && o.behindSubject) continue;
+ // Effect elements filter the video per frame; nothing to stamp.
+ if (o.kind === "effect") continue;
+ // A Lottie sticker seeks per frame; its handle rides the layer.
+ const lottie =
+ o.kind === "sticker" && o.lottie && o.assetId
+ ? ((await import("./lottieAssets").then((m) =>
+ m.sharedLottieHandle(o.assetId!, doc.assets)
+ )) ?? undefined)
+ : undefined;
+ if (isOverlayAnimated(o) || lottie) {
+ const before = layers.length;
+ pushAnimatedLayers(layers, o, Math.min(o.end, duration));
+ for (let i = before; i < layers.length; i++) {
+ layers[i].stackLane = laneOf(o);
+ if (lottie) layers[i].lottie = lottie;
+ }
+ continue;
+ }
+ layers.push({
+ overlay: o,
+ start: o.start,
+ end: Math.min(o.end, duration),
+ stackLane: laneOf(o),
+ });
}
if (doc.subtitles.showOnVideo) {
@@ -280,6 +338,7 @@ function stampText(doc: ExportDoc): StampedLayer[] {
),
start: win.start,
end: Math.min(win.end, duration),
+ stackLane: CAPTION_LANE,
});
}
}
@@ -301,13 +360,14 @@ class StampCache {
constructor(
private width: number,
- private height: number
+ private height: number,
+ private assets: MediaAsset[]
) {}
async bitmapFor(layer: StampedLayer): Promise<ImageBitmap> {
let bitmap = this.drawn.get(layer);
if (!bitmap) {
- const png = await renderOverlayPng(layer.overlay, this.width, this.height);
+ const png = await renderElementPng(layer.overlay, this.width, this.height, this.assets);
bitmap = await createImageBitmap(png);
this.drawn.set(layer, bitmap);
}
@@ -515,7 +575,7 @@ export async function renderProjectToMp4(
target: new StreamTarget(writable, { chunked: true }),
});
- const stamps = new StampCache(settings.width, settings.height);
+ const stamps = new StampCache(settings.width, settings.height, doc.assets);
const readers = new Map<string, ClipReader>();
const readerFor = (asset: MediaAsset) => {
let r = readers.get(asset.id);
@@ -547,7 +607,17 @@ export async function renderProjectToMp4(
// frames rather than being interleaved with them.
if (audio && mix) await audio.add(mix);
- const layers = stampText(doc);
+ const layers = await stampText(doc);
+ // Deepest lane first, so a walk of it is a walk up the stack.
+ const stacked = [...layers].sort((a, b) => b.stackLane - a.stackLane);
+ // The behind pass is created only when something is tagged; prepare()
+ // makes rasters and the segmenter resident before the first frame.
+ let fxScratch: OffscreenCanvas | null = null;
+ let behind: BehindCompositor | null = null;
+ if (hasBehindOverlays(doc.overlays)) {
+ behind = new BehindCompositor();
+ await behind.prepare(doc.overlays, settings.width, settings.height, doc.assets);
+ }
stop();
const spans = getClipSpans(doc.clips, doc.assets, 0);
@@ -608,7 +678,36 @@ export async function renderProjectToMp4(
comp.drawIntoRect(frame, rect, cover, layer.alpha, t, layer.zoom, layer.clip);
}
- await drawStamps(canvas, layers, stamps, t);
+ // Behind-speaker titles: video → text → segmented person, exactly the
+ // preview's pass, sampled per exported frame (no mask throttling).
+ behind?.draw(canvas, doc.overlays, doc.assets, t, { minMaskInterval: 0 });
+
+ // The stack, bottom up: an effect grades what plays under it, so the
+ // elements below it burn in first, the effect runs over that much of the
+ // frame, and the elements above it land on the graded picture. Same
+ // order as the live preview and the ffmpeg graph.
+ let drawn = 0;
+ for (const o of liveEffectsAt(doc.overlays, t)) {
+ const upto = stacked.findIndex((l) => l.stackLane <= laneOf(o));
+ const end = upto < 0 ? stacked.length : upto;
+ if (end > drawn) await drawStamps(canvas, stacked.slice(drawn, end), stamps, t);
+ drawn = Math.max(drawn, end);
+ if (!fxScratch) {
+ fxScratch = new OffscreenCanvas(canvas.width, canvas.height);
+ }
+ applyEffectToCanvas(
+ canvas,
+ fxScratch,
+ o.effect,
+ o.amount,
+ t - o.start,
+ grainTile,
+ o.focus,
+ o.ramp,
+ o.end - o.start
+ );
+ }
+ await drawStamps(canvas, stacked.slice(drawn), stamps, t);
stamps.retire(t);
comp.drawProjectFade(projectFadeGain(doc, t, duration));
@@ -638,6 +737,19 @@ export async function renderProjectToMp4(
}
/** Draw the text layers live at `t` over the finished picture. */
+/** Stamped layers for one animated element: the kit plans the windows (the
+ * same split its frame sequences use), and the render hangs what only it knows
+ * on them — the element they came from, and a Lottie handle to seek. */
+function pushAnimatedLayers(layers: StampedLayer[], o: Overlay, end: number) {
+ const shared = {
+ animStart: o.start,
+ animDur: Math.max(0.1, o.end - o.start),
+ source: o,
+ stackLane: laneOf(o),
+ };
+ for (const layer of planAnimatedLayers(o, end)) layers.push({ ...layer, ...shared });
+}
+
async function drawStamps(
canvas: HTMLCanvasElement | OffscreenCanvas,
layers: StampedLayer[],
@@ -651,7 +763,53 @@ async function drawStamps(
if (!ctx) return;
for (const layer of layers) {
if (t < layer.start || t >= layer.end) continue;
- ctx.drawImage(await stamps.bitmapFor(layer), 0, 0, canvas.width, canvas.height);
+ if (layer.lottie) {
+ // Nothing to cache: the animation's own canvas is sought per frame and
+ // drawn at the sticker's rect under the same delta transform.
+ const o = layer.overlay as StickerOverlay;
+ const ev = evalOverlayFrame(
+ { ...(layer.source ?? o), anim: layer.anim },
+ t - (layer.animStart ?? layer.start)
+ );
+ if (ev.opacity <= 0.001) continue;
+ const scale = Math.min(canvas.width, canvas.height) / 1080;
+ const w = Math.max(1, o.w * canvas.width);
+ const aspect = layer.lottie.width > 0 ? layer.lottie.width / layer.lottie.height : 1;
+ const h = w / aspect;
+ ctx.save();
+ ctx.globalAlpha = ev.opacity;
+ ctx.translate(ev.x * canvas.width + ev.dx * scale, ev.y * canvas.height + ev.dy * scale);
+ ctx.rotate((ev.rotation * Math.PI) / 180);
+ ctx.scale(ev.scale, ev.scale);
+ ctx.drawImage(layer.lottie.seek(t - (layer.animStart ?? layer.start)), -w / 2, -h / 2, w, h);
+ ctx.restore();
+ continue;
+ }
+ const bitmap = await stamps.bitmapFor(layer);
+ if (!layer.anim) {
+ ctx.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
+ continue;
+ }
+ // Unkeyed, the bitmap bakes the element's own rotation/opacity and only
+ // the preset's delta composes on top. Keyed, the bitmap is neutral and the
+ // pose supplies position, scale, rotation and opacity outright — either
+ // way this samples the evaluator the ffmpeg path rasterized into frames.
+ const ev = evalOverlayFrame(
+ { ...(layer.source ?? layer.overlay), anim: layer.anim },
+ t - (layer.animStart ?? layer.start)
+ );
+ if (ev.opacity <= 0.001) continue;
+ const scale = Math.min(canvas.width, canvas.height) / 1080;
+ const cx = layer.overlay.x * canvas.width;
+ const cy = layer.overlay.y * canvas.height;
+ ctx.save();
+ ctx.globalAlpha = ev.opacity;
+ ctx.translate(ev.x * canvas.width + ev.dx * scale, ev.y * canvas.height + ev.dy * scale);
+ ctx.rotate((ev.rotation * Math.PI) / 180);
+ ctx.scale(ev.scale, ev.scale);
+ ctx.translate(-cx, -cy);
+ ctx.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
+ ctx.restore();
}
}
diff --git a/site/src/cut/lib/fontAssets.ts b/site/src/cut/lib/fontAssets.ts
new file mode 100644
index 00000000..a9a93207
--- /dev/null
+++ b/site/src/cut/lib/fontAssets.ts
@@ -0,0 +1,59 @@
+"use client";
+
+/**
+ * Font assets: a .ttf/.otf living in a project's media/ folder as a
+ * type-"font" asset. This module turns those bytes into live FontFaces and
+ * registry entries so the font menu, the DOM preview, and the canvas painters
+ * all resolve them. Family names key on the asset id, which is stable across
+ * sessions and residencies. Projects no longer take new font files; what is
+ * here keeps the ones already saved rendering.
+ */
+
+import {
+ registerFonts,
+ unregisterFonts,
+ uploadedFontId,
+ type MediaAsset,
+} from "./types";
+
+/** Asset ids whose FontFace is loaded (or loading) in this page. */
+const loaded = new Map<string, Promise<void>>();
+
+/** A display label from the uploaded file's name, extension dropped. */
+const fontLabel = (name: string) => name.replace(/\.(ttf|otf|woff2?|TTF|OTF)$/, "") || "Custom font";
+
+/** Register one font asset: fetch its bytes, add the FontFace, and list it in
+ * the font registry. Safe to call repeatedly. */
+export function registerFontAsset(asset: MediaAsset): Promise<void> {
+ let hit = loaded.get(asset.id);
+ if (hit) return hit;
+ const family = `uf-${asset.id}`;
+ hit = (async () => {
+ const res = await fetch(asset.url);
+ if (!res.ok) throw new Error("font fetch failed");
+ const face = new FontFace(family, await res.arrayBuffer());
+ await face.load();
+ document.fonts.add(face);
+ registerFonts([
+ { id: uploadedFontId(asset.id), label: fontLabel(asset.name), stack: `"${family}"` },
+ ]);
+ })().catch(() => {
+ // A gone or corrupt file: drop the marker so a later pass can retry.
+ loaded.delete(asset.id);
+ });
+ loaded.set(asset.id, hit);
+ return hit;
+}
+
+/** Reconcile the registry with the project's font assets: register new ones,
+ * unregister deleted ones. The editor calls this whenever assets change. */
+export function syncFontAssets(assets: MediaAsset[]): void {
+ const fonts = assets.filter((a) => a.type === "font");
+ for (const a of fonts) void registerFontAsset(a);
+ const live = new Set(fonts.map((a) => a.id));
+ const stale = [...loaded.keys()].filter((id) => !live.has(id));
+ if (stale.length) {
+ for (const id of stale) loaded.delete(id);
+ unregisterFonts(stale.map(uploadedFontId));
+ }
+}
diff --git a/site/src/cut/lib/genScene.ts b/site/src/cut/lib/genScene.ts
index 964cdb44..c3afaaee 100644
--- a/site/src/cut/lib/genScene.ts
+++ b/site/src/cut/lib/genScene.ts
@@ -240,7 +240,7 @@ function toReferences(ids: string[] | undefined): RefAsset[] {
const out: RefAsset[] = [];
for (const id of ids) {
const a = assets.find((x) => x.id === id);
- if (a) out.push({ mediaId: a.id, kind: a.type, purpose: "style", name: a.name });
+ if (a && a.type !== "font") out.push({ mediaId: a.id, kind: a.type, purpose: "style", name: a.name });
}
return out;
}
diff --git a/site/src/cut/lib/generate.ts b/site/src/cut/lib/generate.ts
index 9c45b653..3b4800e3 100644
--- a/site/src/cut/lib/generate.ts
+++ b/site/src/cut/lib/generate.ts
@@ -296,7 +296,7 @@ const providerError = (error: unknown): string | null => {
* compose step (see composeGen.ts) unless the caller opted out; on a compose
* failure, falls back to the visual refs as-is with text-ref contents folded
* into the prompt. `maxImages` caps what the generator accepts (video: 1 seed). */
-async function promptAndImages(
+export async function promptAndImages(
target: "video" | "image",
prompt: string,
refs: AssetRef[],
diff --git a/site/src/cut/lib/googleFonts.ts b/site/src/cut/lib/googleFonts.ts
new file mode 100644
index 00000000..f577e058
--- /dev/null
+++ b/site/src/cut/lib/googleFonts.ts
@@ -0,0 +1,62 @@
+"use client";
+
+/**
+ * The bundled Google font families, self-hosted at build time via
+ * next/font/google — the loader downloads the files during `next build` and
+ * serves them from our origin, so nothing fetches Google at runtime and a
+ * rasterize is deterministic offline. Importing this module (the editor root
+ * does) registers every family into the shared font registry.
+ */
+
+import {
+ Anton,
+ Archivo_Black,
+ Bangers,
+ Bebas_Neue,
+ Caveat,
+ DM_Serif_Display,
+ Inter,
+ Lobster,
+ Montserrat,
+ Oswald,
+ Pacifico,
+ Permanent_Marker,
+ Playfair_Display,
+ Poppins,
+ Space_Grotesk,
+} from "next/font/google";
+import { registerFonts } from "./types";
+
+const inter = Inter({ subsets: ["latin"], weight: ["400", "700"], preload: false });
+const montserrat = Montserrat({ subsets: ["latin"], weight: ["400", "700"], preload: false });
+const poppins = Poppins({ subsets: ["latin"], weight: ["400", "700"], preload: false });
+const oswald = Oswald({ subsets: ["latin"], weight: ["400", "700"], preload: false });
+const spaceGrotesk = Space_Grotesk({ subsets: ["latin"], weight: ["400", "700"], preload: false });
+const playfair = Playfair_Display({ subsets: ["latin"], weight: ["400", "700"], preload: false });
+const caveat = Caveat({ subsets: ["latin"], weight: ["400", "700"], preload: false });
+const bebas = Bebas_Neue({ subsets: ["latin"], weight: "400", preload: false });
+const anton = Anton({ subsets: ["latin"], weight: "400", preload: false });
+const archivoBlack = Archivo_Black({ subsets: ["latin"], weight: "400", preload: false });
+const bangers = Bangers({ subsets: ["latin"], weight: "400", preload: false });
+const lobster = Lobster({ subsets: ["latin"], weight: "400", preload: false });
+const pacifico = Pacifico({ subsets: ["latin"], weight: "400", preload: false });
+const permanentMarker = Permanent_Marker({ subsets: ["latin"], weight: "400", preload: false });
+const dmSerifDisplay = DM_Serif_Display({ subsets: ["latin"], weight: "400", preload: false });
+
+registerFonts([
+ { id: "inter", label: "Inter", stack: inter.style.fontFamily },
+ { id: "montserrat", label: "Montserrat", stack: montserrat.style.fontFamily },
+ { id: "poppins", label: "Poppins", stack: poppins.style.fontFamily },
+ { id: "oswald", label: "Oswald", stack: oswald.style.fontFamily },
+ { id: "space-grotesk", label: "Space Grotesk", stack: spaceGrotesk.style.fontFamily },
+ { id: "playfair", label: "Playfair Display", stack: playfair.style.fontFamily },
+ { id: "caveat", label: "Caveat", stack: caveat.style.fontFamily },
+ { id: "bebas", label: "Bebas Neue", stack: bebas.style.fontFamily },
+ { id: "anton", label: "Anton", stack: anton.style.fontFamily },
+ { id: "archivo-black", label: "Archivo Black", stack: archivoBlack.style.fontFamily },
+ { id: "bangers", label: "Bangers", stack: bangers.style.fontFamily },
+ { id: "lobster", label: "Lobster", stack: lobster.style.fontFamily },
+ { id: "pacifico", label: "Pacifico", stack: pacifico.style.fontFamily },
+ { id: "permanent-marker", label: "Permanent Marker", stack: permanentMarker.style.fontFamily },
+ { id: "dm-serif", label: "DM Serif Display", stack: dmSerifDisplay.style.fontFamily },
+]);
diff --git a/site/src/cut/lib/lottieAssets.ts b/site/src/cut/lib/lottieAssets.ts
new file mode 100644
index 00000000..2d3c334f
--- /dev/null
+++ b/site/src/cut/lib/lottieAssets.ts
@@ -0,0 +1,60 @@
+"use client";
+
+/**
+ * Lottie sticker assets: the animation JSON lives in the project's media/
+ * folder (origin "sticker"); this module fetches and caches the documents and
+ * hands out seekable players. Export paths share one handle per asset (their
+ * seeks are sequential); each preview element makes its own, so two copies of
+ * a sticker can sit on different frames.
+ */
+
+import { createLottieHandle, isLottieData, type LottieHandle } from "@donkeycut/effects-kit";
+import type { MediaAsset } from "./types";
+
+const dataCache = new Map<string, Promise<unknown | null>>();
+const sharedHandles = new Map<string, Promise<LottieHandle | null>>();
+
+/** Whether an asset holds a Lottie document (sticker imports store them with
+ * their .json name). */
+export const isLottieAsset = (a: { fileName: string; name?: string }) =>
+ /\.json$/i.test(a.fileName) || /\.json$/i.test(a.name ?? "");
+
+function lottieData(asset: MediaAsset): Promise<unknown | null> {
+ let hit = dataCache.get(asset.id);
+ if (!hit) {
+ hit = fetch(asset.url)
+ .then((res) => (res.ok ? res.json() : null))
+ .then((data) => (isLottieData(data) ? data : null))
+ .catch(() => {
+ dataCache.delete(asset.id);
+ return null;
+ });
+ dataCache.set(asset.id, hit);
+ }
+ return hit;
+}
+
+/** The shared export-side handle for an asset (sequential seeks only). */
+export function sharedLottieHandle(
+ assetId: string,
+ assets: MediaAsset[]
+): Promise<LottieHandle | null> {
+ let hit = sharedHandles.get(assetId);
+ if (!hit) {
+ const asset = assets.find((a) => a.id === assetId);
+ hit = asset
+ ? lottieData(asset).then((data) => (data ? createLottieHandle(data) : null))
+ : Promise.resolve(null);
+ sharedHandles.set(assetId, hit);
+ void hit.then((h) => {
+ if (!h) sharedHandles.delete(assetId);
+ });
+ }
+ return hit;
+}
+
+/** A private handle for one preview element; destroy it on unmount. */
+export async function newLottieInstance(asset: MediaAsset): Promise<LottieHandle | null> {
+ const data = await lottieData(asset);
+ return data ? createLottieHandle(data) : null;
+}
diff --git a/site/src/cut/lib/maskVideo.ts b/site/src/cut/lib/maskVideo.ts
new file mode 100644
index 00000000..6e9b195c
--- /dev/null
+++ b/site/src/cut/lib/maskVideo.ts
@@ -0,0 +1,135 @@
+"use client";
+
+/**
+ * The behind-speaker mask video for the ffmpeg export path. The page — the
+ * only place that can run segmentation — composes the video layers over the
+ * behind window at a small size, segments each frame, and encodes the person
+ * mask as a plain grayscale H.264 clip (white = person; no alpha encoding,
+ * which browsers can't do reliably). The server scales it up, pads it to the
+ * timeline, and `alphamerge`s the person back over the behind-tagged text.
+ */
+
+import {
+ BufferTarget,
+ CanvasSource,
+ getFirstEncodableVideoCodec,
+ Mp4OutputFormat,
+ Output,
+} from "mediabunny";
+import { personSegmenter, segmentSubjectAlpha } from "./cutout";
+import { FrameCompositor, MISSING_FRAME } from "./composite";
+import { overlayPlan, trackZeroPlan } from "./framePlan";
+import { ClipReader, VIDEO_CODECS } from "./exportRender";
+import type { ExportDoc } from "./exportClient";
+import { getClipSpans } from "./store";
+import { frameOf, isFullRect, isTextOverlay, rectOf, type ClipSpan, type MediaAsset } from "./types";
+
+/** Encoded mask rate — the server's fps filter duplicates frames up to the
+ * output rate, and a person moves little in 1/15s. */
+export const MASK_FPS = 15;
+/** Mask frame short side; alphamerge scales it to the output. */
+const MASK_SHORT = 480;
+
+/** Timeline second inside a span mapped to its source second. */
+const sourceTimeAt = (sp: ClipSpan, t: number) => {
+ const speed = sp.clip.speed && sp.clip.speed > 0 ? sp.clip.speed : 1;
+ return Math.min(sp.clip.out, sp.clip.in + Math.max(0, t - sp.start) * speed);
+};
+
+/**
+ * Render the person-mask video covering every behind-tagged title's window
+ * (one union range). Null when nothing is tagged, no person segmenter is
+ * available, or the range is empty — the export simply proceeds without the
+ * effect, matching the preview's front-title degrade.
+ */
+export async function renderBehindMask(
+ doc: ExportDoc,
+ duration: number
+): Promise<{ blob: Blob; from: number } | null> {
+ const behind = doc.overlays.filter(
+ (o) => isTextOverlay(o) && !!o.behindSubject && !o.hidden && !!o.text.trim() && o.start < duration
+ );
+ if (behind.length === 0) return null;
+ const from = Math.max(0, Math.min(...behind.map((o) => o.start)));
+ const to = Math.min(duration, Math.max(...behind.map((o) => o.end)));
+ if (to - from < 0.05) return null;
+ const segmenter = await personSegmenter();
+ if (!segmenter) return null;
+
+ const frame = frameOf(doc.aspect);
+ const scale = MASK_SHORT / Math.min(frame.w, frame.h);
+ const even = (n: number) => 2 * Math.round((n * scale) / 2);
+ const W = even(frame.w);
+ const H = even(frame.h);
+ const codec = await getFirstEncodableVideoCodec(VIDEO_CODECS, { width: W, height: H });
+ if (!codec) return null;
+
+ // A small compositor: the video layers only, at mask resolution.
+ const compose = document.createElement("canvas");
+ compose.width = W;
+ compose.height = H;
+ const comp = new FrameCompositor(compose);
+ const mask = document.createElement("canvas");
+ mask.width = W;
+ mask.height = H;
+ const mctx = mask.getContext("2d")!;
+
+ const spans = getClipSpans(doc.clips, doc.assets);
+ const overlayTracks = [...new Set(doc.clips.filter((c) => c.track !== 0).map((c) => c.track))];
+ const byTrack = new Map(overlayTracks.map((track) => [track, getClipSpans(doc.clips, doc.assets, track)]));
+ const spanOfClip = new Map<string, ClipSpan>();
+ for (const list of byTrack.values()) for (const sp of list) spanOfClip.set(sp.clip.id, sp);
+ const readers = new Map<string, ClipReader>();
+ const readerFor = (asset: MediaAsset) => {
+ let r = readers.get(asset.id);
+ if (!r) readers.set(asset.id, (r = new ClipReader(asset, () => asset.url)));
+ return r;
+ };
+
+ const output = new Output({ format: new Mp4OutputFormat(), target: new BufferTarget() });
+ const video = new CanvasSource(mask, { codec, bitrate: 1_000_000 });
+ const frames = Math.max(1, Math.ceil((to - from) * MASK_FPS));
+ output.addVideoTrack(video, { frameRate: MASK_FPS, maximumPacketCount: frames + 8 });
+
+ try {
+ await output.start();
+ for (let i = 0; i < frames; i++) {
+ const t = Math.min(to, from + i / MASK_FPS);
+ comp.clear();
+ const master = spans.find((sp) => t >= sp.start && t < sp.start + sp.len);
+ if (master && !master.clip.hidden) {
+ const plan = trackZeroPlan(master, spans, t);
+ if (plan.backdrop) {
+ const f = await readerFor(plan.backdrop.span.asset).frameAt(plan.backdrop.at);
+ comp.drawLayer(f, plan.backdrop.span.clip, false, 1, t);
+ }
+ const f = await readerFor(master.asset).frameAt(sourceTimeAt(master, t));
+ comp.drawLayer(f ?? MISSING_FRAME, master.clip, false, 1, t);
+ }
+ for (const layer of overlayPlan(overlayTracks, (track) => byTrack.get(track) ?? [], t)) {
+ const span = spanOfClip.get(layer.clip.id);
+ if (!span) continue;
+ const f = await readerFor(layer.asset).frameAt(sourceTimeAt(span, t));
+ const rect = rectOf(layer.clip);
+ const cover = layer.clip.fit === "fill" || (layer.clip.fit == null && isFullRect(rect));
+ comp.drawIntoRect(f, rect, cover, layer.alpha, t, layer.zoom, layer.clip);
+ }
+ // Luma mask: black frame, the subject's alpha silhouette drawn white.
+ mctx.globalCompositeOperation = "source-over";
+ mctx.fillStyle = "#000000";
+ mctx.fillRect(0, 0, W, H);
+ const alpha = segmentSubjectAlpha(segmenter, compose);
+ if (alpha) mctx.drawImage(alpha, 0, 0, W, H);
+ await video.add(i / MASK_FPS, 1 / MASK_FPS);
+ }
+ await output.finalize();
+ const buffer = (output.target as BufferTarget).buffer;
+ if (!buffer) return null;
+ return { blob: new Blob([buffer], { type: "video/mp4" }), from };
+ } catch {
+ await output.cancel().catch(() => {});
+ return null;
+ } finally {
+ for (const r of readers.values()) r.dispose();
+ }
+}
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 6c3e5e42..f1725f15 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -231,7 +231,7 @@ export async function uploadProjectImage(
projectId: string,
file: Blob,
fileName: string,
- opts?: { name?: string; origin?: "generated"; failMessage?: string; backend?: CutBackend }
+ opts?: { name?: string; origin?: "generated" | "sticker"; failMessage?: string; backend?: CutBackend }
): Promise<MediaAsset> {
// Callers whose work outlives navigation (finishing AI generations) pin the
// backend they started on; everyone else rides the active one.
diff --git a/site/src/cut/lib/overlayModel.test.ts b/site/src/cut/lib/overlayModel.test.ts
new file mode 100644
index 00000000..2c5688a9
--- /dev/null
+++ b/site/src/cut/lib/overlayModel.test.ts
@@ -0,0 +1,226 @@
+import { describe, expect, test } from "bun:test";
+import {
+ evalOverlayAnim,
+ hasOverlayAnim,
+ loopPeriod,
+ shapeMetrics,
+ type OverlayAnim,
+} from "@donkeycut/effects-kit";
+import {
+ stampOverlayKinds,
+ stripDefaultOverlayKinds,
+ type Overlay,
+ type ShapeOverlay,
+ type TextOverlay,
+} from "./types";
+
+/** A title exactly as pre-union documents stored it: no `kind`. */
+const legacyTitle = (): TextOverlay => ({
+ id: "t1",
+ text: "Hello",
+ start: 1,
+ end: 4,
+ x: 0.5,
+ y: 0.42,
+ size: 88,
+ font: "sf",
+ weight: 700,
+ color: "#FFFFFF",
+ shadow: true,
+ plate: false,
+ lane: 0,
+});
+
+describe("overlay kind stamping", () => {
+ test("loader stamps text, serializer strips it back — byte-stable round trip", () => {
+ const doc = [legacyTitle(), { ...legacyTitle(), id: "t2", plateRadius: 0.4 }];
+ const before = JSON.stringify(doc);
+ const stamped = stampOverlayKinds(doc);
+ for (const o of stamped) expect(o.kind).toBe("text");
+ expect(JSON.stringify(stripDefaultOverlayKinds(stamped))).toBe(before);
+ });
+
+ test("shapes and stickers keep their kind through serialize", () => {
+ const shape: ShapeOverlay = {
+ id: "s1",
+ kind: "shape",
+ shape: "rect",
+ start: 0,
+ end: 3,
+ x: 0.5,
+ y: 0.5,
+ w: 0.3,
+ h: 0.2,
+ fill: "#FFFFFF",
+ };
+ const sticker: Overlay = {
+ id: "k1",
+ kind: "sticker",
+ assetId: "a1",
+ start: 0,
+ end: 3,
+ x: 0.5,
+ y: 0.5,
+ w: 0.25,
+ };
+ const out = stripDefaultOverlayKinds([shape, sticker]);
+ expect(out[0].kind).toBe("shape");
+ expect(out[1].kind).toBe("sticker");
+ });
+
+ test("stamping already-stamped elements changes nothing", () => {
+ const stamped = stampOverlayKinds([legacyTitle()]);
+ expect(stampOverlayKinds(stamped)).toEqual(stamped);
+ });
+});
+
+describe("shapeMetrics", () => {
+ const frame = { width: 1080, height: 1920, scale: 1 }; // 9:16 at design size
+
+ test("rect centers and sizes in frame pixels", () => {
+ const o: ShapeOverlay = {
+ id: "s",
+ kind: "shape",
+ shape: "rect",
+ start: 0,
+ end: 1,
+ x: 0.5,
+ y: 0.25,
+ w: 0.4,
+ h: 0.1,
+ fill: "#fff",
+ radius: 24,
+ stroke: { color: "#000", width: 6 },
+ };
+ const m = shapeMetrics(o, frame);
+ expect(m.cx).toBe(540);
+ expect(m.cy).toBe(480);
+ expect(m.w).toBeCloseTo(432);
+ expect(m.h).toBeCloseTo(192);
+ expect(m.radius).toBe(24);
+ expect(m.strokeWidth).toBe(6);
+ });
+
+ test("line thickness rides h with a 2px scaled floor", () => {
+ const thin: ShapeOverlay = {
+ id: "l",
+ kind: "shape",
+ shape: "line",
+ start: 0,
+ end: 1,
+ x: 0.5,
+ y: 0.5,
+ w: 0.4,
+ h: 0.0001,
+ fill: "#fff",
+ };
+ expect(shapeMetrics(thin, frame).thickness).toBe(2);
+ expect(shapeMetrics({ ...thin, h: 6 / 1920 }, frame).thickness).toBeCloseTo(6);
+ // Half-resolution render halves the floor with the scale.
+ expect(shapeMetrics(thin, { width: 540, height: 960, scale: 0.5 }).thickness).toBe(1);
+ });
+
+ test("arrow head scales with thickness but never eats more than half the length", () => {
+ const arrow: ShapeOverlay = {
+ id: "a",
+ kind: "shape",
+ shape: "arrow",
+ start: 0,
+ end: 1,
+ x: 0.5,
+ y: 0.5,
+ w: 0.1,
+ h: 20 / 1920,
+ fill: "#fff",
+ };
+ const m = shapeMetrics(arrow, frame);
+ expect(m.headLen).toBeCloseTo(Math.min(m.thickness * 3, m.w / 2));
+ const stubby = shapeMetrics({ ...arrow, w: 0.02 }, frame);
+ expect(stubby.headLen).toBeLessThanOrEqual(stubby.w / 2 + 1e-9);
+ });
+
+ test("metrics scale linearly with render resolution", () => {
+ const o: ShapeOverlay = {
+ id: "s",
+ kind: "shape",
+ shape: "ellipse",
+ start: 0,
+ end: 1,
+ x: 0.3,
+ y: 0.6,
+ w: 0.5,
+ h: 0.2,
+ fill: "#fff",
+ stroke: { color: "#000", width: 8 },
+ };
+ const at1x = shapeMetrics(o, frame);
+ const at2x = shapeMetrics(o, { width: 2160, height: 3840, scale: 2 });
+ expect(at2x.cx).toBeCloseTo(at1x.cx * 2);
+ expect(at2x.w).toBeCloseTo(at1x.w * 2);
+ expect(at2x.strokeWidth).toBeCloseTo(at1x.strokeWidth * 2);
+ });
+});
+
+describe("evalOverlayAnim", () => {
+ const dur = 5;
+
+ test("no anim (or empty anim) is the identity", () => {
+ expect(evalOverlayAnim(undefined, 1, dur)).toEqual({ dx: 0, dy: 0, scale: 1, rotate: 0, alpha: 1 });
+ expect(hasOverlayAnim({})).toBe(false);
+ expect(hasOverlayAnim({ in: { style: "fade", seconds: 0.5 } })).toBe(true);
+ });
+
+ test("fade in ramps alpha 0→1 monotonically and idles after", () => {
+ const anim: OverlayAnim = { in: { style: "fade", seconds: 1 } };
+ expect(evalOverlayAnim(anim, 0, dur).alpha).toBe(0);
+ let prev = -1;
+ for (let t = 0; t <= 1; t += 0.1) {
+ const a = evalOverlayAnim(anim, t, dur).alpha;
+ expect(a).toBeGreaterThanOrEqual(prev);
+ prev = a;
+ }
+ expect(evalOverlayAnim(anim, 2, dur).alpha).toBe(1);
+ expect(evalOverlayAnim(anim, 2, dur).dx).toBe(0);
+ });
+
+ test("slideleft enters from the right and lands exactly at rest", () => {
+ const anim: OverlayAnim = { in: { style: "slideleft", seconds: 0.5 } };
+ expect(evalOverlayAnim(anim, 0, dur).dx).toBeGreaterThan(0);
+ expect(evalOverlayAnim(anim, 0.5, dur).dx).toBeCloseTo(0, 5);
+ // Exit runs the motion onward: slideleft leaves toward the left.
+ const out: OverlayAnim = { out: { style: "slideleft", seconds: 0.5 } };
+ expect(evalOverlayAnim(out, dur - 0.01, dur).dx).toBeLessThan(0);
+ expect(evalOverlayAnim(out, dur - 0.5, dur).dx).toBeCloseTo(0, 5);
+ });
+
+ test("fade out ramps alpha to 0 at the very end", () => {
+ const anim: OverlayAnim = { out: { style: "fade", seconds: 1 } };
+ expect(evalOverlayAnim(anim, dur - 1.01, dur).alpha).toBe(1);
+ expect(evalOverlayAnim(anim, dur, dur).alpha).toBe(0);
+ });
+
+ test("typewriter reports character progress instead of moving", () => {
+ const anim: OverlayAnim = { in: { style: "typewriter", seconds: 1 } };
+ const mid = evalOverlayAnim(anim, 0.5, dur);
+ expect(mid.textProgress).toBeCloseTo(0.5, 5);
+ expect(mid.alpha).toBe(1);
+ expect(evalOverlayAnim(anim, 2, dur).textProgress).toBeUndefined();
+ });
+
+ test("loops are periodic and loopPeriod matches", () => {
+ const anim: OverlayAnim = { loop: { style: "pulse", speed: 2 } };
+ const p = loopPeriod(anim)!;
+ expect(p).toBeCloseTo(0.6, 5);
+ const a = evalOverlayAnim(anim, 1.1, dur);
+ const b = evalOverlayAnim(anim, 1.1 + p, dur);
+ expect(a.scale).toBeCloseTo(b.scale, 5);
+ expect(loopPeriod({ in: { style: "fade", seconds: 1 } }) === null).toBe(true);
+ });
+
+ test("loop composes with the in ramp", () => {
+ const anim: OverlayAnim = { in: { style: "fade", seconds: 1 }, loop: { style: "float", speed: 1 } };
+ const s = evalOverlayAnim(anim, 0.5, dur);
+ expect(s.alpha).toBeGreaterThan(0);
+ expect(s.alpha).toBeLessThan(1);
+ });
+});
diff --git a/site/src/cut/lib/stickerCreate.ts b/site/src/cut/lib/stickerCreate.ts
new file mode 100644
index 00000000..1e4b77d8
--- /dev/null
+++ b/site/src/cut/lib/stickerCreate.ts
@@ -0,0 +1,80 @@
+"use client";
+
+/**
+ * The custom-sticker pipeline: prompt → hosted image generation (die-cut
+ * framing folded around the idea) → background removal → white outline →
+ * saved as an origin-"sticker" project asset. The Elements panel and the
+ * assistant's create_sticker tool share this one path.
+ */
+
+import type { AssetRef } from "./assetRef";
+import { bytesFromBase64 } from "./bytes";
+import { makeStickerCutout } from "./cutout";
+import { promptAndImages } from "./generate";
+import { hostedPost } from "./hosted";
+import { uploadProjectImage } from "./media";
+import { useEditor } from "./store";
+import { mediaSlug, type MediaAsset } from "./types";
+
+/** The die-cut framing that makes a generation come back sticker-shaped: one
+ * centered subject on a plain background the cutout pass can lift it off. */
+const STICKER_FRAMING =
+ "Sticker style: one centered subject with a bold, clean silhouette, on a plain " +
+ "solid light-gray background, no text, no other objects, no border.";
+
+const stickerPrompt = (idea: string) => `A single ${idea.trim()}. ${STICKER_FRAMING}`;
+
+export type StickerStage = "generating" | "cutting" | "saving";
+
+/** Generate, cut out, and save a custom sticker; the asset is registered in
+ * the open project. References ride along as input images, composed the same
+ * way the image panel composes them. Throws user-facing errors (sign-in,
+ * credits, provider). */
+export async function createCustomSticker(
+ projectId: string,
+ idea: string,
+ onStage?: (stage: StickerStage) => void,
+ opts: { refs?: AssetRef[] } = {}
+): Promise<MediaAsset> {
+ onStage?.("generating");
+ const refs = opts.refs ?? [];
+ // With references the composed prompt describes the picture, so the framing
+ // follows it; on its own the idea is folded into the framing directly.
+ const { prompt: composed, images } = await promptAndImages(
+ "image",
+ idea,
+ refs,
+ refs.length > 0,
+ Infinity
+ );
+ const res = await hostedPost("/api/inference/assets", {
+ kind: "image",
+ prompt: refs.length > 0 ? `${composed}\n\n${STICKER_FRAMING}` : stickerPrompt(idea),
+ ...(images.length > 0 ? { inputs: { images } } : {}),
+ parameters: { aspectRatio: "1:1", imageSize: "1K" },
+ });
+ if (!res.ok) {
+ const body = (await res.json().catch(() => ({}))) as { error?: string };
+ throw new Error(body.error ?? "Image generation failed.");
+ }
+ const gen = (await res.json()) as { outputs: { dataBase64?: string; contentType?: string }[] };
+ const out = gen.outputs?.find((o) => o.dataBase64);
+ if (!out?.dataBase64) throw new Error("The provider returned no image.");
+ const source = new Blob([bytesFromBase64(out.dataBase64)], {
+ type: out.contentType ?? "image/png",
+ });
+
+ onStage?.("cutting");
+ // A failed cutout still yields a sticker — the uncut square — rather than a
+ // dead end; regenerating retries the cut.
+ const cut =
+ (await makeStickerCutout(source, { plainBackdrop: true }).catch(() => null)) ?? source;
+
+ onStage?.("saving");
+ const asset = await uploadProjectImage(projectId, cut, `${mediaSlug(idea, "sticker")}.png`, {
+ name: idea,
+ origin: "sticker",
+ });
+ useEditor.getState().addAsset(asset);
+ return asset;
+}
diff --git a/site/src/cut/lib/stylePresets.ts b/site/src/cut/lib/stylePresets.ts
new file mode 100644
index 00000000..a24b341b
--- /dev/null
+++ b/site/src/cut/lib/stylePresets.ts
@@ -0,0 +1,88 @@
+"use client";
+
+/**
+ * Named text-style presets, saved in the shared Library. A preset rides the
+ * existing template rails: it is a library template with no media and exactly
+ * one text overlay — the style carrier — so both residencies store, list,
+ * and delete it with machinery that already exists. The localStorage style in
+ * textStyle.ts stays what it was: the implicit "last used" style.
+ */
+
+import { pickTextStyle } from "@donkeycut/effects-kit";
+import { deleteTemplate, fetchLibrary, saveTemplate } from "./library";
+import type { Residency } from "./residency";
+import type { TextStyle } from "./textStyle";
+import {
+ isTextOverlay,
+ type LibraryTemplate,
+ type TextOverlay,
+} from "./types";
+
+export interface StylePreset {
+ id: string;
+ name: string;
+ residency: Residency;
+ style: TextStyle;
+}
+
+/** The carrier's element id, which is what marks a template as a preset. Real
+ * overlays carry a `uid()`, so the two never collide and a user's own
+ * single-title template stays an ordinary template. */
+const CARRIER_ID = "style";
+
+/** Whether a library template is a saved style rather than a template the user
+ * can drop on the timeline. Template surfaces filter on this so presets stay
+ * in the text inspector where they were saved. */
+export function isStylePresetTemplate(t: LibraryTemplate): boolean {
+ if (t.media.length || t.layers.length || t.audio.length || t.cues.length) return false;
+ if (t.texts.length !== 1) return false;
+ const carrier = t.texts[0];
+ return isTextOverlay(carrier) && carrier.id === CARRIER_ID;
+}
+
+/** Read a template as a style preset, or null when it is an ordinary one. */
+export function presetOf(t: LibraryTemplate & { residency: Residency }): StylePreset | null {
+ if (!isStylePresetTemplate(t)) return null;
+ const carrier = t.texts[0] as TextOverlay;
+ return { id: t.id, name: t.name, residency: t.residency, style: pickTextStyle(carrier) };
+}
+
+/** Every style preset across both shelves, newest first. */
+export async function listStylePresets(): Promise<StylePreset[]> {
+ const lib = await fetchLibrary();
+ return lib.templates
+ .map(presetOf)
+ .filter((p): p is StylePreset => p !== null);
+}
+
+/** Save the given title's look as a named preset on the active shelf. */
+export async function saveStylePreset(
+ projectId: string,
+ name: string,
+ overlay: TextOverlay
+): Promise<StylePreset | null> {
+ const carrier: TextOverlay = {
+ id: CARRIER_ID,
+ kind: "text",
+ text: "Aa",
+ start: 0,
+ end: 3,
+ x: 0.5,
+ y: 0.5,
+ ...pickTextStyle(overlay),
+ };
+ const saved = await saveTemplate(projectId, {
+ name,
+ duration: 3,
+ media: [],
+ layers: [],
+ audio: [],
+ texts: [carrier],
+ cues: [],
+ });
+ return presetOf(saved);
+}
+
+export function deleteStylePreset(preset: StylePreset): Promise<void> {
+ return deleteTemplate(preset.residency, preset.id);
+}
diff --git a/site/src/cut/lib/textRender.ts b/site/src/cut/lib/textRender.ts
index b0a3a8f8..6cd95548 100644
--- a/site/src/cut/lib/textRender.ts
+++ b/site/src/cut/lib/textRender.ts
@@ -1,125 +1,106 @@
"use client";
-import { fontStack, type TextOverlay } from "./types";
+import {
+ renderElementPng as kitRenderElementPng,
+ renderOverlayFrames as kitRenderOverlayFrames,
+ type OverlayFrameSet,
+ type RenderEnv,
+ type StickerImage,
+} from "@donkeycut/effects-kit";
+import { fontStack, type MediaAsset, type Overlay } from "./types";
-export const LINE_HEIGHT = 1.25;
-export const PLATE_PAD_X = 0.55; // em
-export const PLATE_PAD_Y = 0.3; // em
-export const PLATE_RADIUS = 0.32; // em
-export const PLATE_COLOR = "#000000";
-export const PLATE_OPACITY = 0.55;
-export const PLATE_FILL = "rgba(0, 0, 0, 0.55)";
-export const SHADOW = { color: "rgba(0, 0, 0, 0.65)", blur: 14, offsetY: 2 };
+// The shared text metrics and painters live in the effects kit; these
+// re-exports keep the app's preview components on the same constants.
+export {
+ LINE_HEIGHT,
+ PLATE_COLOR,
+ PLATE_FILL,
+ PLATE_OPACITY,
+ PLATE_PAD_X,
+ PLATE_PAD_Y,
+ PLATE_RADIUS,
+ plateFill,
+ SHADOW,
+} from "@donkeycut/effects-kit";
-/** A title's plate fill as an rgba() string, defaulting to translucent black
- * when color/opacity are unset. Shared by the preview and the export burn-in. */
-export function plateFill(o: { plateColor?: string; plateOpacity?: number }): string {
- const a = o.plateOpacity ?? PLATE_OPACITY;
- const m = /^#?([0-9a-fA-F]{6})$/.exec((o.plateColor ?? PLATE_COLOR).trim());
- if (!m) return `rgba(0, 0, 0, ${a})`;
- const n = parseInt(m[1], 16);
- return `rgba(${(n >> 16) & 255}, ${(n >> 8) & 255}, ${n & 255}, ${a})`;
+/** Decoded sticker images by asset id, shared across one page's renders. An
+ * <img> decode handles SVG too (createImageBitmap on an SVG blob does not). */
+const stickerCache = new Map<string, Promise<StickerImage | null>>();
+
+function decodeSticker(url: string): Promise<StickerImage | null> {
+ return fetch(url)
+ .then((res) => (res.ok ? res.blob() : Promise.reject(new Error("fetch failed"))))
+ .then(
+ (blob) =>
+ new Promise<StickerImage | null>((resolve) => {
+ const objectUrl = URL.createObjectURL(blob);
+ const img = new Image();
+ img.onload = () => {
+ URL.revokeObjectURL(objectUrl);
+ resolve({
+ source: img,
+ // An SVG with no intrinsic size reports 0×0; treat it as square.
+ width: img.naturalWidth || 512,
+ height: img.naturalHeight || 512,
+ });
+ };
+ img.onerror = () => {
+ URL.revokeObjectURL(objectUrl);
+ resolve(null);
+ };
+ img.src = objectUrl;
+ })
+ )
+ .catch(() => null);
+}
+
+/** The kit render env bound to this app: Cut's font stacks, and sticker
+ * assets resolved from the given project media (their URLs already point at
+ * the active backend — local engine or signed R2). */
+export function cutRenderEnv(assets: MediaAsset[]): RenderEnv {
+ return {
+ fontStack,
+ resolveLottie: (assetId) =>
+ import("./lottieAssets").then((m) => m.sharedLottieHandle(assetId, assets)),
+ resolveAsset: (assetId) => {
+ const asset = assets.find((a) => a.id === assetId);
+ if (!asset) return Promise.resolve(null);
+ let hit = stickerCache.get(asset.id);
+ if (!hit) {
+ hit = decodeSticker(asset.url).then((img) => {
+ // A failed decode is not worth pinning; the next render retries.
+ if (!img) stickerCache.delete(asset.id);
+ return img;
+ });
+ stickerCache.set(asset.id, hit);
+ }
+ return hit;
+ },
+ };
}
/**
- * Render an overlay to a transparent full-frame PNG at the export resolution.
- * Uses the same font stacks and metrics as the DOM preview so the export
- * matches what the user sees.
+ * Render an overlay element (text, shape, or sticker) to a transparent
+ * full-frame PNG at the export resolution, matching the DOM preview's
+ * metrics. `assets` supplies sticker bytes; text and shapes need none.
*/
-export async function renderOverlayPng(
- overlay: TextOverlay,
+export function renderElementPng(
+ overlay: Overlay,
width: number,
- height: number
+ height: number,
+ assets: MediaAsset[] = []
): Promise<Blob> {
- // Overlay sizes are frame pixels with a 1080 design short side (9:16 is
- // 1080 wide, 16:9 is 1080 tall), so scaling by the short side keeps text
- // the same visual size in either aspect and at any export resolution.
- const scale = Math.min(width, height) / 1080;
- const canvas = document.createElement("canvas");
- canvas.width = width;
- canvas.height = height;
- const ctx = canvas.getContext("2d")!;
-
- const fpx = overlay.size * scale;
- ctx.font = `${overlay.weight} ${fpx}px ${fontStack(overlay.font)}`;
- ctx.textAlign = "center";
- ctx.textBaseline = "middle";
-
- const lines = overlay.text.split("\n");
- const lineH = fpx * LINE_HEIGHT;
- const totalH = lines.length * lineH;
- const cx = overlay.x * width;
- const cy = overlay.y * height;
-
- if (overlay.plate) {
- const maxW = Math.max(...lines.map((l) => ctx.measureText(l).width), 1);
- const padX = PLATE_PAD_X * fpx;
- const padY = PLATE_PAD_Y * fpx;
- const r = (overlay.plateRadius ?? PLATE_RADIUS) * fpx;
- const w = maxW + padX * 2;
- const h = totalH + padY * 2;
- ctx.fillStyle = plateFill(overlay);
- ctx.beginPath();
- ctx.roundRect(cx - w / 2, cy - h / 2, w, h, r);
- ctx.fill();
- }
-
- if (overlay.shadow) {
- ctx.shadowColor = SHADOW.color;
- ctx.shadowBlur = SHADOW.blur * scale;
- ctx.shadowOffsetY = SHADOW.offsetY * scale;
- }
-
- ctx.fillStyle = overlay.color;
- // Karaoke: draw word by word so the spoken word gets the accent color and an
- // underline; the word index counts across all lines.
- let k = 0;
- lines.forEach((line, i) => {
- const y = cy - totalH / 2 + lineH * (i + 0.5);
- if (overlay.highlightWord === undefined) {
- ctx.fillText(line, cx, y);
- return;
- }
- const words = line.split(" ").filter(Boolean);
- const spaceW = ctx.measureText(" ").width;
- const widths = words.map((w) => ctx.measureText(w).width);
- const lineW = widths.reduce((a, b) => a + b, 0) + spaceW * (words.length - 1);
- let x = cx - lineW / 2;
- ctx.textAlign = "left";
- words.forEach((w, wi) => {
- const active = k === overlay.highlightWord;
- if (active && overlay.highlightMode === "box") {
- // Accent box behind the word, contrast text on top — drawn with the
- // shadow off so the box and its word stay crisp.
- const pad = 0.12 * fpx;
- const prevShadow = ctx.shadowColor;
- ctx.shadowColor = "transparent";
- ctx.fillStyle = overlay.highlightColor ?? "#FFE94A";
- ctx.beginPath();
- ctx.roundRect(x - pad, y - fpx * 0.5 - pad, widths[wi] + pad * 2, fpx + pad * 2, 0.18 * fpx);
- ctx.fill();
- ctx.fillStyle = overlay.highlightText ?? "#111114";
- ctx.fillText(w, x, y);
- ctx.shadowColor = prevShadow;
- } else if (active) {
- ctx.fillStyle = overlay.highlightColor ?? "#FFE94A";
- ctx.fillText(w, x, y);
- if (overlay.highlightMode !== "color")
- ctx.fillRect(x, y + fpx * 0.42, widths[wi], Math.max(2 * scale, fpx * 0.07));
- } else {
- ctx.fillStyle = overlay.color;
- ctx.fillText(w, x, y);
- }
- x += widths[wi] + spaceW;
- k++;
- });
- ctx.textAlign = "center";
- });
+ return kitRenderElementPng(overlay, width, height, cutRenderEnv(assets));
+}
- return new Promise((resolve, reject) =>
- canvas.toBlob(
- (b) => (b ? resolve(b) : reject(new Error("Could not render text overlay."))),
- "image/png"
- )
- );
+/** Rasterize an animated element into its region-cropped frame set (the
+ * export's ffconcat slideshow food), bound to this app's render env. */
+export function renderElementFrames(
+ overlay: Overlay,
+ width: number,
+ height: number,
+ fps: number,
+ assets: MediaAsset[] = []
+): Promise<OverlayFrameSet> {
+ return kitRenderOverlayFrames(overlay, width, height, fps, cutRenderEnv(assets));
}
diff --git a/site/src/cut/lib/textStyle.ts b/site/src/cut/lib/textStyle.ts
index 4a043e29..dddf2e32 100644
--- a/site/src/cut/lib/textStyle.ts
+++ b/site/src/cut/lib/textStyle.ts
@@ -1,12 +1,7 @@
-import type { TextOverlay } from "./types";
-
-// The style fields a title carries, minus its text/position/id/timing. Remembered
-// across clips and projects so repeated titles share one look. Position and size
-// stay at sensible defaults; only the visual style is reused.
-export type TextStyle = Pick<
- TextOverlay,
- "size" | "font" | "weight" | "color" | "shadow" | "plate" | "plateColor" | "plateOpacity" | "plateRadius"
->;
+// What a title's look is made of lives with the element model; this module is
+// about remembering one across clips and projects so repeated titles share it.
+export type { TextStyle } from "@donkeycut/effects-kit";
+import type { TextStyle } from "@donkeycut/effects-kit";
const KEY = "cut-text-style";
diff --git a/site/src/cut/server/cloud/media.ts b/site/src/cut/server/cloud/media.ts
index 9bf5ec6e..3d5ec362 100644
--- a/site/src/cut/server/cloud/media.ts
+++ b/site/src/cut/server/cloud/media.ts
@@ -173,7 +173,8 @@ export const mediaCloud = {
if (over) return over;
const nameField = form.get("name");
const name = typeof nameField === "string" && nameField.trim() ? nameField.trim() : file.name;
- const origin = form.get("origin") === "generated" ? "generated" : undefined;
+ const raw = form.get("origin");
+ const origin = raw === "generated" || raw === "sticker" ? raw : undefined;
const fileName = dedupeName(
safeFileName(file.name),
await takenMediaNames(userId, projectId)