Skip to content

Open source CapCut alternative.

TypeScript

refactor(site): every cutout resolves to the one baked matte [rebuild]

David Le7de0d46adc27
diff --git a/site/packages/effects-kit/src/removal.test.ts b/site/packages/effects-kit/src/removal.test.ts
index 38726d89..9d84b399 100644
--- a/site/packages/effects-kit/src/removal.test.ts
+++ b/site/packages/effects-kit/src/removal.test.ts
@@ -1,93 +1,5 @@
import { describe, expect, test } from "bun:test";
-import {
- CHROMA_DEFAULT_INTENSITY,
- CHROMA_DEFAULT_SOFTNESS,
- CHROMA_DEFAULT_SPILL,
- CHROMA_NEAR_MAX,
- chromaAlphaInto,
- chromaAlphaOf,
- chromaParams,
- removalActive,
- removalFingerprint,
- removalNeedsBake,
- type ClipRemoval,
-} from "./removal";
-
-/** One RGBA pixel buffer of solid color. */
-function frame(px: [number, number, number][], alpha = 255): Uint8ClampedArray {
- const data = new Uint8ClampedArray(px.length * 4);
- px.forEach(([r, g, b], i) => {
- data[i * 4] = r;
- data[i * 4 + 1] = g;
- data[i * 4 + 2] = b;
- data[i * 4 + 3] = alpha;
- });
- return data;
-}
-
-describe("chroma params", () => {
- test("defaults fill and the soft band sits past the tolerance", () => {
- const p = chromaParams({ color: "#00ff00" });
- expect(p.near).toBeCloseTo(CHROMA_DEFAULT_INTENSITY * CHROMA_NEAR_MAX, 5);
- expect(p.far).toBeGreaterThan(p.near);
- expect(p.spill).toBe(CHROMA_DEFAULT_SPILL);
- expect(Math.hypot(p.dirCb, p.dirCr)).toBeCloseTo(1, 5);
- });
-
- test("intensity widens the removed core and softness widens the band", () => {
- const tight = chromaParams({ color: "#00ff00", intensity: 0.2, softness: CHROMA_DEFAULT_SOFTNESS });
- const wide = chromaParams({ color: "#00ff00", intensity: 0.9, softness: CHROMA_DEFAULT_SOFTNESS });
- expect(wide.near).toBeGreaterThan(tight.near);
- const hard = chromaParams({ color: "#00ff00", softness: 0.05 });
- const soft = chromaParams({ color: "#00ff00", softness: 1 });
- expect(soft.far - soft.near).toBeGreaterThan(hard.far - hard.near);
- });
-
- test("alpha ramps monotonically from removed to kept", () => {
- const p = chromaParams({ color: "#00ff00" });
- expect(chromaAlphaOf(0, p)).toBe(0);
- expect(chromaAlphaOf(p.near, p)).toBe(0);
- expect(chromaAlphaOf(p.far, p)).toBe(1);
- expect(chromaAlphaOf(1, p)).toBe(1);
- const mid = chromaAlphaOf((p.near + p.far) / 2, p);
- expect(mid).toBeGreaterThan(0);
- expect(mid).toBeLessThan(1);
- });
-});
-
-describe("chroma keying", () => {
- test("the key color goes transparent and distant colors stay", () => {
- const px = frame([
- [0, 255, 0], // the key itself
- [255, 0, 0], // far from the key
- [255, 255, 255], // neutral
- ]);
- chromaAlphaInto(px, { color: "#00ff00" });
- expect(px[3]).toBe(0);
- expect(px[7]).toBe(255);
- expect(px[11]).toBe(255);
- });
-
- test("spill suppression pulls the key's tint off kept fringe", () => {
- // A green-fringed pixel just past the key's soft band keeps its alpha
- // while its green excess shrinks.
- const px = frame([[120, 230, 90]]);
- const before = px[1] - (px[0] + px[2]) / 2;
- chromaAlphaInto(px, { color: "#00ff00", intensity: 0.6, spill: 1 });
- const after = px[1] - (px[0] + px[2]) / 2;
- expect(px[3]).toBeGreaterThan(200);
- expect(after).toBeLessThan(before);
- });
-
- test("spill 0 leaves kept colors untouched", () => {
- const px = frame([[150, 205, 130]]);
- const want = [...px];
- chromaAlphaInto(px, { color: "#00ff00", intensity: 0.1, softness: 0.1, spill: 0 });
- expect(px[0]).toBe(want[0]);
- expect(px[1]).toBe(want[1]);
- expect(px[2]).toBe(want[2]);
- });
-});
+import { removalActive, removalFingerprint, removalNeedsBake, type ClipRemoval } from "./removal";
describe("removal model", () => {
const seeds = { prompts: [{ t: 1, points: [{ x: 0.5, y: 0.5, label: 1 as const }] }] };
@@ -126,17 +38,18 @@ describe("removal model", () => {
test("active and needs-bake read the mode", () => {
expect(removalActive(undefined)).toBe(false);
expect(removalActive({ mode: "auto" })).toBe(true);
- expect(removalActive({ mode: "chroma" })).toBe(false);
- expect(removalActive({ mode: "chroma", chroma: { color: "#00ff00" } })).toBe(true);
- // A bake is owed only once it was explicitly started.
+ // Off keeps the removal on the clip but renders and bakes nothing.
+ expect(removalActive({ mode: "auto", off: true })).toBe(false);
+ expect(removalNeedsBake({ mode: "auto", off: true })).toBe(false);
+ // A bake is owed only once Apply (or the chat tool) requests it.
expect(removalNeedsBake({ mode: "auto" })).toBe(false);
expect(removalNeedsBake({ mode: "auto", requested: true })).toBe(true);
+ expect(removalNeedsBake({ mode: "custom" })).toBe(false);
+ expect(removalNeedsBake({ mode: "custom", requested: true })).toBe(true);
const baked: ClipRemoval = {
mode: "auto",
- requested: true,
matte: { assetId: "m", fingerprint: "f", quality: "local", in: 0 },
};
expect(removalNeedsBake(baked)).toBe(false);
- expect(removalNeedsBake({ mode: "chroma" })).toBe(false);
});
});
diff --git a/site/packages/effects-kit/src/removal.ts b/site/packages/effects-kit/src/removal.ts
index 2e724ef3..4666d099 100644
--- a/site/packages/effects-kit/src/removal.ts
+++ b/site/packages/effects-kit/src/removal.ts
@@ -1,34 +1,14 @@
/**
- * Background removal: the per-clip cutout model and the reference math every
- * renderer shares.
+ * Background removal: the per-clip cutout model every renderer shares.
*
- * A removal turns a clip's picture into a keyed layer: an alpha matte decides
- * what stays (a chroma key computed live, or an AI matte baked to a grayscale
- * video the clip references), a stroke draws ink around the silhouette, and a
- * backdrop fills in behind it. The model lives here so the doc, the panels,
- * the chat tools, and all three renderers read one shape — and the chroma
- * math lives beside it so the GPU shader, the CPU pass, and the export's
- * matte videos resolve a key the same way to the pixel.
+ * A removal turns a clip's picture into a keyed layer: an AI matte baked to a
+ * grayscale video the clip references decides what stays, a stroke draws ink
+ * around the silhouette, and a backdrop fills in behind it. The model lives
+ * here so the doc, the panels, the chat tools, and all three renderers read
+ * one shape.
*/
-export type RemovalMode = "auto" | "custom" | "chroma";
-
-/** A chroma key: the picked backdrop color plus how it reaches. */
-export interface ChromaKey {
- /** Picked key color, "#rrggbb". */
- color: string;
- /** Tolerance 0..1 — how far from the key a color still keys out. */
- intensity?: number;
- /** Edge rolloff 0..1 — how wide the soft band past the tolerance runs. */
- softness?: number;
- /** Spill suppression 0..1 — how hard key-colored fringe on the kept
- * pixels is pulled back toward neutral. */
- spill?: number;
-}
-
-export const CHROMA_DEFAULT_INTENSITY = 0.4;
-export const CHROMA_DEFAULT_SOFTNESS = 0.25;
-export const CHROMA_DEFAULT_SPILL = 0.5;
+export type RemovalMode = "auto" | "custom";
export const STROKE_STYLES = ["glow", "hand", "cut", "solid", "offset", "dotted"] as const;
export type StrokeStyleId = (typeof STROKE_STYLES)[number];
@@ -46,6 +26,7 @@ export const STROKE_STYLE_LABELS: Record<StrokeStyleId, string> = {
export const STROKE_DEFAULT_WIDTH = 12;
export const STROKE_WIDTH_MAX = 60;
export const STROKE_OFFSET_MAX = 120;
+export const STROKE_FEATHER_MAX = 40;
export interface RemovalStroke {
style: StrokeStyleId;
@@ -53,6 +34,9 @@ export interface RemovalStroke {
color: string;
/** Ink thickness, design px; absent = STROKE_DEFAULT_WIDTH. */
width?: number;
+ /** Edge softness: the finished ink blurred by this many design px;
+ * absent = crisp. */
+ feather?: number;
/** Silhouette displacement for the "offset" style, design px per axis. */
offsetX?: number;
offsetY?: number;
@@ -77,8 +61,15 @@ export interface RemovalSeeds {
export interface ClipRemoval {
mode: RemovalMode;
- /** The key, for mode "chroma". */
- chroma?: ChromaKey;
+ /** The cutout is switched off: nothing renders and no bake runs, but the
+ * baked matte and every setting stay on the clip, so switching back on is
+ * instant and re-bills nothing. */
+ off?: boolean;
+ /** The direction. Absent, the selection stays and the picture around it is
+ * removed; true, the selection itself is removed and the rest stays. The
+ * selection is the same either way — the mode only picks how it is found —
+ * so a baked matte serves both directions and flipping re-bakes nothing. */
+ invert?: boolean;
/** Custom mode by description: what to keep, in a few words ("the dog").
* The concept tracker mattes every match; painted seeds take over when the
* user brushes instead. */
@@ -91,24 +82,30 @@ export interface ClipRemoval {
matte?: { assetId: string; fingerprint: string; quality: "local" | "hq"; in: number };
/** Custom-mode selection. */
seeds?: RemovalSeeds;
- /** The bake was explicitly started — the panel's Start/Apply button or the
- * chat tool. AI modes bake nothing until this is set, so picking a mode by
- * itself never spends compute. */
+ /** The bake was explicitly started — the panel's Apply button or the chat
+ * tool. Nothing bakes until this is set: a mode pick, strokes, and words
+ * only describe the selection, so no work runs (and no credits are spent)
+ * on their own. */
requested?: boolean;
+ /** The hosted quality pass was asked for — the panel's Apply button once
+ * the free matte stands, or the chat tool. Auto mode's paid refine runs
+ * only once this is set; custom bakes are the hosted tracker's from the
+ * start. */
+ refine?: boolean;
stroke?: RemovalStroke;
backdrop?: RemovalBackdrop;
}
/** Whether the removal changes the picture at all. */
export function removalActive(r?: ClipRemoval): boolean {
- if (!r) return false;
- if (r.mode === "chroma") return !!r.chroma?.color;
- return true;
+ return !!r && !r.off;
}
-/** An AI-mode removal whose started bake has no matte yet — still owed. */
+/** A removal whose owed bake has no matte yet: a bake is owed once Apply
+ * (or the chat tool) requests it, and a switched-off cutout owes nothing. */
export function removalNeedsBake(r?: ClipRemoval): boolean {
- return !!r && (r.mode === "auto" || r.mode === "custom") && !!r.requested && !r.matte;
+ if (!r || r.off || r.matte) return false;
+ return !!r.requested;
}
/** What a baked matte was computed from: the source and the selection.
@@ -133,78 +130,6 @@ export function removalFingerprint(
return hash.toString(36);
}
-// ---------------------------------------------------------------------------
-// Chroma reference math. The WebGL shader mirrors these formulas with the
-// same constants, so the GPU pass, this CPU pass, and the export mattes all
-// key the same frame to the same alpha.
-
-/** Chroma-plane distance that keys fully out at intensity 1. */
-export const CHROMA_NEAR_MAX = 0.38;
-/** Width of the soft band past the tolerance at softness 1. */
-export const CHROMA_SOFT_MAX = 0.3;
-/** How far past the soft band spill suppression keeps reaching. */
-export const CHROMA_SPILL_REACH = 2;
-
-export interface ChromaParams {
- /** Key chroma in BT.709 Cb/Cr, each -0.5..0.5. */
- keyCb: number;
- keyCr: number;
- /** Unit direction of the key's chroma, for the spill projection. */
- dirCb: number;
- dirCr: number;
- /** Distances in chroma-plane units: fully removed inside `near`, fully
- * kept past `far`. */
- near: number;
- far: number;
- spill: number;
-}
-
-function hexChannel(hex: string, i: number): number {
- const v = parseInt(hex.slice(1 + i * 2, 3 + i * 2), 16);
- return Number.isFinite(v) ? v / 255 : 0;
-}
-
-/** BT.709 chroma of an r/g/b in 0..1. */
-function chromaOf(r: number, g: number, b: number): { cb: number; cr: number } {
- const y = 0.2126 * r + 0.7152 * g + 0.0722 * b;
- return { cb: (b - y) / 1.8556, cr: (r - y) / 1.5748 };
-}
-
-/** Resolve a key's sliders into the numbers the passes run on. */
-export function chromaParams(key: ChromaKey): ChromaParams {
- const r = hexChannel(key.color, 0);
- const g = hexChannel(key.color, 1);
- const b = hexChannel(key.color, 2);
- const { cb, cr } = chromaOf(r, g, b);
- const len = Math.hypot(cb, cr);
- const intensity = key.intensity ?? CHROMA_DEFAULT_INTENSITY;
- const softness = key.softness ?? CHROMA_DEFAULT_SOFTNESS;
- const near = Math.max(0.02, intensity * CHROMA_NEAR_MAX);
- return {
- keyCb: cb,
- keyCr: cr,
- dirCb: len > 0.001 ? cb / len : 0,
- dirCr: len > 0.001 ? cr / len : 1,
- near,
- far: near + Math.max(0.01, softness * CHROMA_SOFT_MAX),
- spill: key.spill ?? CHROMA_DEFAULT_SPILL,
- };
-}
-
-const smooth = (t: number) => t * t * (3 - 2 * t);
-
-/** The alpha a pixel keeps under the key, 0..1, from its chroma distance. */
-export function chromaAlphaOf(d: number, p: ChromaParams): number {
- const t = Math.min(1, Math.max(0, (d - p.near) / (p.far - p.near)));
- return smooth(t);
-}
-
-/**
- * Key an RGBA buffer in place: each pixel's alpha multiplies by how far its
- * chroma sits from the key, and kept pixels near the key lose their fringe —
- * the key's chroma direction is projected out of them, scaled by `spill` and
- * by how close they sit. This is the reference pass; the shader repeats it.
- */
/** The baked-matte contract every producer and reader shares: encoded matte
* rate (coverage moves little in 1/15s, and every consumer's fps filter or
* nearest-frame read duplicates up to the output rate), the frame's short
@@ -243,36 +168,3 @@ export function coverageToGray(px: Uint8ClampedArray): void {
px[i + 3] = 255;
}
}
-
-export function chromaAlphaInto(px: Uint8ClampedArray, key: ChromaKey): void {
- const p = chromaParams(key);
- const spillFar = p.near + (p.far - p.near) * CHROMA_SPILL_REACH;
- for (let i = 0; i < px.length; i += 4) {
- const r = px[i] / 255;
- const g = px[i + 1] / 255;
- const b = px[i + 2] / 255;
- const y = 0.2126 * r + 0.7152 * g + 0.0722 * b;
- const cb = (b - y) / 1.8556;
- const cr = (r - y) / 1.5748;
- const d = Math.hypot(cb - p.keyCb, cr - p.keyCr);
- const a = chromaAlphaOf(d, p);
- if (a < 1 || d < spillFar) {
- px[i + 3] = Math.round(px[i + 3] * a);
- if (a > 0 && p.spill > 0 && d < spillFar) {
- const proj = cb * p.dirCb + cr * p.dirCr;
- if (proj > 0) {
- const reach = 1 - Math.min(1, Math.max(0, (d - p.near) / (spillFar - p.near)));
- const cut = proj * p.spill * smooth(reach);
- const ncb = cb - p.dirCb * cut;
- const ncr = cr - p.dirCr * cut;
- const nr = y + 1.5748 * ncr;
- const nb = y + 1.8556 * ncb;
- const ng = (y - 0.2126 * nr - 0.0722 * nb) / 0.7152;
- px[i] = Math.max(0, Math.min(255, Math.round(nr * 255)));
- px[i + 1] = Math.max(0, Math.min(255, Math.round(ng * 255)));
- px[i + 2] = Math.max(0, Math.min(255, Math.round(nb * 255)));
- }
- }
- }
- }
-}
diff --git a/site/packages/effects-kit/src/strokeInk.ts b/site/packages/effects-kit/src/strokeInk.ts
index 09a43559..e696a304 100644
--- a/site/packages/effects-kit/src/strokeInk.ts
+++ b/site/packages/effects-kit/src/strokeInk.ts
@@ -184,6 +184,28 @@ function whiteSilhouette(silhouette: CanvasImageSource, w: number, h: number): H
return sil;
}
+/** Everything around the silhouette as white fill — the stamp source for
+ * inset ink, where the band grows inward from the boundary. */
+let compCanvas: HTMLCanvasElement | null = null;
+function whiteComplement(silhouette: CanvasImageSource, w: number, h: number): HTMLCanvasElement {
+ const comp = (compCanvas = sizedScratch(compCanvas, w, h));
+ const ctx = comp.getContext("2d")!;
+ ctx.globalCompositeOperation = "source-over";
+ ctx.fillStyle = "#ffffff";
+ ctx.fillRect(0, 0, w, h);
+ ctx.globalCompositeOperation = "destination-out";
+ ctx.drawImage(silhouette, 0, 0, w, h);
+ ctx.globalCompositeOperation = "source-over";
+ return comp;
+}
+
+/** Trim what this paint laid down to the silhouette's own pixels. */
+function trimToSilhouette(ctx: Canvas2D, silhouette: CanvasImageSource, w: number, h: number) {
+ ctx.globalCompositeOperation = "destination-in";
+ ctx.drawImage(silhouette, 0, 0, w, h);
+ ctx.globalCompositeOperation = "source-over";
+}
+
/** Stamp `sil` at `count` points around a circle of radius `r`. */
function stampAround(ctx: Canvas2D, sil: CanvasImageSource, r: number, count = DILATE_STAMPS) {
for (let k = 0; k < count; k++) {
@@ -212,6 +234,18 @@ function contoursOf(silhouette: CanvasImageSource, w: number, h: number): Contou
* `w`×`h` frame the silhouette was drawn at. `scale` converts the stroke's
* design px (1080 short side); `t` is the layer's local seconds, driving the
* hand style's boil.
+ *
+ * The silhouette is always the selection. `inset` flips which side of its
+ * boundary the dilate styles grow toward: plain, the band grows outward and
+ * the interior hides behind the kept subject; inset (a removal that cuts the
+ * selection out), the band grows inward and is trimmed to the selection, so
+ * the ink rims the hole instead of flooding it. The contour styles walk the
+ * boundary itself and need no flip. `ctx` must hold only this ink — the
+ * inset trim clears everything else.
+ *
+ * `feather` softens the finished ink's edge: the styles paint crisp into a
+ * scratch and the blurred scratch lands in `ctx`, so every style feathers
+ * the same way and the host's colorize pass keeps the soft alpha.
*/
export function paintStrokeInk(
ctx: Canvas2D,
@@ -220,7 +254,53 @@ export function paintStrokeInk(
h: number,
stroke: RemovalStroke,
t: number,
- scale: number
+ scale: number,
+ inset = false
+): void {
+ const feather = Math.max(0, (stroke.feather ?? 0) * scale);
+ if (feather >= 0.5) {
+ const fc = (featherCanvas = sizedScratch(featherCanvas, w, h));
+ const fctx = fc.getContext("2d")!;
+ fctx.setTransform(1, 0, 0, 1, 0, 0);
+ fctx.globalCompositeOperation = "source-over";
+ fctx.clearRect(0, 0, w, h);
+ paintInkCore(fctx, silhouette, w, h, stroke, t, scale, inset);
+ ctx.save();
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.globalCompositeOperation = "source-over";
+ if ("filter" in ctx) {
+ ctx.filter = `blur(${(feather / 2).toFixed(1)}px)`;
+ ctx.drawImage(fc, 0, 0);
+ ctx.filter = "none";
+ } else {
+ // No blur available (a bare context): fade the edge by stamping the
+ // ink at rings of falling alpha, the way the glow style degrades.
+ const bare = ctx as Canvas2D;
+ bare.globalAlpha = 0.55;
+ bare.drawImage(fc, 0, 0);
+ for (let ring = 1; ring <= 3; ring++) {
+ bare.globalAlpha = 0.3 / ring;
+ stampAround(bare, fc, (feather * ring) / 3);
+ }
+ bare.globalAlpha = 1;
+ }
+ ctx.restore();
+ return;
+ }
+ paintInkCore(ctx, silhouette, w, h, stroke, t, scale, inset);
+}
+
+let featherCanvas: HTMLCanvasElement | null = null;
+
+function paintInkCore(
+ ctx: Canvas2D,
+ silhouette: CanvasImageSource,
+ w: number,
+ h: number,
+ stroke: RemovalStroke,
+ t: number,
+ scale: number,
+ inset: boolean
): void {
const width = Math.max(1, (stroke.width ?? STROKE_DEFAULT_WIDTH) * scale);
ctx.save();
@@ -229,12 +309,18 @@ export function paintStrokeInk(
ctx.fillStyle = "#ffffff";
ctx.strokeStyle = "#ffffff";
if (stroke.style === "solid") {
- stampAround(ctx, whiteSilhouette(silhouette, w, h), width);
+ if (inset) {
+ stampAround(ctx, whiteComplement(silhouette, w, h), width);
+ trimToSilhouette(ctx, silhouette, w, h);
+ } else {
+ stampAround(ctx, whiteSilhouette(silhouette, w, h), width);
+ }
} else if (stroke.style === "offset") {
const sil = whiteSilhouette(silhouette, w, h);
ctx.drawImage(sil, (stroke.offsetX ?? 0) * scale, (stroke.offsetY ?? 0) * scale);
+ if (inset) trimToSilhouette(ctx, silhouette, w, h);
} else if (stroke.style === "glow") {
- const sil = whiteSilhouette(silhouette, w, h);
+ const sil = inset ? whiteComplement(silhouette, w, h) : whiteSilhouette(silhouette, w, h);
if ("filter" in ctx) {
ctx.filter = `blur(${Math.max(1, width).toFixed(1)}px)`;
// Twice, so the halo reads against bright footage too.
@@ -251,6 +337,7 @@ export function paintStrokeInk(
}
bare.globalAlpha = 1;
}
+ if (inset) trimToSilhouette(ctx, silhouette, w, h);
} else {
// Contour styles.
const rings = contoursOf(silhouette, w, h);
diff --git a/site/public/mediapipe/magic_touch.tflite b/site/public/mediapipe/magic_touch.tflite
deleted file mode 100644
index 6abfeeea..00000000
Binary files a/site/public/mediapipe/magic_touch.tflite and /dev/null differ
diff --git a/site/src/cut/lib/chromaGpu.ts b/site/src/cut/lib/chromaGpu.ts
deleted file mode 100644
index d347b353..00000000
--- a/site/src/cut/lib/chromaGpu.ts
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * GPU chroma key for the preview and in-tab export: the clip's frame renders
- * through a WebGL2 pass that repeats the kit's reference math —
- * `chromaAlphaInto` in removal.ts — formula for formula, so the shader and
- * the CPU fallback key a frame to the same alpha. The canvas comes from the
- * raster seam; under a headless factory WebGL2 is absent and callers fall
- * back to the CPU pass.
- */
-
-import { CHROMA_SPILL_REACH, chromaParams, type ChromaKey } from "@donkeycut/effects-kit";
-import { gpuPassSingleton, uploadPassSource } from "./gpuPass";
-import type { RasterSurface } from "./raster";
-
-// BT.709 luma and the kit's Cb/Cr normalization; smoothstep matches the CPU
-// pass's cubic.
-const FRAG = `#version 300 es
-precision highp float;
-uniform sampler2D uSrc;
-uniform vec2 uKey; // key cb, cr
-uniform vec2 uDir; // unit key chroma direction
-uniform vec3 uRange; // near, far, spillFar
-uniform float uSpill;
-in vec2 vUv;
-out vec4 outColor;
-
-void main() {
- vec4 s = texture(uSrc, vUv);
- float y = dot(s.rgb, vec3(0.2126, 0.7152, 0.0722));
- float cb = (s.b - y) / 1.8556;
- float cr = (s.r - y) / 1.5748;
- float d = distance(vec2(cb, cr), uKey);
- float a = smoothstep(uRange.x, uRange.y, d);
- vec3 rgb = s.rgb;
- if (uSpill > 0.0 && d < uRange.z) {
- float proj = cb * uDir.x + cr * uDir.y;
- if (proj > 0.0) {
- float reach = 1.0 - clamp((d - uRange.x) / (uRange.z - uRange.x), 0.0, 1.0);
- float cut = proj * uSpill * (reach * reach * (3.0 - 2.0 * reach));
- float ncb = cb - uDir.x * cut;
- float ncr = cr - uDir.y * cut;
- float nr = y + 1.5748 * ncr;
- float nb = y + 1.8556 * ncb;
- float ng = (y - 0.2126 * nr - 0.0722 * nb) / 0.7152;
- rgb = clamp(vec3(nr, ng, nb), 0.0, 1.0);
- }
- }
- outColor = vec4(rgb, s.a * a);
-}`;
-
-const acquire = gpuPassSingleton(FRAG, ({ gl, program }) => {
- const keyLoc = gl.getUniformLocation(program, "uKey");
- const dirLoc = gl.getUniformLocation(program, "uDir");
- const rangeLoc = gl.getUniformLocation(program, "uRange");
- const spillLoc = gl.getUniformLocation(program, "uSpill");
- if (!keyLoc || !dirLoc || !rangeLoc || !spillLoc) return null;
- return { keyLoc, dirLoc, rangeLoc, spillLoc };
-});
-
-/**
- * Render `source` through the key and return a canvas holding the keyed
- * frame, or null when WebGL2 is unavailable (headless) — callers then use
- * the kit's CPU pass. The returned canvas is shared and valid until the
- * next call.
- */
-export function chromaKeyGpu(
- source: CanvasImageSource,
- w: number,
- h: number,
- key: ChromaKey
-): RasterSurface | null {
- const held = acquire(w, h);
- if (!held) return null;
- const { pass, ext } = held;
- const { gl } = pass;
- try {
- const p = chromaParams(key);
- uploadPassSource(pass, source);
- gl.uniform2f(ext.keyLoc, p.keyCb, p.keyCr);
- gl.uniform2f(ext.dirLoc, p.dirCb, p.dirCr);
- gl.uniform3f(ext.rangeLoc, p.near, p.far, p.near + (p.far - p.near) * CHROMA_SPILL_REACH);
- gl.uniform1f(ext.spillLoc, p.spill);
- gl.clearColor(0, 0, 0, 0);
- gl.clear(gl.COLOR_BUFFER_BIT);
- gl.drawArrays(gl.TRIANGLES, 0, 3);
- return pass.canvas;
- } catch {
- return null;
- }
-}
diff --git a/site/src/cut/lib/composite.ts b/site/src/cut/lib/composite.ts
index 68e42fff..cd03106d 100644
--- a/site/src/cut/lib/composite.ts
+++ b/site/src/cut/lib/composite.ts
@@ -19,8 +19,7 @@
* it was reached by playing there or by rendering the 135th frame.
*/
-import { applyLutToImageData, applyMaskToCanvas, buildGradeLut, chromaAlphaInto, gradeKey, gradeNeedsLut, gradeTint, gradeToCssFilter, grainTile, isNeutralGrade, lookCssFilter, lookPost, maskComposite, paintStrokeInk, removalActive, type GradeLut } from "@donkeycut/effects-kit";
-import { chromaKeyGpu } from "./chromaGpu";
+import { applyLutToImageData, applyMaskToCanvas, buildGradeLut, gradeKey, gradeNeedsLut, gradeTint, gradeToCssFilter, grainTile, isNeutralGrade, lookCssFilter, lookPost, maskComposite, paintStrokeInk, removalActive, type GradeLut } from "@donkeycut/effects-kit";
import { applyLutGpu } from "./gradeGpu";
import { createRasterCanvas } from "./raster";
import { clipCovers, clipPosed, clipPoseAt, clipZoom, contentRect, DEFAULT_BACKGROUND, isFullRect, rectOf, shadowInk } from "./types";
@@ -338,9 +337,9 @@ export class FrameCompositor {
}
/**
- * The clip's graded picture with its background removal applied: the key
- * multiplies an alpha into the frame (a chroma key computed here, or the
- * baked AI matte the host provides), the stroke's ink goes down behind the
+ * The clip's graded picture with its background removal applied: the baked
+ * AI matte the host provides multiplies an alpha into the frame, the
+ * stroke's ink goes down behind the
* silhouette, and the backdrop fills in behind everything. Runs in the
* clip's source pixel space, before the fit/fill draw, so the keyed layer
* rides zoom, pan, masks, poses and transitions like any other picture.
@@ -394,9 +393,8 @@ export class FrameCompositor {
// The matte answers before any scratch is staged: while the bake is still
// running the picture stays plain — the stroke and backdrop wait with it —
// and a full-frame clear and composite per tick would be pure waste.
- const matte =
- r.mode === "chroma" ? null : clip ? (this.removalMatteProvider?.(clip, at) ?? null) : null;
- if (r.mode !== "chroma" && !matte) return base;
+ const matte = clip ? (this.removalMatteProvider?.(clip, at) ?? null) : null;
+ if (!matte) return base;
const { surface } = this.scratch("removalScratch", w, h);
const ctx = surface.getContext("2d") as Ctx | null;
if (!ctx) return base;
@@ -405,28 +403,18 @@ export class FrameCompositor {
ctx.globalCompositeOperation = "source-over";
ctx.clearRect(0, 0, w, h);
- // 1. The key: alpha into the picture.
- if (r.mode === "chroma" && r.chroma) {
- const gpu = chromaKeyGpu(base, w, h, r.chroma);
- if (gpu) {
- ctx.drawImage(gpu as CanvasImageSource, 0, 0, w, h);
- } else {
- ctx.drawImage(base, 0, 0, w, h);
- const img = ctx.getImageData(0, 0, w, h);
- chromaAlphaInto(img.data, r.chroma);
- ctx.putImageData(img, 0, 0);
- }
- } else {
- ctx.drawImage(base, 0, 0, w, h);
- const { surface: cover } = this.scratch("removalMatte", w, h);
- const cctx = cover.getContext("2d") as Ctx | null;
- if (cctx) {
- cctx.setTransform(1, 0, 0, 1, 0, 0);
- cctx.clearRect(0, 0, w, h);
- cctx.imageSmoothingEnabled = true;
- cctx.drawImage(matte!, 0, 0, w, h);
- maskComposite(ctx, cover as CanvasImageSource, false);
- }
+ // 1. The key: alpha into the picture. `invert` flips the direction — the
+ // selection is removed and the rest of the picture stays.
+ const inv = !!r.invert;
+ ctx.drawImage(base, 0, 0, w, h);
+ const { surface: cover } = this.scratch("removalMatte", w, h);
+ const cctx = cover.getContext("2d") as Ctx | null;
+ if (cctx) {
+ cctx.setTransform(1, 0, 0, 1, 0, 0);
+ cctx.clearRect(0, 0, w, h);
+ cctx.imageSmoothingEnabled = true;
+ cctx.drawImage(matte, 0, 0, w, h);
+ maskComposite(ctx, cover as CanvasImageSource, inv);
}
const tLocal = Math.max(0, at - (clip?.start ?? 0));
@@ -435,7 +423,10 @@ export class FrameCompositor {
// size wherever the clip lands.
const ds = Math.min(w, h) / 1080;
- // 2. Stroke ink around the keyed silhouette, behind the picture.
+ // 2. Stroke ink around the selection's silhouette, behind the picture.
+ // The ink always traces the selection — inverted, that is the hole the
+ // removal cut, whose boundary is the same curve, and the ink paints
+ // inset so it rims the hole where it shows through.
if (r.stroke) {
const { surface: sil } = this.scratch("removalSil", w, h);
const sctx = sil.getContext("2d") as Ctx | null;
@@ -445,7 +436,11 @@ export class FrameCompositor {
sctx.setTransform(1, 0, 0, 1, 0, 0);
sctx.globalCompositeOperation = "source-over";
sctx.clearRect(0, 0, w, h);
- sctx.drawImage(surface, 0, 0);
+ if (!inv) {
+ sctx.drawImage(surface, 0, 0);
+ } else {
+ sctx.drawImage(cover as CanvasImageSource, 0, 0);
+ }
sctx.globalCompositeOperation = "source-in";
sctx.fillStyle = "#ffffff";
sctx.fillRect(0, 0, w, h);
@@ -453,7 +448,7 @@ export class FrameCompositor {
ictx.setTransform(1, 0, 0, 1, 0, 0);
ictx.globalCompositeOperation = "source-over";
ictx.clearRect(0, 0, w, h);
- paintStrokeInk(ictx, sil as CanvasImageSource, w, h, r.stroke, tLocal, ds);
+ paintStrokeInk(ictx, sil as CanvasImageSource, w, h, r.stroke, tLocal, ds, inv);
// Color the white ink, then lay it under the picture.
ictx.globalCompositeOperation = "source-in";
ictx.fillStyle = r.stroke.color;
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index b07e610c..a8870799 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -16,7 +16,7 @@ import { renderRemovalPieces } from "./removalVideo";
import { createRasterCanvas, rasterCanvasToPng } from "./raster";
import { clipSpeed, getClipSpans, overlayLayers, projectDuration, spanSequence, useEditor } from "./store";
import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
-import { chromaAlphaInto, isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, paintMaskLuma, paintStrokeInk } from "@donkeycut/effects-kit";
+import { isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, paintMaskLuma, paintStrokeInk } from "@donkeycut/effects-kit";
import { renderElementFrames, renderElementPng } from "./textRender";
import { clipCovers, clipKeyed, clipPosed, clipPoseAt, clipZoom, contentRect, frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, regionPx, removalActive, shadowInk, subjectMasked } from "./types";
import { liveReader } from "./liveReader";
@@ -319,9 +319,11 @@ function removalShadowSilhouette(
// a thick or offset stroke moves the shadow's edge with it.
const ink = createRasterCanvas(2, 2);
const inkCtx = ink.getContext("2d") as CanvasRenderingContext2D | null;
- const withStroke = (w: number, h: number, tLocal: number): CanvasImageSource => {
+ // The ink paints from the selection (inset when the selection is the hole),
+ // before any complement, so it rims what the stroke really shows.
+ const paintInk = (w: number, h: number, tLocal: number): boolean => {
const stroke = r!.stroke;
- if (!stroke || !inkCtx) return canvas as CanvasImageSource;
+ if (!stroke || !inkCtx) return false;
if (ink.width !== w || ink.height !== h) {
ink.width = w;
ink.height = h;
@@ -329,39 +331,18 @@ function removalShadowSilhouette(
inkCtx.setTransform(1, 0, 0, 1, 0, 0);
inkCtx.globalCompositeOperation = "source-over";
inkCtx.clearRect(0, 0, w, h);
- paintStrokeInk(inkCtx, canvas as CanvasImageSource, w, h, stroke, tLocal, Math.min(w, h) / 1080);
- ctx.globalCompositeOperation = "destination-over";
- ctx.drawImage(ink, 0, 0);
- ctx.globalCompositeOperation = "source-over";
- return canvas as CanvasImageSource;
+ paintStrokeInk(
+ inkCtx,
+ canvas as CanvasImageSource,
+ w,
+ h,
+ stroke,
+ tLocal,
+ Math.min(w, h) / 1080,
+ !!r!.invert
+ );
+ return true;
};
- if (r!.mode === "chroma") {
- const key = r!.chroma;
- if (!key) return null;
- const reader = liveReader(asset);
- return {
- at: async (tLocal) => {
- const frame = await reader.frameAt(still ? 0 : clip.in + tLocal * speed);
- if (frame.kind !== "ready") return null;
- // Sampled small — the silhouette blurs into a shadow anyway.
- const s = Math.min(1, 480 / Math.max(1, Math.min(frame.width, frame.height)));
- const w = Math.max(2, Math.round(frame.width * s));
- const h = Math.max(2, Math.round(frame.height * s));
- stage(w, h);
- ctx.drawImage(frame.image, 0, 0, w, h);
- const px = ctx.getImageData(0, 0, w, h);
- chromaAlphaInto(px.data, key);
- for (let i = 0; i < px.data.length; i += 4) {
- px.data[i] = 255;
- px.data[i + 1] = 255;
- px.data[i + 2] = 255;
- }
- ctx.putImageData(px, 0, 0);
- return withStroke(w, h, tLocal);
- },
- dispose: () => reader.dispose(),
- };
- }
const matte = r!.matte ? assets.find((a) => a.id === r!.matte!.assetId) : undefined;
if (!matte) return null;
const reader = liveReader(matte);
@@ -378,7 +359,19 @@ function removalShadowSilhouette(
const px = ctx.getImageData(0, 0, frame.width, frame.height);
matteLumaToAlpha(px.data);
ctx.putImageData(px, 0, 0);
- return withStroke(frame.width, frame.height, tLocal);
+ const inked = paintInk(frame.width, frame.height, tLocal);
+ // Inverted, the clip shows everything around the matte, so the shadow's
+ // silhouette is the complement.
+ if (r!.invert) {
+ for (let i = 3; i < px.data.length; i += 4) px.data[i] = 255 - px.data[i];
+ ctx.putImageData(px, 0, 0);
+ }
+ if (inked) {
+ ctx.globalCompositeOperation = "destination-over";
+ ctx.drawImage(ink, 0, 0);
+ ctx.globalCompositeOperation = "source-over";
+ }
+ return canvas as CanvasImageSource;
},
dispose: () => reader.dispose(),
};
@@ -406,7 +399,7 @@ async function renderClipShadowPictures(
const sh = clip.boxStyle?.shadow;
if (!sh) return undefined;
// A cutout's shadow falls from the keyed subject, sampled per frame from
- // the matte (or the chroma key), the way the preview casts it.
+ // the matte, the way the preview casts it.
const silhouette = removalShadowSilhouette(clip, assets);
const scale = Math.min(W, H) / 1080;
const frame = { width: W, height: H, scale };
diff --git a/site/src/cut/lib/removal/interactive.ts b/site/src/cut/lib/removal/interactive.ts
deleted file mode 100644
index 9aa7ee26..00000000
--- a/site/src/cut/lib/removal/interactive.ts
+++ /dev/null
@@ -1,328 +0,0 @@
-"use client";
-
-/**
- * The custom-removal selection session: what the brush has picked on the
- * current frame, held as a working mask the stage draws as a red overlay.
- * Quick strokes run the on-device tap-to-select segmenter for instant
- * feedback; manual strokes paint the mask directly. Every finished stroke
- * also yields its seed record — the point prompts and paint bitmaps the doc
- * stores, which the hosted tracker replays across the whole clip.
- */
-
-import { matteLumaToAlpha } from "@donkeycut/effects-kit";
-import { interactiveSegmenter, segmentTouchAlpha } from "../cutout";
-import { sampleClipSource } from "../previewCanvas";
-import { createRasterCanvas, rasterCanvasToDataUrl, type RasterSurface } from "../raster";
-import type { WandScratch } from "./touchMask";
-import type { RemovalSeeds } from "../types";
-
-/** Working-mask short side: enough for a crisp overlay, cheap to re-segment
- * on every pointer move. */
-const WORK_SHORT = 512;
-/** Stored paint bitmaps stay small — they seed a tracker, never a render. */
-const PAINT_SHORT = 256;
-
-type Point = { x: number; y: number };
-
-export class QuickSelectSession {
- /** The current frame, drawn at working size. */
- private work: RasterSurface;
- /** White where the selection is. */
- private mask: RasterSurface;
- /** The stroke in flight, previewed over (or out of) the mask until the
- * pointer lifts. */
- private tentative: RasterSurface;
- /** Scratch for compositing mask + tentative into the red overlay tint. */
- private view: RasterSurface;
- private tentativeErase = false;
- private hasTentative = false;
- private w = 2;
- private h = 2;
- private seg: Awaited<ReturnType<typeof interactiveSegmenter>> = null;
- /** The working frame's pixels and the wand's tone field, read once per
- * frame and shared across a stroke's per-move re-segmentations. */
- private frameRgba: Uint8ClampedArray | null = null;
- private wandScratch: WandScratch = {};
-
- private constructor() {
- this.work = createRasterCanvas(2, 2);
- this.mask = createRasterCanvas(2, 2);
- this.tentative = createRasterCanvas(2, 2);
- this.view = createRasterCanvas(2, 2);
- }
-
- /** Null when the tap-to-select model is unavailable — the manual brush and
- * erase still work through the returned session in that case. */
- static async open(): Promise<QuickSelectSession> {
- const s = new QuickSelectSession();
- s.seg = await interactiveSegmenter();
- return s;
- }
-
- get quickAvailable(): boolean {
- return !!this.seg;
- }
-
- /** Pull the clip's current decoder frame into the working canvas. False
- * when no frame is ready yet. */
- refreshFrame(clipId: string): boolean {
- const src = sampleClipSource(clipId);
- if (!src) return false;
- const sw = Number((src as { width?: number }).width ?? 0) || 16;
- const sh = Number((src as { height?: number }).height ?? 0) || 9;
- const scale = WORK_SHORT / Math.max(1, Math.min(sw, sh));
- const w = Math.max(2, Math.round(sw * scale));
- const h = Math.max(2, Math.round(sh * scale));
- if (this.w !== w || this.h !== h) {
- this.w = w;
- this.h = h;
- this.work.width = w;
- this.work.height = h;
- const m = this.mask.getContext("2d") as CanvasRenderingContext2D | null;
- const keep = m && (this.mask.width > 2 ? this.mask : null);
- const prev = keep ? createRasterCanvas(this.mask.width, this.mask.height) : null;
- if (prev && keep) {
- (prev.getContext("2d") as CanvasRenderingContext2D).drawImage(keep as CanvasImageSource, 0, 0);
- }
- this.mask.width = w;
- this.mask.height = h;
- if (prev && m) m.drawImage(prev as CanvasImageSource, 0, 0, w, h);
- this.tentative.width = w;
- this.tentative.height = h;
- this.hasTentative = false;
- }
- const ctx = this.work.getContext("2d") as CanvasRenderingContext2D | null;
- if (!ctx) return false;
- ctx.drawImage(src, 0, 0, w, h);
- this.frameRgba = null;
- this.wandScratch = {};
- return true;
- }
-
- /** The working frame's pixels, read once per refreshed frame. */
- private rgbaOf(): Uint8ClampedArray | undefined {
- if (!this.frameRgba) {
- const ctx = this.work.getContext("2d") as CanvasRenderingContext2D | null;
- if (!ctx) return undefined;
- this.frameRgba = ctx.getImageData(0, 0, this.w, this.h).data;
- }
- return this.frameRgba;
- }
-
- /** Run tap-to-select over the stroke and fold the result into the mask.
- * `erase` subtracts the picked object. True when something registered. */
- quickStroke(points: Point[], erase: boolean): boolean {
- if (!this.seg || points.length === 0) return false;
- const alpha = segmentTouchAlpha(this.seg, this.work as HTMLCanvasElement, points, {
- rgba: this.rgbaOf(),
- scratch: this.wandScratch,
- });
- if (!alpha) return false;
- const ctx = this.mask.getContext("2d") as CanvasRenderingContext2D | null;
- if (!ctx) return false;
- ctx.globalCompositeOperation = erase ? "destination-out" : "source-over";
- ctx.imageSmoothingEnabled = true;
- ctx.drawImage(alpha, 0, 0, this.w, this.h);
- ctx.globalCompositeOperation = "source-over";
- return true;
- }
-
- /** Preview a quick stroke while the pointer is still down: the picked
- * object shows over (or carves out of) the mask, folded in only on
- * `commitTentative`. */
- quickPreview(points: Point[], erase: boolean): boolean {
- if (!this.seg || points.length === 0) return false;
- const alpha = segmentTouchAlpha(this.seg, this.work as HTMLCanvasElement, points, {
- rgba: this.rgbaOf(),
- scratch: this.wandScratch,
- });
- const ctx = this.tentative.getContext("2d") as CanvasRenderingContext2D | null;
- if (!ctx) return false;
- ctx.clearRect(0, 0, this.w, this.h);
- if (!alpha) {
- this.hasTentative = false;
- return false;
- }
- ctx.imageSmoothingEnabled = true;
- ctx.drawImage(alpha, 0, 0, this.w, this.h);
- this.tentativeErase = erase;
- this.hasTentative = true;
- return true;
- }
-
- /** Fold the previewed stroke into the mask. */
- commitTentative(): void {
- if (!this.hasTentative) return;
- const ctx = this.mask.getContext("2d") as CanvasRenderingContext2D | null;
- if (ctx) {
- ctx.globalCompositeOperation = this.tentativeErase ? "destination-out" : "source-over";
- ctx.drawImage(this.tentative as CanvasImageSource, 0, 0);
- ctx.globalCompositeOperation = "source-over";
- }
- this.dropTentative();
- }
-
- dropTentative(): void {
- this.hasTentative = false;
- const ctx = this.tentative.getContext("2d") as CanvasRenderingContext2D | null;
- ctx?.clearRect(0, 0, this.w, this.h);
- }
-
- /** Paint one manual stroke into the mask. `radius` is a fraction of the
- * frame's short side. */
- paintStroke(points: Point[], radius: number, erase: boolean): void {
- const ctx = this.mask.getContext("2d") as CanvasRenderingContext2D | null;
- if (!ctx || points.length === 0) return;
- const r = Math.max(1, radius * Math.min(this.w, this.h));
- ctx.globalCompositeOperation = erase ? "destination-out" : "source-over";
- ctx.strokeStyle = "#ffffff";
- ctx.fillStyle = "#ffffff";
- ctx.lineWidth = r * 2;
- ctx.lineCap = "round";
- ctx.lineJoin = "round";
- ctx.beginPath();
- ctx.moveTo(points[0].x * this.w, points[0].y * this.h);
- for (let i = 1; i < points.length; i++) ctx.lineTo(points[i].x * this.w, points[i].y * this.h);
- if (points.length === 1) {
- ctx.arc(points[0].x * this.w, points[0].y * this.h, r, 0, Math.PI * 2);
- ctx.fill();
- } else {
- ctx.stroke();
- }
- ctx.globalCompositeOperation = "source-over";
- }
-
- /** The stroke's stored paint bitmap: the stroke alone, small grayscale PNG
- * (white marks) for the seed record. */
- async paintSeed(points: Point[], radius: number): Promise<string> {
- const scale = PAINT_SHORT / Math.max(1, Math.min(this.w, this.h));
- const w = Math.max(2, Math.round(this.w * scale));
- const h = Math.max(2, Math.round(this.h * scale));
- const c = createRasterCanvas(w, h);
- const ctx = c.getContext("2d") as CanvasRenderingContext2D;
- ctx.fillStyle = "#000000";
- ctx.fillRect(0, 0, w, h);
- const r = Math.max(1, radius * Math.min(w, h));
- ctx.strokeStyle = "#ffffff";
- ctx.fillStyle = "#ffffff";
- ctx.lineWidth = r * 2;
- ctx.lineCap = "round";
- ctx.lineJoin = "round";
- ctx.beginPath();
- ctx.moveTo(points[0].x * w, points[0].y * h);
- for (let i = 1; i < points.length; i++) ctx.lineTo(points[i].x * w, points[i].y * h);
- if (points.length === 1) {
- ctx.arc(points[0].x * w, points[0].y * h, r, 0, Math.PI * 2);
- ctx.fill();
- } else {
- ctx.stroke();
- }
- return rasterCanvasToDataUrl(c, "image/png");
- }
-
- /** Rebuild the mask from the seeds recorded on the current frame: prompts
- * re-segment, paint bitmaps replay over them. `at` is the frame's source
- * second; a stroke recorded on another frame stays off this overlay — the
- * segmenter would pick whatever sits under its points now, a different
- * object than the stroke selected. */
- replaySeeds(
- seeds: RemovalSeeds | undefined,
- decode: (url: string) => Promise<CanvasImageSource | null>,
- at: number
- ): Promise<void> {
- this.reset();
- if (!seeds) return Promise.resolve();
- const here = (t: number) => Math.abs(t - at) < 0.05;
- for (const p of seeds.prompts) {
- if (!here(p.t)) continue;
- const keeps = p.points.filter((pt) => pt.label === 1);
- const drops = p.points.filter((pt) => pt.label === 0);
- if (keeps.length) this.quickStroke(keeps, false);
- if (drops.length) this.quickStroke(drops, true);
- }
- const paints = (seeds.paint ?? []).filter((p) => here(p.t));
- return (async () => {
- const ctx = this.mask.getContext("2d") as CanvasRenderingContext2D | null;
- if (!ctx) return;
- for (const p of paints) {
- for (const [url, erase] of [
- [p.add, false],
- [p.erase, true],
- ] as const) {
- if (!url) continue;
- const img = await decode(url);
- if (!img) continue;
- ctx.globalCompositeOperation = erase ? "destination-out" : "source-over";
- // The paint is white on black; its luma is the stroke, so cut the
- // black away first via a lightened draw onto a scratch.
- const s = createRasterCanvas(this.w, this.h);
- const sctx = s.getContext("2d") as CanvasRenderingContext2D;
- sctx.drawImage(img, 0, 0, this.w, this.h);
- const px = sctx.getImageData(0, 0, this.w, this.h);
- matteLumaToAlpha(px.data);
- sctx.putImageData(px, 0, 0);
- ctx.drawImage(s as CanvasImageSource, 0, 0);
- }
- }
- ctx.globalCompositeOperation = "source-over";
- })();
- }
-
- reset(): void {
- const ctx = this.mask.getContext("2d") as CanvasRenderingContext2D | null;
- ctx?.clearRect(0, 0, this.mask.width, this.mask.height);
- }
-
- /** Whether anything is selected right now. */
- hasSelection(): boolean {
- const ctx = this.mask.getContext("2d") as CanvasRenderingContext2D | null;
- if (!ctx) return false;
- const px = ctx.getImageData(0, 0, this.mask.width, this.mask.height).data;
- for (let i = 3; i < px.length; i += 64) if (px[i] > 32) return true;
- return false;
- }
-
- /** Paint the selection (committed mask plus the stroke in flight) into a
- * display context as the red overlay. */
- paintOverlay(ctx: CanvasRenderingContext2D, w: number, h: number): void {
- if (this.view.width !== this.w || this.view.height !== this.h) {
- this.view.width = this.w;
- this.view.height = this.h;
- }
- const view = this.view;
- const vctx = view.getContext("2d") as CanvasRenderingContext2D;
- vctx.globalCompositeOperation = "source-over";
- vctx.clearRect(0, 0, this.w, this.h);
- vctx.drawImage(this.mask as CanvasImageSource, 0, 0);
- if (this.hasTentative) {
- vctx.globalCompositeOperation = this.tentativeErase ? "destination-out" : "source-over";
- vctx.drawImage(this.tentative as CanvasImageSource, 0, 0);
- vctx.globalCompositeOperation = "source-over";
- }
- vctx.globalCompositeOperation = "source-in";
- vctx.fillStyle = "#ff2244";
- vctx.fillRect(0, 0, this.w, this.h);
- ctx.clearRect(0, 0, w, h);
- ctx.globalAlpha = 0.5;
- ctx.imageSmoothingEnabled = true;
- ctx.drawImage(view as CanvasImageSource, 0, 0, w, h);
- ctx.globalAlpha = 1;
- }
-
- /** The selection mask (white, alpha-carrying) for the stage's red overlay. */
- overlay(): CanvasImageSource {
- return this.mask as CanvasImageSource;
- }
-
- /** The current frame at working size, for the loupe. */
- frame(): CanvasImageSource {
- return this.work as CanvasImageSource;
- }
-
- get width() {
- return this.w;
- }
- get height() {
- return this.h;
- }
-}
diff --git a/site/src/cut/lib/removal/keyColor.ts b/site/src/cut/lib/removal/keyColor.ts
deleted file mode 100644
index cc265e79..00000000
--- a/site/src/cut/lib/removal/keyColor.ts
+++ /dev/null
@@ -1,64 +0,0 @@
-"use client";
-
-import { sampleClipFrameData } from "../previewCanvas";
-import { createRasterCanvas } from "../raster";
-import type { MediaAsset, VideoClip } from "../types";
-import { liveReader } from "../liveReader";
-
-const SAMPLE_W = 96;
-const SAMPLE_H = 54;
-
-/** The median of a small frame readout's border pixels, where a backdrop
- * lives. */
-function medianBorder(data: Uint8ClampedArray, w: number): string {
- const h = Math.round(data.length / 4 / w);
- const chans: number[][] = [[], [], []];
- const take = (x: number, y: number) => {
- const i = (y * w + x) * 4;
- for (let c = 0; c < 3; c++) chans[c].push(data[i + c]);
- };
- for (let x = 0; x < w; x += 2) {
- take(x, 0);
- take(x, h - 1);
- }
- for (let y = 1; y < h - 1; y += 2) {
- take(0, y);
- take(w - 1, y);
- }
- const hex = chans
- .map((v) => {
- v.sort((a, b) => a - b);
- return v[Math.floor(v.length / 2)].toString(16).padStart(2, "0");
- })
- .join("");
- return `#${hex}`;
-}
-
-/** The key color the clip's live preview frame suggests; null when no
- * decoder currently shows the clip (off the playhead, headless). */
-export function suggestKeyColor(clipId: string): string | null {
- const data = sampleClipFrameData(clipId);
- return data ? medianBorder(data, SAMPLE_W) : null;
-}
-
-/** The key color read from the clip's own footage, decoded at its in-point —
- * for callers with no live decoder on the clip (the chat tool, a clip away
- * from the playhead). Null when the picture can't be read. */
-export async function clipKeyColor(asset: MediaAsset, clip: VideoClip): Promise<string | null> {
- const reader = liveReader(asset);
- try {
- const frame = await reader.frameAt(asset.type === "image" ? 0 : Math.max(0, clip.in));
- if (frame.kind !== "ready") return null;
- const canvas = createRasterCanvas(SAMPLE_W, SAMPLE_H);
- const ctx = canvas.getContext("2d", {
- willReadFrequently: true,
- }) as CanvasRenderingContext2D | null;
- if (!ctx) return null;
- ctx.drawImage(frame.image, 0, 0, SAMPLE_W, SAMPLE_H);
- return medianBorder(ctx.getImageData(0, 0, SAMPLE_W, SAMPLE_H).data, SAMPLE_W);
- } catch {
- return null;
- } finally {
- reader.dispose();
- }
-}
diff --git a/site/src/cut/lib/removal/touchMask.ts b/site/src/cut/lib/removal/touchMask.ts
deleted file mode 100644
index 55047f52..00000000
--- a/site/src/cut/lib/removal/touchMask.ts
+++ /dev/null
@@ -1,212 +0,0 @@
-/**
- * Smart select's mask math, two tiers. The tap-to-select model handles what
- * it is built for — objects: its confidence mask is gated by the confidence
- * at the stroke itself and by connectivity to it, so the object-centric
- * model's bleed into the salient subject stays out of a pick. Where the
- * model refuses to anchor — backdrops, walls, sky, anything it will not call
- * an object — the pick falls to a geodesic color wand: distance from the
- * stroke accumulates color change along the cheapest path, so a smooth
- * region costs its total drift while a real boundary is a cost cliff, and
- * the selection snaps there. The wand is the interactive-rate shape of the
- * classic scribble segmentation energy (color model + contrast-aware cut).
- */
-
-/** Floor confidence for a pixel to count as picked at all. */
-export const TOUCH_FLOOR = 0.35;
-/** How far below the stroke's own confidence a pixel may sit and still join
- * the pick. */
-export const TOUCH_MARGIN = 0.3;
-
-type Point = { x: number; y: number };
-
-/** Index of the mask pixel under a frame-fraction point. */
-const at = (p: Point, mw: number, mh: number) =>
- Math.min(mh - 1, Math.max(0, Math.round(p.y * mh))) * mw +
- Math.min(mw - 1, Math.max(0, Math.round(p.x * mw)));
-
-const median = (v: number[]) => {
- const s = [...v].sort((a, b) => a - b);
- return s[s.length >> 1];
-};
-
-/**
- * The picked region's alpha bytes (0..255, one per mask pixel), or null when
- * the model puts no confidence on the touched spot itself — the stroke landed
- * on something the model refuses to call an object, and the caller falls
- * through to the color wand. Points are frame fractions.
- *
- * The stroke's first point alone decides which tier answers: it never moves
- * while a drag grows, so the preview stays with one tier for the whole
- * stroke. The median over the stroke then sets the confidence floor, which
- * keeps one stray sample grazing model bleed from widening the pick.
- */
-export function touchAlpha(
- conf: Float32Array,
- mw: number,
- mh: number,
- points: Point[]
-): Uint8ClampedArray | null {
- if (points.length === 0) return null;
- if (conf[at(points[0], mw, mh)] <= TOUCH_FLOOR) return null;
- const seedConf = Math.max(TOUCH_FLOOR, median(points.map((p) => conf[at(p, mw, mh)])));
- const floor = Math.max(TOUCH_FLOOR, seedConf - TOUCH_MARGIN);
- const alpha = new Uint8ClampedArray(mw * mh);
- let kept = 0;
- const ramp = (c: number) => Math.min(255, Math.round((c - floor) * 4 * 255));
- // Flood from the stroke over above-floor pixels; 4-connected.
- const seen = new Uint8Array(mw * mh);
- const queue: number[] = [];
- const visit = (i: number) => {
- if (!seen[i] && conf[i] > floor) {
- seen[i] = 1;
- queue.push(i);
- }
- };
- for (const p of points) visit(at(p, mw, mh));
- while (queue.length) {
- const i = queue.pop()!;
- alpha[i] = ramp(conf[i]);
- if (alpha[i] > 128) kept++;
- const x = i % mw;
- if (x > 0) visit(i - 1);
- if (x < mw - 1) visit(i + 1);
- if (i >= mw) visit(i - mw);
- if (i < mw * (mh - 1)) visit(i + mw);
- }
- return kept > 0 ? alpha : null;
-}
-
-/** Color distance (L1 mean, 0..255) within which a pixel counts as one of
- * the stroke's own tones. */
-export const WAND_TONE = 26;
-/** How far past the tones a pixel may sit and still be selectable when the
- * pick flows through it — soft shadows, grain, antialiased slivers. */
-const WAND_NEAR = 22;
-/** Off-tone cost budget a path may spend bridging thin foreign runs — a
- * groove, a seam, a highlight line — before the pick stops. */
-const WAND_BARRIER = 64;
-/** Chamfer sweeps; two settle straight paths, the third the wiggly ones. */
-const WAND_PASSES = 3;
-
-/** The stroke's color signatures: its sampled colors, greedily merged so
- * near-identical samples collapse into one representative. */
-function signatures(rgba: Uint8ClampedArray, seeds: number[]): number[][] {
- const sig: number[][] = [];
- for (const i of seeds) {
- const c = [rgba[i * 4], rgba[i * 4 + 1], rgba[i * 4 + 2]];
- const near = sig.some(
- (s) => Math.abs(s[0] - c[0]) + Math.abs(s[1] - c[1]) + Math.abs(s[2] - c[2]) < 36
- );
- if (!near) sig.push(c);
- if (sig.length >= 16) break;
- }
- return sig;
-}
-
-/** Per-frame scratch a caller may hand `wandAlpha`: the tone field is the
- * wand's heavy pass and depends only on the picture and the stroke's
- * signature set, so across the pointer moves of one stroke it is usually
- * reusable as-is. */
-export interface WandScratch {
- key?: string;
- tone?: Float32Array;
-}
-
-/**
- * The color wand, shaped like SIOX's foreground select: the stroke's colors
- * cluster into signatures, every pixel scores by its distance to the nearest
- * signature, and the pick grows from the stroke over on-signature pixels.
- * Growth carries a small budget for off-signature cost, so thin foreign runs
- * — grooves, seams, texture lines — are bridged and their far side joins,
- * while a real subject boundary is wide and costly enough to stop at.
- * Scribbling over more tones adds signatures and widens the pick with it.
- */
-export function wandAlpha(
- rgba: Uint8ClampedArray,
- w: number,
- h: number,
- points: Point[],
- scratch?: WandScratch
-): Uint8ClampedArray | null {
- if (points.length === 0) return null;
- const seeds = points.map((p) => at(p, w, h));
- const sig = signatures(rgba, seeds);
- // Distance from each pixel's color to the nearest signature — cached in
- // the scratch while the signature set holds.
- const key = `${w}x${h}|${sig.map((s) => s.join(",")).join(";")}`;
- let tone = scratch?.key === key ? scratch.tone : undefined;
- if (!tone) {
- tone = new Float32Array(w * h);
- for (let i = 0; i < w * h; i++) {
- const r = rgba[i * 4];
- const g = rgba[i * 4 + 1];
- const b = rgba[i * 4 + 2];
- let best = Infinity;
- for (const s of sig) {
- const d = (Math.abs(s[0] - r) + Math.abs(s[1] - g) + Math.abs(s[2] - b)) / 3;
- if (d < best) best = d;
- }
- tone[i] = best;
- }
- if (scratch) {
- scratch.key = key;
- scratch.tone = tone;
- }
- }
- // Geodesic growth from the stroke: entering an on-signature pixel is free,
- // an off-signature pixel charges its excess, and the accumulated charge
- // along the cheapest 8-connected path is capped by the bridge budget.
- const cost = (i: number) => Math.max(0, tone[i] - WAND_TONE);
- const dist = new Float32Array(w * h).fill(Infinity);
- for (const i of seeds) dist[i] = 0;
- for (let pass = 0; pass < WAND_PASSES; pass++) {
- let changed = false;
- for (let y = 0; y < h; y++) {
- for (let x = 0; x < w; x++) {
- const i = y * w + x;
- let d = dist[i];
- const c = cost(i);
- if (x > 0) d = Math.min(d, dist[i - 1] + c);
- if (y > 0) {
- d = Math.min(d, dist[i - w] + c);
- if (x > 0) d = Math.min(d, dist[i - w - 1] + c);
- if (x < w - 1) d = Math.min(d, dist[i - w + 1] + c);
- }
- if (d < dist[i]) {
- dist[i] = d;
- changed = true;
- }
- }
- }
- for (let y = h - 1; y >= 0; y--) {
- for (let x = w - 1; x >= 0; x--) {
- const i = y * w + x;
- let d = dist[i];
- const c = cost(i);
- if (x < w - 1) d = Math.min(d, dist[i + 1] + c);
- if (y < h - 1) {
- d = Math.min(d, dist[i + w] + c);
- if (x < w - 1) d = Math.min(d, dist[i + w + 1] + c);
- if (x > 0) d = Math.min(d, dist[i + w - 1] + c);
- }
- if (d < dist[i]) {
- dist[i] = d;
- changed = true;
- }
- }
- }
- // Straight paths settle in one pass, wiggly ones in two or three; once a
- // pass moves nothing the field is final.
- if (!changed) break;
- }
- const alpha = new Uint8ClampedArray(w * h);
- const core = WAND_BARRIER * 0.6;
- for (let i = 0; i < w * h; i++) {
- if (tone[i] > WAND_TONE + WAND_NEAR) continue;
- const d = dist[i];
- if (d <= core) alpha[i] = 255;
- else if (d < WAND_BARRIER)
- alpha[i] = Math.round(255 * (1 - (d - core) / (WAND_BARRIER - core)));
- }
- return alpha;
-}
diff --git a/site/src/cut/lib/removalVideo.ts b/site/src/cut/lib/removalVideo.ts
index eda2fd54..3ad32ce6 100644
--- a/site/src/cut/lib/removalVideo.ts
+++ b/site/src/cut/lib/removalVideo.ts
@@ -10,7 +10,7 @@
* channel, which is why the layer travels as a pair.
*/
-import { matteLumaToAlpha } from "@donkeycut/effects-kit";
+import { matteLumaToAlpha, removalActive } from "@donkeycut/effects-kit";
import { openCanvasVideo, scaledEvenSize } from "./canvasVideo";
import { FrameCompositor, type Frame } from "./composite";
import { createRasterCanvas, decodeRasterImageUrl, type RasterSurface } from "./raster";
@@ -68,10 +68,9 @@ export async function renderRemovalPieces(
opts: { fps: number; maxShort: number; bakeLook: boolean }
): Promise<RemovalPieces | null> {
const r = clip.removal;
- if (!r) return null;
+ if (!removalActive(r) || !r) return null;
const matteAsset = r.matte ? assets.find((a) => a.id === r.matte!.assetId) : undefined;
- const ai = r.mode === "auto" || r.mode === "custom";
- if (ai && !matteAsset) return null;
+ if (!matteAsset) return null;
const speed = clipSpeed(clip);
const still = asset.type === "image";
diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.ts
index 5b9d03ca..54545aef 100644
--- a/site/src/cut/lib/types.ts
+++ b/site/src/cut/lib/types.ts
@@ -555,10 +555,10 @@ export interface VideoClip {
kf?: OverlayKey[];
/** Rounded corners and a border stroke on the clip's box; absent = plain. */
boxStyle?: BoxStyle;
- /** Background removal: keys the clip's picture to an alpha matte (chroma
- * key, or an AI matte baked to a grayscale video asset), with an optional
- * stroke around the silhouette and a backdrop filled in behind it (see the
- * kit's removal.ts). Absent = the whole picture shows. */
+ /** Background removal: keys the clip's picture to an AI matte baked to a
+ * grayscale video asset, with an optional stroke around the silhouette and
+ * a backdrop filled in behind it (see the kit's removal.ts). Absent = the
+ * whole picture shows. */
removal?: ClipRemoval;
}
@@ -577,7 +577,6 @@ export {
STROKE_STYLES,
} from "@donkeycut/effects-kit";
export type {
- ChromaKey,
ClipRemoval,
RemovalBackdrop,
RemovalMode,