Skip to content

Open source CapCut alternative.

TypeScript

feat(site): mask and keyframe model in the effects kit [rebuild]

David Le937cdf5966d6
diff --git a/site/packages/effects-kit/src/index.ts b/site/packages/effects-kit/src/index.ts
index 642c0b61..6174655e 100644
--- a/site/packages/effects-kit/src/index.ts
+++ b/site/packages/effects-kit/src/index.ts
@@ -1,6 +1,7 @@
export * from "./types";
export * from "./anim";
export * from "./keys";
+export * from "./mask";
export * from "./render";
export * from "./registry";
export * from "./cutout";
diff --git a/site/packages/effects-kit/src/keys.ts b/site/packages/effects-kit/src/keys.ts
index c914177a..ccacdb64 100644
--- a/site/packages/effects-kit/src/keys.ts
+++ b/site/packages/effects-kit/src/keys.ts
@@ -38,7 +38,9 @@ export interface OverlayPose {
opacity: number;
}
-/** The fields `evalOverlayFrame` reads — every overlay kind has them. */
+/** The fields `evalOverlayFrame` reads — every overlay kind has them. The
+ * mask is read structurally (keyed or not), keeping this module free of the
+ * mask model. */
interface Posable {
start: number;
end: number;
@@ -48,6 +50,7 @@ interface Posable {
opacity?: number;
anim?: OverlayAnim;
kf?: OverlayKey[];
+ mask?: { kf?: { t: number }[] };
}
/** Two keys closer than this are the same key. Half a frame at 30fps, so a
@@ -70,46 +73,65 @@ export function hasOverlayKeys(o: Posable): boolean {
return !!o.kf && o.kf.length > 0;
}
-/** Whether anything about the element moves — a preset, keys, or both. An
- * element that does not move renders as one still picture. */
+/** Whether anything about the element moves — a preset, pose keys, or a
+ * keyframed mask. An element that does not move renders as one still
+ * picture. */
export function isOverlayAnimated(o: Posable): boolean {
- return hasOverlayAnim(o.anim) || hasOverlayKeys(o);
+ return hasOverlayAnim(o.anim) || hasOverlayKeys(o) || !!(o.mask?.kf && o.mask.kf.length > 0);
}
/** Keys in play order. Callers may hand them over unsorted (a key added at the
* playhead lands wherever the user was). */
-export function sortedKeys(keys: OverlayKey[]): OverlayKey[] {
+export function sortedKeys<K extends { t: number }>(keys: K[]): K[] {
return [...keys].sort((a, b) => a.t - b.t);
}
-/** The pose at `tLocal` seconds into the element: interpolated between the
- * surrounding keys, held flat outside them, resting when there are no keys. */
-export function poseAt(o: Posable, tLocal: number): OverlayPose {
- const keys = o.kf;
- if (!keys || keys.length === 0) return restingPose(o);
+/**
+ * The interpolation core every key track shares: find the surrounding keys,
+ * hold flat outside them, and hand the pair to `mix`. The pose track and the
+ * mask track are both thin wrappers over this.
+ */
+export function lerpKeys<K extends { t: number }>(
+ keys: K[],
+ tLocal: number,
+ mix: (a: K, b: K, p: number) => K
+): K {
const ks = sortedKeys(keys);
- if (tLocal <= ks[0].t) return poseOf(ks[0]);
+ if (tLocal <= ks[0].t) return ks[0];
const last = ks[ks.length - 1];
- if (tLocal >= last.t) return poseOf(last);
+ if (tLocal >= last.t) return last;
let i = 0;
while (i < ks.length - 1 && ks[i + 1].t <= tLocal) i++;
const a = ks[i];
const b = ks[i + 1];
const span = b.t - a.t;
const p = span > 1e-6 ? (tLocal - a.t) / span : 0;
- const mix = (u: number, v: number) => u + (v - u) * p;
- return {
- x: mix(a.x, b.x),
- y: mix(a.y, b.y),
- scale: mix(a.scale, b.scale),
- // Rotation takes the short way around, so a key at 350° into one at 10°
- // turns 20° forward instead of 340° back.
- rotation: a.rotation + shortestTurn(a.rotation, b.rotation) * p,
- opacity: mix(a.opacity, b.opacity),
- };
+ return mix(a, b, p);
}
-function shortestTurn(from: number, to: number): number {
+/** The pose at `tLocal` seconds into the element: interpolated between the
+ * surrounding keys, held flat outside them, resting when there are no keys. */
+export function poseAt(o: Posable, tLocal: number): OverlayPose {
+ const keys = o.kf;
+ if (!keys || keys.length === 0) return restingPose(o);
+ const k = lerpKeys(keys, tLocal, (a, b, p) => {
+ const mix = (u: number, v: number) => u + (v - u) * p;
+ return {
+ t: mix(a.t, b.t),
+ x: mix(a.x, b.x),
+ y: mix(a.y, b.y),
+ scale: mix(a.scale, b.scale),
+ // Rotation takes the short way around, so a key at 350° into one at 10°
+ // turns 20° forward instead of 340° back.
+ rotation: a.rotation + shortestTurn(a.rotation, b.rotation) * p,
+ opacity: mix(a.opacity, b.opacity),
+ };
+ });
+ return poseOf(k);
+}
+
+/** The signed short-way-around turn from one angle to another, degrees. */
+export function shortestTurn(from: number, to: number): number {
let d = (to - from) % 360;
if (d > 180) d -= 360;
if (d < -180) d += 360;
@@ -129,18 +151,18 @@ export function keyAt(o: Posable, t: number): OverlayKey {
/** Insert or replace a key, keeping the list in play order. A key dropped on
* top of an existing one (within `KEY_EPSILON`) replaces it. */
-export function upsertKey(keys: OverlayKey[] | undefined, key: OverlayKey): OverlayKey[] {
+export function upsertKey<K extends { t: number }>(keys: K[] | undefined, key: K): K[] {
const rest = (keys ?? []).filter((k) => Math.abs(k.t - key.t) > KEY_EPSILON);
return sortedKeys([...rest, key]);
}
/** Drop the key at `t`, or return the list unchanged when none sits there. */
-export function removeKeyAt(keys: OverlayKey[] | undefined, t: number): OverlayKey[] {
+export function removeKeyAt<K extends { t: number }>(keys: K[] | undefined, t: number): K[] {
return (keys ?? []).filter((k) => Math.abs(k.t - t) > KEY_EPSILON);
}
/** The key sitting at `t`, if there is one. */
-export function keyIndexAt(keys: OverlayKey[] | undefined, t: number): number {
+export function keyIndexAt(keys: { t: number }[] | undefined, t: number): number {
return (keys ?? []).findIndex((k) => Math.abs(k.t - t) <= KEY_EPSILON);
}
diff --git a/site/packages/effects-kit/src/mask.test.ts b/site/packages/effects-kit/src/mask.test.ts
new file mode 100644
index 00000000..8f3ae369
--- /dev/null
+++ b/site/packages/effects-kit/src/mask.test.ts
@@ -0,0 +1,79 @@
+import { describe, expect, test } from "bun:test";
+import { isOverlayAnimated } from "./keys";
+import {
+ behindSubjectMask,
+ hasMaskKeys,
+ isMaskAnimated,
+ maskFrameAt,
+ maskKeyAt,
+ restingMaskFrame,
+ type Mask,
+ type MaskKey,
+} from "./mask";
+
+const mask = (over: Partial<Mask> = {}): Mask => ({ kind: "rect", ...over });
+const key = (t: number, over: Partial<MaskKey> = {}): MaskKey => ({
+ t,
+ x: 0,
+ y: 0,
+ w: 0.5,
+ h: 0.5,
+ rotation: 0,
+ feather: 0,
+ ...over,
+});
+
+describe("mask frames", () => {
+ test("no keys resolves to the resting geometry with defaults filled", () => {
+ expect(maskFrameAt(mask(), 1)).toEqual({ x: 0, y: 0, w: 0.5, h: 0.5, rotation: 0, feather: 0 });
+ expect(restingMaskFrame(mask({ x: 0.1, feather: 24 }))).toEqual({
+ x: 0.1,
+ y: 0,
+ w: 0.5,
+ h: 0.5,
+ rotation: 0,
+ feather: 24,
+ });
+ });
+
+ test("geometry moves between keys and holds outside them", () => {
+ const m = mask({ kf: [key(1, { w: 0.2 }), key(3, { w: 0.8, feather: 40 })] });
+ expect(maskFrameAt(m, 0).w).toBeCloseTo(0.2);
+ expect(maskFrameAt(m, 2).w).toBeCloseTo(0.5);
+ expect(maskFrameAt(m, 2).feather).toBeCloseTo(20);
+ expect(maskFrameAt(m, 9).w).toBeCloseTo(0.8);
+ });
+
+ test("rotation takes the short way around", () => {
+ const m = mask({ kf: [key(0, { rotation: -170 }), key(2, { rotation: 170 })] });
+ expect(maskFrameAt(m, 1).rotation).toBeCloseTo(-180);
+ });
+
+ test("a new key captures the live geometry, so adding one changes nothing", () => {
+ const m = mask({ kf: [key(0, { x: -0.2 }), key(4, { x: 0.3 })] });
+ const mid = maskKeyAt(m, 2);
+ const after = { ...m, kf: [...m.kf!, mid] };
+ for (const t of [0, 1, 2, 3, 4]) {
+ expect(maskFrameAt(after, t).x).toBeCloseTo(maskFrameAt(m, t).x);
+ }
+ });
+});
+
+describe("mask animation flags", () => {
+ test("keys make the mask animated, and the element with it", () => {
+ const still = mask();
+ const keyed = mask({ kf: [key(0)] });
+ expect(hasMaskKeys(still)).toBe(false);
+ expect(isMaskAnimated(keyed)).toBe(true);
+ const el = { start: 0, end: 4, x: 0.5, y: 0.5 };
+ expect(isOverlayAnimated({ ...el, mask: still })).toBe(false);
+ expect(isOverlayAnimated({ ...el, mask: keyed })).toBe(true);
+ });
+
+ test("behind-subject is an inverted subject mask", () => {
+ expect(behindSubjectMask(mask({ kind: "subject", invert: true }))).toBe(true);
+ expect(behindSubjectMask(mask({ kind: "subject" }))).toBe(false);
+ expect(behindSubjectMask(mask({ invert: true }))).toBe(false);
+ expect(behindSubjectMask(undefined)).toBe(false);
+ });
+});
diff --git a/site/packages/effects-kit/src/mask.ts b/site/packages/effects-kit/src/mask.ts
new file mode 100644
index 00000000..59ee1417
--- /dev/null
+++ b/site/packages/effects-kit/src/mask.ts
@@ -0,0 +1,284 @@
+/**
+ * Masks: per-layer coverage that trims a layer's pixels to a shape — a
+ * rounded rect, an ellipse, a half-plane, a band — or to the person in the
+ * shot. The mask travels with its layer: its center is an offset from the
+ * layer's anchor, so a keyframed or dragged layer carries its mask along in
+ * every renderer.
+ *
+ * One painter produces every mask pixel. The DOM preview turns its output
+ * into a CSS mask-image, the canvas compositors multiply it into a layer's
+ * alpha, and the ffmpeg export consumes it as client-painted grayscale
+ * pictures — so the three render paths cannot disagree about coverage.
+ *
+ * `kind: "subject"` has no geometry of its own: coverage is the person matte
+ * the host computes (MediaPipe live, a mask video for export). The painter
+ * skips it; hosts composite it through the same alpha-multiply seam.
+ */
+
+import { lerpKeys, shortestTurn } from "./keys";
+
+export type MaskKind = "rect" | "circle" | "linear" | "mirror" | "subject";
+
+/** One keyed state of a mask's geometry, `t` seconds from the layer's start.
+ * Same interpolation rules as the pose track: linear between keys, held flat
+ * outside them. */
+export interface MaskKey {
+ t: number;
+ x: number; // center offset from the layer anchor, fraction of frame width
+ y: number; // center offset, fraction of frame height
+ w: number; // fraction of frame width (linear ignores; mirror ignores)
+ h: number; // fraction of frame height (mirror: band height; linear ignores)
+ rotation: number; // degrees clockwise
+ feather: number; // edge softness, px at the 1080 design short side
+}
+
+export interface Mask {
+ kind: MaskKind;
+ /** Center offset from the layer anchor, frame fractions; absent = centered
+ * on the layer. An offset keeps the mask riding the layer's pose. */
+ x?: number;
+ y?: number;
+ /** Size, frame fractions; absent = 0.5. */
+ w?: number;
+ h?: number;
+ /** Degrees clockwise; for linear/mirror this angles the edge line. */
+ rotation?: number;
+ /** Edge softness, design px; absent = hard edge. */
+ feather?: number;
+ /** Keep the pixels outside the shape (or outside the person). */
+ invert?: boolean;
+ /** Rect corner radius, design px. */
+ radius?: number;
+ /** Keyframed geometry, its own track beside the layer's pose keys. */
+ kf?: MaskKey[];
+}
+
+/** The mask's resolved geometry at one moment. */
+export interface MaskFrame {
+ x: number;
+ y: number;
+ w: number;
+ h: number;
+ rotation: number;
+ feather: number;
+}
+
+export function restingMaskFrame(m: Mask): MaskFrame {
+ return {
+ x: m.x ?? 0,
+ y: m.y ?? 0,
+ w: m.w ?? 0.5,
+ h: m.h ?? 0.5,
+ rotation: m.rotation ?? 0,
+ feather: m.feather ?? 0,
+ };
+}
+
+export function hasMaskKeys(m?: Mask): boolean {
+ return !!m?.kf && m.kf.length > 0;
+}
+
+/** Whether the mask's pixels change over time on their own. */
+export function isMaskAnimated(m?: Mask): boolean {
+ return hasMaskKeys(m);
+}
+
+/** A mask that puts its layer behind the person in the shot — the shape the
+ * behind-speaker feature migrates into. */
+export function behindSubjectMask(m?: Mask): boolean {
+ return !!m && m.kind === "subject" && !!m.invert;
+}
+
+/** The mask's geometry at `tLocal` seconds into the layer: interpolated
+ * between keys, held flat outside them, resting when there are none. */
+export function maskFrameAt(m: Mask, tLocal: number): MaskFrame {
+ if (!m.kf || m.kf.length === 0) return restingMaskFrame(m);
+ const k = lerpKeys(m.kf, tLocal, (a, b, p) => {
+ const mix = (u: number, v: number) => u + (v - u) * p;
+ return {
+ t: mix(a.t, b.t),
+ x: mix(a.x, b.x),
+ y: mix(a.y, b.y),
+ w: mix(a.w, b.w),
+ h: mix(a.h, b.h),
+ rotation: a.rotation + shortestTurn(a.rotation, b.rotation) * p,
+ feather: mix(a.feather, b.feather),
+ };
+ });
+ return { x: k.x, y: k.y, w: k.w, h: k.h, rotation: k.rotation, feather: k.feather };
+}
+
+/** A key holding the mask's geometry at `t`, ready to be added — capturing
+ * the live frame keeps the first key a visual no-op. */
+export function maskKeyAt(m: Mask, t: number): MaskKey {
+ return { t, ...maskFrameAt(m, t) };
+}
+
+/** The frame geometry the painter needs: output size plus the design-px
+ * scale (min(width, height) / 1080). Matches render.ts's PaintFrame. */
+interface MaskPaintFrame {
+ width: number;
+ height: number;
+ scale: number;
+}
+
+/** Painters run on plain and offscreen canvases alike. */
+type Canvas2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+
+/** Wider than any rotated frame diagonal, for the unbounded fills. */
+function bigSpan(frame: MaskPaintFrame): number {
+ return (frame.width + frame.height) * 2;
+}
+
+/**
+ * Paint the mask's coverage — white where the layer keeps its pixels, before
+ * `invert` is considered — into `ctx` under its current transform. `anchor`
+ * is the layer's anchor in frame fractions; the mask centers at anchor plus
+ * its own offset. Skips `kind: "subject"` (the host owns that matte).
+ */
+export function paintMaskCoverage(
+ ctx: Canvas2D,
+ m: Mask,
+ tLocal: number,
+ frame: MaskPaintFrame,
+ anchor: { x: number; y: number }
+): void {
+ if (m.kind === "subject") return;
+ const f = maskFrameAt(m, tLocal);
+ const cx = (anchor.x + f.x) * frame.width;
+ const cy = (anchor.y + f.y) * frame.height;
+ const w = Math.max(1, f.w * frame.width);
+ const h = Math.max(1, f.h * frame.height);
+ const feather = Math.max(0, f.feather * frame.scale);
+ const big = bigSpan(frame);
+ ctx.save();
+ ctx.translate(cx, cy);
+ ctx.rotate((f.rotation * Math.PI) / 180);
+ ctx.fillStyle = "#ffffff";
+ if (m.kind === "rect") {
+ // Soft edges come from a gaussian of the hard shape; σ = feather / 2 puts
+ // the visible transition at about the feather width.
+ if (feather > 0 && "filter" in ctx) ctx.filter = `blur(${feather / 2}px)`;
+ ctx.beginPath();
+ ctx.roundRect(-w / 2, -h / 2, w, h, (m.radius ?? 0) * frame.scale);
+ ctx.fill();
+ } else if (m.kind === "circle") {
+ if (feather > 0 && "filter" in ctx) ctx.filter = `blur(${feather / 2}px)`;
+ ctx.beginPath();
+ ctx.ellipse(0, 0, w / 2, h / 2, 0, 0, Math.PI * 2);
+ ctx.fill();
+ } else if (m.kind === "linear") {
+ // Half-plane: at rotation 0 the top half stays. The gradient band spans
+ // the feather, centered on the edge line.
+ if (feather > 0) {
+ const g = ctx.createLinearGradient(0, -feather / 2, 0, feather / 2);
+ g.addColorStop(0, "rgba(255,255,255,1)");
+ g.addColorStop(1, "rgba(255,255,255,0)");
+ ctx.fillStyle = g;
+ ctx.fillRect(-big, -feather / 2, big * 2, feather);
+ ctx.fillStyle = "#ffffff";
+ ctx.fillRect(-big, -big, big * 2, big - feather / 2);
+ } else {
+ ctx.fillRect(-big, -big, big * 2, big);
+ }
+ } else if (m.kind === "mirror") {
+ // A band of height `h` centered on the line; both edges feathered. The
+ // feather never exceeds the band, so the center stays solid.
+ const fe = Math.min(feather, h);
+ if (fe > 0) {
+ const top = ctx.createLinearGradient(0, -h / 2 - fe / 2, 0, -h / 2 + fe / 2);
+ top.addColorStop(0, "rgba(255,255,255,0)");
+ top.addColorStop(1, "rgba(255,255,255,1)");
+ ctx.fillStyle = top;
+ ctx.fillRect(-big, -h / 2 - fe / 2, big * 2, fe);
+ const bottom = ctx.createLinearGradient(0, h / 2 - fe / 2, 0, h / 2 + fe / 2);
+ bottom.addColorStop(0, "rgba(255,255,255,1)");
+ bottom.addColorStop(1, "rgba(255,255,255,0)");
+ ctx.fillStyle = bottom;
+ ctx.fillRect(-big, h / 2 - fe / 2, big * 2, fe);
+ ctx.fillStyle = "#ffffff";
+ ctx.fillRect(-big, -h / 2 + fe / 2, big * 2, Math.max(0, h - fe));
+ } else {
+ ctx.fillRect(-big, -h / 2, big * 2, h);
+ }
+ }
+ ctx.restore();
+}
+
+/**
+ * Paint the mask as a luma picture — white keeps the pixel, black drops it,
+ * with `invert` already baked in — for a box cut out of the frame whose
+ * top-left sits at (boxX, boxY) in frame px. The canvas is box-sized and
+ * fully opaque, so encoders and ffmpeg's gray conversion read the feather
+ * from luma. This is what the export's mask stills sample.
+ */
+export function paintMaskLuma(
+ canvas: HTMLCanvasElement | OffscreenCanvas,
+ m: Mask,
+ tLocal: number,
+ frame: MaskPaintFrame,
+ anchor: { x: number; y: number },
+ boxX: number,
+ boxY: number
+): void {
+ const ctx = canvas.getContext("2d") as Canvas2D | null;
+ if (!ctx) return;
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.globalCompositeOperation = "source-over";
+ // Inverted: a white ground with the coverage differenced out of it, so a
+ // feathered edge reads as 1 − coverage. Plain: coverage over black.
+ ctx.fillStyle = m.invert ? "#ffffff" : "#000000";
+ ctx.fillRect(0, 0, canvas.width, canvas.height);
+ if (m.invert) ctx.globalCompositeOperation = "difference";
+ ctx.translate(-boxX, -boxY);
+ paintMaskCoverage(ctx, m, tLocal, frame, anchor);
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.globalCompositeOperation = "source-over";
+}
+
+/**
+ * Multiply `coverage`'s alpha into `target` at identity transform: keep the
+ * covered pixels, or the uncovered ones when inverted. `coverage` must match
+ * the target's pixel size.
+ */
+export function maskComposite(
+ target: Canvas2D,
+ coverage: CanvasImageSource,
+ invert?: boolean
+): void {
+ target.save();
+ target.setTransform(1, 0, 0, 1, 0, 0);
+ target.globalAlpha = 1;
+ target.globalCompositeOperation = invert ? "destination-out" : "destination-in";
+ target.drawImage(coverage, 0, 0);
+ target.restore();
+}
+
+/**
+ * Trim `target`'s pixels to the mask: paint coverage into `scratch` (sized to
+ * match) under `transform` — the same transform the layer's pixels were drawn
+ * with, so the mask rides the layer — then composite. Subject masks return
+ * untouched; the host composites its matte through `maskComposite` itself.
+ */
+export function applyMaskToCanvas(
+ target: Canvas2D,
+ scratch: HTMLCanvasElement | OffscreenCanvas,
+ m: Mask,
+ tLocal: number,
+ frame: MaskPaintFrame,
+ anchor: { x: number; y: number },
+ transform?: DOMMatrix
+): void {
+ if (m.kind === "subject") return;
+ const tw = target.canvas.width;
+ const th = target.canvas.height;
+ if (scratch.width !== tw) scratch.width = tw;
+ if (scratch.height !== th) scratch.height = th;
+ const sctx = scratch.getContext("2d") as Canvas2D;
+ sctx.setTransform(1, 0, 0, 1, 0, 0);
+ sctx.clearRect(0, 0, tw, th);
+ if (transform) sctx.setTransform(transform);
+ paintMaskCoverage(sctx, m, tLocal, frame, anchor);
+ sctx.setTransform(1, 0, 0, 1, 0, 0);
+ maskComposite(target, scratch as CanvasImageSource, m.invert);
+}
diff --git a/site/packages/effects-kit/src/render.ts b/site/packages/effects-kit/src/render.ts
index fa0a6211..bca8cdfe 100644
--- a/site/packages/effects-kit/src/render.ts
+++ b/site/packages/effects-kit/src/render.ts
@@ -13,6 +13,7 @@ import {
type OverlayAnimStyle,
} from "./anim";
import { evalOverlayFrame, hasOverlayKeys, poseAt, poseExtent, sortedKeys } from "./keys";
+import { applyMaskToCanvas, isMaskAnimated } from "./mask";
import type { LottieHandle } from "./lottie";
import { elementPlugin } from "./registry";
import {
@@ -478,6 +479,20 @@ export async function renderElementPng(
}
await paintElement(ctx, overlay, frame, env);
+ if (overlay.mask) {
+ // The mask is painted under the element's own transform, so it rotates
+ // and travels with the element. A keyframed mask routes through the
+ // animated frame sampler; here its resting (first) geometry applies.
+ applyMaskToCanvas(
+ ctx,
+ document.createElement("canvas"),
+ overlay.mask,
+ 0,
+ frame,
+ { x: overlay.x, y: overlay.y },
+ ctx.getTransform()
+ );
+ }
return pngBlob(canvas);
}
@@ -692,6 +707,7 @@ export async function renderOverlayFrames(
canvas.width = rw;
canvas.height = rh;
const ctx = canvas.getContext("2d")!;
+ const maskScratch = overlay.mask ? document.createElement("canvas") : null;
const drawAt = async (tLocal: number): Promise<Blob> => {
ctx.setTransform(1, 0, 0, 1, 0, 0);
@@ -715,7 +731,21 @@ export async function renderOverlayFrames(
// Typing reveals characters; it must not also fade them.
ctx.globalAlpha = poseAt(overlay, tLocal).opacity;
}
+ const maskTransform = ctx.getTransform();
await paintElement(ctx, el, { ...frame, t: tLocal }, env);
+ if (overlay.mask && maskScratch) {
+ // Painted under the frame's element transform, so the mask rides the
+ // pose; its own keys evaluate at this frame's time.
+ applyMaskToCanvas(
+ ctx,
+ maskScratch,
+ overlay.mask,
+ tLocal,
+ frame,
+ { x: overlay.x, y: overlay.y },
+ maskTransform
+ );
+ }
return pngBlob(canvas);
};
@@ -727,9 +757,10 @@ export async function renderOverlayFrames(
};
const step = 1 / fps;
- // A keyframed pose changes on its own schedule, so there is no still middle
- // and no cycle to repeat: sample the whole element frame by frame.
- if (keyed) {
+ // A keyframed pose (or a keyframed mask) changes on its own schedule, so
+ // there is no still middle and no cycle to repeat: sample the whole element
+ // frame by frame.
+ if (keyed || isMaskAnimated(overlay.mask)) {
const n = Math.max(1, Math.round(dur * fps));
for (let i = 0; i < n; i++) {
await push(i * step, i === n - 1 ? dur - (n - 1) * step : step);
diff --git a/site/packages/effects-kit/src/types.ts b/site/packages/effects-kit/src/types.ts
index 5b6bc81c..7942cb97 100644
--- a/site/packages/effects-kit/src/types.ts
+++ b/site/packages/effects-kit/src/types.ts
@@ -8,6 +8,7 @@
import type { OverlayAnim } from "./anim";
import type { EffectOverlay } from "./effects";
import type { OverlayKey } from "./keys";
+import type { Mask } from "./mask";
/** Fields every overlay element carries, whatever its kind. */
export interface OverlayBase {
@@ -33,6 +34,9 @@ export interface OverlayBase {
* Absent or empty = the element holds its resting pose; presets compose
* over whichever of the two is in play. */
kf?: OverlayKey[];
+ /** Coverage that trims the element's pixels to a shape or to the person in
+ * the shot (see mask.ts); absent = the whole element shows. */
+ mask?: Mask;
}
/** How a highlighted word lights up in karaoke text: accent color only,
@@ -78,10 +82,6 @@ export interface TextOverlay extends OverlayBase {
plateRadius?: number; // plate corner radius in em
plateColor?: string; // plate fill color
plateOpacity?: number; // plate fill opacity 0..1
- /** Draw this text behind the person in the shot (host-composited: video →
- * text → segmented person back on top). Falls back to a normal front title
- * when no person is detectable. */
- behindSubject?: boolean;
/** Karaoke burn-in: index of the display word (whitespace-split across all
* lines) drawn per the accent treatment. */
highlightWord?: number;