Skip to content

Open source CapCut alternative.

TypeScript
18.8 KB
Raw Blame
/**
 * Masks: per-layer coverage that trims a layer's pixels to a shape — a
 * rounded rect, an ellipse, a half-plane, a band, a heart, a star, a polygon,
 * a pen-drawn outline — or to the person in the shot. The mask travels with its layer: its center is an offset from the
 * layer's anchor, so a keyframed or dragged layer carries its mask along in
 * every renderer.
 *
 * One painter produces every mask pixel. The DOM preview turns its output
 * into a CSS mask-image, the canvas compositors multiply it into a layer's
 * alpha, and the ffmpeg export consumes it as client-painted grayscale
 * pictures — so the three render paths cannot disagree about coverage.
 *
 * `kind: "subject"` has no geometry of its own: coverage is the person matte
 * the host computes (MediaPipe live, a mask video for export). The painter
 * skips it; hosts composite it through the same alpha-multiply seam.
 */

import { lerpKeys, shortestTurn } from "./keys";
import { shapePathD, tracePolyShape } from "./shapePath";
import type { ShapeKind } from "./types";

export type MaskKind =
  | "rect"
  | "square"
  | "circle"
  | "linear"
  | "mirror"
  | "heart"
  | "star"
  | "triangle"
  | "diamond"
  | "hexagon"
  | "pen"
  | "subject";

/** Every mask kind with the name the UI shows for it, in picker order. The
 * shape picker, the chat tool's enum and the catalog all read this list. */
export const MASK_SHAPES: { id: MaskKind; label: string }[] = [
  { id: "rect", label: "Rectangle" },
  { id: "square", label: "Square" },
  { id: "circle", label: "Circle" },
  { id: "linear", label: "Linear" },
  { id: "mirror", label: "Mirror" },
  { id: "heart", label: "Heart" },
  { id: "star", label: "Star" },
  { id: "triangle", label: "Triangle" },
  { id: "diamond", label: "Diamond" },
  { id: "hexagon", label: "Hexagon" },
  { id: "pen", label: "Pen" },
  { id: "subject", label: "Subject" },
];

export const MASK_KINDS: MaskKind[] = MASK_SHAPES.map((s) => s.id);

/** The geometric kinds: everything with an outline of its own. */
export const MASK_SHAPE_KINDS: MaskKind[] = MASK_KINDS.filter((k) => k !== "subject");

/** Feather ceiling, design px at the 1080 short side. */
export const MASK_FEATHER_MAX = 600;
/** A straight edge's feather, from coverage `a` to `b` along the gradient.
 * The ramp eases at both ends (a smoothstep), the falloff a blurred edge has,
 * so a wide feather reads as soft light. A linear ramp shows its two corners
 * as bands. */
function featherStops(g: CanvasGradient, a: number, b: number) {
  for (let i = 0; i <= FEATHER_STEPS; i++) {
    const u = i / FEATHER_STEPS;
    const v = a + (b - a) * u * u * (3 - 2 * u);
    g.addColorStop(u, `rgba(255,255,255,${v.toFixed(3)})`);
  }
}
const FEATHER_STEPS = 8;

/** Corner radius ceiling for rect and square masks, design px. */
export const MASK_RADIUS_MAX = 400;

/** The polygon outline a mask kind traces, for the kinds that share the shape
 * element's tracer; null for the box, ellipse, band and matte kinds. */
export function maskPolyKind(kind: MaskKind): ShapeKind | null {
  switch (kind) {
    case "heart":
    case "star":
    case "triangle":
    case "diamond":
    case "hexagon":
      return kind;
    default:
      return null;
  }
}

/** One corner of a pen mask's outline, as an offset from the mask's center
 * in fractions of its w × h box: (−0.5, −0.5) is the box's top-left corner.
 * The box carries the outline, so the mask's position, size, rotation and
 * keys move the drawing as one piece. */
export interface MaskPoint {
  x: number;
  y: number;
}

/** The fewest corners a pen outline closes with. */
export const PEN_MIN_POINTS = 3;

/** Whether a pen mask has an outline to cut with. Short of that the mask is
 * still being drawn and the whole picture shows. */
export function penClosed(m: Mask): boolean {
  return m.kind === "pen" && (m.points?.length ?? 0) >= PEN_MIN_POINTS;
}

/** Whether the mask keeps the outside of its coverage. A pen outline still
 * being drawn shows the whole picture whichever way the toggle sits, so the
 * corners land on what the user sees. */
export function maskInverts(m: Mask): boolean {
  return !!m.invert && !(m.kind === "pen" && !penClosed(m));
}

/** Trace a pen outline into `sink`, its box w × h px centered on (dx, dy). */
export function tracePen(
  sink: { moveTo(x: number, y: number): void; lineTo(x: number, y: number): void; closePath(): void },
  points: MaskPoint[],
  w: number,
  h: number,
  dx = 0,
  dy = 0
): void {
  points.forEach((p, i) => {
    const x = dx + p.x * w;
    const y = dy + p.y * h;
    if (i === 0) sink.moveTo(x, y);
    else sink.lineTo(x, y);
  });
  sink.closePath();
}

/** Which size axes a kind has: a square one side, a mirror band one height,
 * a linear edge none, every filled outline both. */
export function maskSizeAxes(kind: MaskKind): ("w" | "h")[] {
  switch (kind) {
    case "square":
      return ["w"];
    case "mirror":
      return ["h"];
    case "linear":
    case "subject":
      return [];
    default:
      return ["w", "h"];
  }
}

/** Whether a kind takes a corner radius. */
export function maskHasRadius(kind: MaskKind): boolean {
  return kind === "rect" || kind === "square";
}

/** The hard edge of a mask's shape as SVG path data, centered on the origin
 * (offset by dx, dy) in the mask's own unrotated space, from the same
 * geometry the painter fills. A linear edge is a line across `span`, a
 * mirror band two of them. */
export function maskOutlinePathD(
  m: Mask,
  w: number,
  h: number,
  span: number,
  radiusPx = 0,
  dx = 0,
  dy = 0
): string {
  const n = (v: number) => String(+v.toFixed(2));
  const side = m.kind === "square" ? w : h;
  const poly = maskPolyKind(m.kind);
  if (poly) return shapePathD(poly, w, h, dx - w / 2, dy - h / 2);
  if (m.kind === "pen") {
    if (!penClosed(m)) return "";
    const parts: string[] = [];
    tracePen(
      {
        moveTo: (x, y) => parts.push(`M${n(x)} ${n(y)}`),
        lineTo: (x, y) => parts.push(`L${n(x)} ${n(y)}`),
        closePath: () => parts.push("Z"),
      },
      m.points!,
      w,
      h,
      dx,
      dy
    );
    return parts.join(" ");
  }
  if (m.kind === "rect" || m.kind === "square") {
    const r = Math.min(radiusPx, w / 2, side / 2);
    const x0 = dx - w / 2;
    const y0 = dy - side / 2;
    if (r <= 0) return `M${n(x0)} ${n(y0)} h${n(w)} v${n(side)} h${n(-w)} Z`;
    return (
      `M${n(x0 + r)} ${n(y0)} h${n(w - 2 * r)} a${n(r)} ${n(r)} 0 0 1 ${n(r)} ${n(r)} ` +
      `v${n(side - 2 * r)} a${n(r)} ${n(r)} 0 0 1 ${n(-r)} ${n(r)} h${n(-(w - 2 * r))} ` +
      `a${n(r)} ${n(r)} 0 0 1 ${n(-r)} ${n(-r)} v${n(-(side - 2 * r))} a${n(r)} ${n(r)} 0 0 1 ${n(r)} ${n(-r)} Z`
    );
  }
  if (m.kind === "circle") {
    const rx = w / 2;
    const ry = h / 2;
    return (
      `M${n(dx - rx)} ${n(dy)} a${n(rx)} ${n(ry)} 0 1 0 ${n(2 * rx)} 0 ` +
      `a${n(rx)} ${n(ry)} 0 1 0 ${n(-2 * rx)} 0 Z`
    );
  }
  if (m.kind === "linear") return `M${n(dx - span)} ${n(dy)} H${n(dx + span)}`;
  if (m.kind === "mirror")
    return `M${n(dx - span)} ${n(dy - h / 2)} H${n(dx + span)} M${n(dx - span)} ${n(dy + h / 2)} H${n(dx + span)}`;
  return "";
}

/** One keyed state of a mask's geometry, `t` seconds from the layer's start.
 * Same interpolation rules as the pose track: linear between keys, held flat
 * outside them. */
export interface MaskKey {
  t: number;
  x: number; // center offset from the layer anchor, fraction of frame width
  y: number; // center offset, fraction of frame height
  w: number; // fraction of frame width (square: the side; linear/mirror ignore)
  h: number; // fraction of frame height (mirror: band height; square/linear ignore)
  rotation: number; // degrees clockwise
  feather: number; // edge softness, px at the 1080 design short side
  /** Box corner radius, design px; absent = the mask's own. */
  radius?: number;
  /** A pen outline of the key's own, for an outline whose corners move — a
   * tracked shape. Keys without one draw the mask's `points`. */
  points?: MaskPoint[];
}

export interface Mask {
  kind: MaskKind;
  /** Center offset from the layer anchor, frame fractions; absent = centered
   * on the layer. An offset keeps the mask riding the layer's pose. */
  x?: number;
  y?: number;
  /** Size, frame fractions; absent = 0.5. */
  w?: number;
  h?: number;
  /** Degrees clockwise; for linear/mirror this angles the edge line. */
  rotation?: number;
  /** Edge softness, design px; absent = hard edge. */
  feather?: number;
  /** Keep the pixels outside the shape (or outside the person). */
  invert?: boolean;
  /** Rect or square corner radius, design px; a key can carry its own. */
  radius?: number;
  /** A pen mask's corners, in order around the outline. Fewer than
   * PEN_MIN_POINTS means the outline is still being drawn. */
  points?: MaskPoint[];
  /** Keyframed geometry, its own track beside the layer's pose keys. */
  kf?: MaskKey[];
}

/** The mask's resolved geometry at one moment. */
export interface MaskFrame {
  x: number;
  y: number;
  w: number;
  h: number;
  rotation: number;
  feather: number;
  radius: number;
  /** The keyed pen outline at this moment; absent when the keys carry none
   * and the mask's own `points` draw. */
  points?: MaskPoint[];
}

export function restingMaskFrame(m: Mask): MaskFrame {
  // A pen outline is drawn in frame space, so its box starts as the frame.
  const size = m.kind === "pen" ? 1 : 0.5;
  return {
    x: m.x ?? 0,
    y: m.y ?? 0,
    w: m.w ?? size,
    h: m.h ?? size,
    rotation: m.rotation ?? 0,
    feather: m.feather ?? 0,
    radius: m.radius ?? 0,
  };
}

export function hasMaskKeys(m?: Mask): boolean {
  return !!m?.kf && m.kf.length > 0;
}

/** Whether the mask's pixels change over time on their own. */
export function isMaskAnimated(m?: Mask): boolean {
  return hasMaskKeys(m);
}

/** A mask that puts its layer behind the person in the shot — the shape the
 * behind-speaker feature migrates into. */
export function behindSubjectMask(m?: Mask): boolean {
  return !!m && m.kind === "subject" && !!m.invert;
}

/** The mask's geometry at `tLocal` seconds into the layer: interpolated
 * between keys, held flat outside them, resting when there are none. */
export function maskFrameAt(m: Mask, tLocal: number): MaskFrame {
  if (!m.kf || m.kf.length === 0) return restingMaskFrame(m);
  const rest = m.radius ?? 0;
  const k = lerpKeys(m.kf, tLocal, (a, b, p) => {
    const mix = (u: number, v: number) => u + (v - u) * p;
    return {
      t: mix(a.t, b.t),
      x: mix(a.x, b.x),
      y: mix(a.y, b.y),
      w: mix(a.w, b.w),
      h: mix(a.h, b.h),
      rotation: a.rotation + shortestTurn(a.rotation, b.rotation) * p,
      feather: mix(a.feather, b.feather),
      radius: mix(a.radius ?? rest, b.radius ?? rest),
      points: mixPoints(a.points, b.points, p),
    };
  });
  return {
    x: k.x,
    y: k.y,
    w: k.w,
    h: k.h,
    rotation: k.rotation,
    feather: k.feather,
    radius: k.radius ?? rest,
    ...(k.points ? { points: k.points } : {}),
  };
}

/** Two keyed outlines blended corner by corner. Outlines with different
 * corner counts cannot blend, so the earlier one holds until the next key. */
function mixPoints(a: MaskPoint[] | undefined, b: MaskPoint[] | undefined, p: number): MaskPoint[] | undefined {
  if (!a || !b || a.length !== b.length) return a ?? b;
  return a.map((q, i) => ({ x: q.x + (b[i].x - q.x) * p, y: q.y + (b[i].y - q.y) * p }));
}

/** A key holding the mask's geometry at `t`, ready to be added — capturing
 * the live frame keeps the first key a visual no-op. */
export function maskKeyAt(m: Mask, t: number): MaskKey {
  const { radius, ...f } = maskFrameAt(m, t);
  return maskHasRadius(m.kind) ? { t, ...f, radius } : { t, ...f };
}

/** The frame geometry the painter needs: output size plus the design-px
 * scale (min(width, height) / 1080). Matches render.ts's PaintFrame. */
interface MaskPaintFrame {
  width: number;
  height: number;
  scale: number;
}

/** Painters run on plain and offscreen canvases alike. */
type Canvas2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;

/** Wider than any rotated frame diagonal, for the unbounded fills. */
function bigSpan(frame: MaskPaintFrame): number {
  return (frame.width + frame.height) * 2;
}

/**
 * Paint the mask's coverage — white where the layer keeps its pixels, before
 * `invert` is considered — into `ctx` under its current transform. `anchor`
 * is the layer's anchor in frame fractions; the mask centers at anchor plus
 * its own offset. Skips `kind: "subject"` (the host owns that matte).
 */
export function paintMaskCoverage(
  ctx: Canvas2D,
  m: Mask,
  tLocal: number,
  frame: MaskPaintFrame,
  anchor: { x: number; y: number }
): void {
  if (m.kind === "subject") return;
  const f = maskFrameAt(m, tLocal);
  const cx = (anchor.x + f.x) * frame.width;
  const cy = (anchor.y + f.y) * frame.height;
  const w = Math.max(1, f.w * frame.width);
  const h = Math.max(1, f.h * frame.height);
  const feather = Math.max(0, f.feather * frame.scale);
  const big = bigSpan(frame);
  ctx.save();
  ctx.translate(cx, cy);
  ctx.rotate((f.rotation * Math.PI) / 180);
  ctx.fillStyle = "#ffffff";
  if (m.kind === "rect" || m.kind === "square") {
    // Soft edges come from a gaussian of the hard shape; σ = feather / 2 puts
    // the visible transition at about the feather width. A square's side is
    // `w` of the frame width on both axes, so it stays square in pixels on
    // any aspect.
    const rh = m.kind === "square" ? w : h;
    if (feather > 0 && "filter" in ctx) ctx.filter = `blur(${feather / 2}px)`;
    ctx.beginPath();
    ctx.roundRect(-w / 2, -rh / 2, w, rh, Math.max(0, f.radius) * frame.scale);
    ctx.fill();
  } else if (m.kind === "circle") {
    if (feather > 0 && "filter" in ctx) ctx.filter = `blur(${feather / 2}px)`;
    ctx.beginPath();
    ctx.ellipse(0, 0, w / 2, h / 2, 0, 0, Math.PI * 2);
    ctx.fill();
  } else if (maskPolyKind(m.kind)) {
    // The shape element's own outline, filled into the w×h box.
    if (feather > 0 && "filter" in ctx) ctx.filter = `blur(${feather / 2}px)`;
    ctx.beginPath();
    tracePolyShape(ctx, maskPolyKind(m.kind)!, w, h, -w / 2, -h / 2);
    ctx.fill();
  } else if (m.kind === "pen") {
    // The drawn outline filled into the w×h box. While it is still being
    // drawn the whole picture shows, so the user draws over what they see.
    if (!penClosed(m)) {
      ctx.fillRect(-big, -big, big * 2, big * 2);
    } else {
      if (feather > 0 && "filter" in ctx) ctx.filter = `blur(${feather / 2}px)`;
      ctx.beginPath();
      tracePen(ctx, f.points ?? m.points!, w, h);
      ctx.fill();
    }
  } else if (m.kind === "linear") {
    // Half-plane: at rotation 0 the top half stays. The gradient band spans
    // the feather, centered on the edge line.
    if (feather > 0) {
      const g = ctx.createLinearGradient(0, -feather / 2, 0, feather / 2);
      featherStops(g, 1, 0);
      ctx.fillStyle = g;
      ctx.fillRect(-big, -feather / 2, big * 2, feather);
      ctx.fillStyle = "#ffffff";
      ctx.fillRect(-big, -big, big * 2, big - feather / 2);
    } else {
      ctx.fillRect(-big, -big, big * 2, big);
    }
  } else if (m.kind === "mirror") {
    // A band of height `h` centered on the line; both edges feathered. The
    // feather never exceeds the band, so the center stays solid.
    const fe = Math.min(feather, h);
    if (fe > 0) {
      const top = ctx.createLinearGradient(0, -h / 2 - fe / 2, 0, -h / 2 + fe / 2);
      featherStops(top, 0, 1);
      ctx.fillStyle = top;
      ctx.fillRect(-big, -h / 2 - fe / 2, big * 2, fe);
      const bottom = ctx.createLinearGradient(0, h / 2 - fe / 2, 0, h / 2 + fe / 2);
      featherStops(bottom, 1, 0);
      ctx.fillStyle = bottom;
      ctx.fillRect(-big, h / 2 - fe / 2, big * 2, fe);
      ctx.fillStyle = "#ffffff";
      ctx.fillRect(-big, -h / 2 + fe / 2, big * 2, Math.max(0, h - fe));
    } else {
      ctx.fillRect(-big, -h / 2, big * 2, h);
    }
  }
  ctx.restore();
}

/**
 * Paint the mask as a luma picture — white keeps the pixel, black drops it,
 * with `invert` already baked in — for a box cut out of the frame whose
 * top-left sits at (boxX, boxY) in frame px. The canvas is box-sized and
 * fully opaque, so encoders and ffmpeg's gray conversion read the feather
 * from luma. This is what the export's mask stills sample.
 */
export function paintMaskLuma(
  canvas: HTMLCanvasElement | OffscreenCanvas,
  m: Mask,
  tLocal: number,
  frame: MaskPaintFrame,
  anchor: { x: number; y: number },
  boxX: number,
  boxY: number
): void {
  const ctx = canvas.getContext("2d") as Canvas2D | null;
  if (!ctx) return;
  ctx.setTransform(1, 0, 0, 1, 0, 0);
  ctx.globalCompositeOperation = "source-over";
  // Inverted: a white ground with the coverage differenced out of it, so a
  // feathered edge reads as 1 − coverage. Plain: coverage over black.
  const invert = maskInverts(m);
  ctx.fillStyle = invert ? "#ffffff" : "#000000";
  ctx.fillRect(0, 0, canvas.width, canvas.height);
  if (invert) ctx.globalCompositeOperation = "difference";
  ctx.translate(-boxX, -boxY);
  paintMaskCoverage(ctx, m, tLocal, frame, anchor);
  ctx.setTransform(1, 0, 0, 1, 0, 0);
  ctx.globalCompositeOperation = "source-over";
}

/**
 * Multiply `coverage`'s alpha into `target` at identity transform: keep the
 * covered pixels, or the uncovered ones when inverted. `coverage` must match
 * the target's pixel size.
 */
export function maskComposite(
  target: Canvas2D,
  coverage: CanvasImageSource,
  invert?: boolean
): void {
  target.save();
  target.setTransform(1, 0, 0, 1, 0, 0);
  target.globalAlpha = 1;
  // A shadowed element leaves its shadow set, and browsers shadow this draw
  // too: the cut would keep the layer at the shadow's opacity.
  target.shadowColor = "transparent";
  target.shadowBlur = 0;
  target.shadowOffsetX = 0;
  target.shadowOffsetY = 0;
  target.globalCompositeOperation = invert ? "destination-out" : "destination-in";
  target.drawImage(coverage, 0, 0);
  target.restore();
}

/**
 * Trim `target`'s pixels to the mask: paint coverage into `scratch` (sized to
 * match) under `transform` — the same transform the layer's pixels were drawn
 * with, so the mask rides the layer — then composite. Subject masks return
 * untouched; the host composites its matte through `maskComposite` itself.
 */
export function applyMaskToCanvas(
  target: Canvas2D,
  scratch: HTMLCanvasElement | OffscreenCanvas,
  m: Mask,
  tLocal: number,
  frame: MaskPaintFrame,
  anchor: { x: number; y: number },
  transform?: DOMMatrix
): void {
  if (m.kind === "subject") return;
  const tw = target.canvas.width;
  const th = target.canvas.height;
  if (scratch.width !== tw) scratch.width = tw;
  if (scratch.height !== th) scratch.height = th;
  const sctx = scratch.getContext("2d") as Canvas2D;
  sctx.setTransform(1, 0, 0, 1, 0, 0);
  sctx.clearRect(0, 0, tw, th);
  if (transform) sctx.setTransform(transform);
  paintMaskCoverage(sctx, m, tLocal, frame, anchor);
  sctx.setTransform(1, 0, 0, 1, 0, 0);
  maskComposite(target, scratch as CanvasImageSource, maskInverts(m));
}