Skip to content

Open source CapCut alternative.

TypeScript
30.8 KB
Raw Blame
"use client";

/**
 * The subject-mask pass, shared by the live preview and the in-tab export.
 * The person matte drives two things: elements behind the speaker (with the
 * video layers already on the canvas, snapshot them, draw the behind-tagged
 * rasters, then segment the snapshot and draw the person back on top) and
 * elements or clips trimmed to the speaker. When no person registers the
 * behind effect degrades to the plain picture.
 */

import {
  applyWordDraw,
  countStep,
  countText,
  cutAtFaces,
  diveView,
  divesAt,
  erodeLevel,
  evalOverlayFrame,
  fontCycleCuts,
  measureDiveFocus,
  peekDiveFocus,
  glyphStateAt,
  hasGlyphMotion,
  maskComposite,
  overlayWords,
  paintElementCrop,
  typedChars,
  withFontCycle,
  wordSampleWindows,
  zapSeed,
  type OverlayFrameState,
  type PaintPhase,
  type ZapPhase,
} from "@donkeycut/effects-kit";
import { ElementFx, elementLook } from "@donkeycut/effects-kit";
import { personSegmenter, segmentSubjectAlpha } from "./cutout";
import { allowance, holdMemory } from "./memoryBudget";
import { overlayLiveAt } from "./overlayLive";
import { createRasterCanvas, snapshotRaster } from "./raster";
import { cutRenderEnv } from "./textRender";
import {
  behindSubjectOverlay,
  frontSubjectOverlay,
  isEffectOverlay,
  isTextOverlay,
  subjectMasked,
  type MediaAsset,
  type Overlay,
} from "./types";

type Segmenter = import("@mediapipe/tasks-vision").ImageSegmenter;

/** An element's own length, the span its word emphasis is timed across. */
const spanOf = (o: Overlay): number => Math.max(0.1, o.end - o.start);

/** Pictures here are full-frame, so a word effect that ramps is sampled
 * coarsely: enough slices that an arrival reads, few enough that a behind-
 * subject line does not hold a hundred frames of pixels resident. */
const BEHIND_WORD_FPS = 8;

/** The spans an element's word effect holds still for — one picture each.
 * Nothing here means the element draws as one piece. */
function rasterSpans(o: Overlay): { start: number; end: number }[] | null {
  if (!isTextOverlay(o)) return null;
  // A font cycle changes the type at every step: each face gets its own
  // picture, cut inside the word spans when both run.
  const words = overlayWords(o);
  const faces = fontCycleCuts(o, spanOf(o));
  if (!words && faces.length === 0) return null;
  const base = words ? wordSampleWindows(o.text, words, spanOf(o), BEHIND_WORD_FPS) : [{ start: 0, end: spanOf(o) }];
  const spans = cutAtFaces(base, faces);
  return spans.length > 0 ? spans : null;
}

/** The picture indices an element needs: one per span while a word effect
 * walks it, and the plain picture (-1) otherwise. */
function rasterSlices(o: Overlay): number[] {
  const spans = rasterSpans(o);
  return spans ? spans.map((_, i) => i) : [-1];
}

/** Which picture stands for `tLocal`, or -1 for the plain one. */
function rasterSliceAt(o: Overlay, tLocal: number): number {
  const spans = rasterSpans(o);
  if (!spans) return -1;
  const i = spans.findIndex((s) => tLocal >= s.start && tLocal < s.end);
  return i < 0 ? spans.length - 1 : i;
}

/**
 * Which picture an element needs at one moment: its word slice, how many
 * characters a typewriter has typed (-1 = the whole text), and whether the
 * typing bar is lit — the same reading the front path's painters take from
 * the evaluator. A whole-text picture keys at zero or above (two per word
 * slice, bar dark and lit); a typing picture keys below zero, one per typed
 * count, and a counting picture likewise, one per step its numbers take.
 * A disintegrating or crackling moment keys below zero too, one picture per
 * erosion step and arc deal.
 */
interface Look {
  key: number;
  word: number;
  chars: number;
  caret: boolean;
  /** Where a counting picture's numbers stand, 0..1 (count). */
  count?: number;
  /** The share of the ink a disintegration has eaten, at its erosion step. */
  erode?: number;
  /** The arcs a zap hit paints over the picture. */
  zap?: ZapPhase;
}

/** The picture `o` needs at `tLocal`. */
function lookAt(o: Overlay, tLocal: number, ev: OverlayFrameState): Look {
  const word = rasterSliceAt(o, tLocal);
  const caret = ev.caret === true;
  const n = isTextOverlay(o) ? [...o.text].length : 0;
  const chars = isTextOverlay(o) && ev.textProgress !== undefined ? typedChars(o.text, ev.textProgress) : -1;
  const bar = caret ? 1 : 0;
  // A count keys its pictures by how far its numbers have counted, out of
  // every step they take ("100 %" takes 100).
  const count = isTextOverlay(o) ? ev.countProgress : undefined;
  const text = isTextOverlay(o) ? o.text : "";
  const step = count !== undefined ? countStep(text, count) : chars;
  const steps = count !== undefined ? countStep(text, 1) : n;
  // An erosion step and an arc deal fold into a phase slot: 0 is neither,
  // e.g. step 3 under deal 5 is 3 + 25 × 6 with 24 erosion steps.
  const level = ev.erode !== undefined ? erodeLevel(ev.erode) : 0;
  const zap = ev.zap ? { amount: ev.zap, seed: zapSeed(tLocal) } : undefined;
  const levels = erodeLevel(1) + 1;
  const phase = level + levels * (zap ? zap.seed + 1 : 0);
  const phases = levels * (zapSeed(spanOf(o)) + 2);
  const picture = ((word + 1) * (steps + 2) + step + 1) * 2 + bar;
  const key = step < 0 && phase === 0 ? (word + 1) * 2 + bar : -1 - (picture * phases + phase);
  return {
    key,
    word,
    chars,
    caret,
    ...(count !== undefined ? { count } : {}),
    ...(level > 0 ? { erode: level / (levels - 1) } : {}),
    ...(zap ? { zap } : {}),
  };
}

/** The phase a picture with its typing bar lit is painted at. */
const CARET_LIT: PaintPhase = { caret: true };

/** Segmentation input width — small on purpose; this runs per frame. */
const SEG_WIDTH = 256;

/** Whether the element has pixels the pass could draw at all. */
function drawable(o: Overlay): boolean {
  if (o.hidden || isEffectOverlay(o)) return false;
  if (isTextOverlay(o) && !o.text.trim()) return false;
  return true;
}

/** The behind-the-speaker elements live at `t` (any drawable kind). */
export function behindOverlaysAt(overlays: Overlay[], t: number): Overlay[] {
  return overlays.filter(
    (o) => behindSubjectOverlay(o) && drawable(o) && overlayLiveAt(o, t)
  );
}

/** Whether any element in the document sits behind the speaker. */
export function hasBehindOverlays(overlays: Overlay[]): boolean {
  return overlays.some((o) => behindSubjectOverlay(o) && drawable(o));
}

/** Whether anything in the document reads the person matte at all. */
export function hasSubjectOverlays(overlays: Overlay[]): boolean {
  return overlays.some((o) => subjectMasked(o) && drawable(o));
}

/** One computed matte frame: `alpha` holds the person's silhouette, and null
 * means the frame was segmented and no person registered — coverage is
 * empty, so a front subject layer shows nothing and a behind layer shows
 * whole. A compositor hands out null (no frame at all) only while the
 * segmenter is still loading. */
export interface SubjectMatte {
  alpha: HTMLCanvasElement | null;
}

/** The latest matte the running preview computed, published for the DOM
 * layer: front subject-masked elements turn it into a CSS mask-image. Only
 * the preview's compositor writes it — an in-tab export runs its own
 * compositor on its own clock and keeps out of the live editor's mattes. */
let published: { canvas: HTMLCanvasElement | null; at: number } | null = null;
export function subjectMatteSnapshot(): { canvas: HTMLCanvasElement | null; at: number } | null {
  return published;
}

/** Scratch off the raster seam. The pass types its surfaces as page canvases;
 * headless they are server canvases with the same 2D interface. */
const scratchCanvas = () => createRasterCanvas(1, 1) as HTMLCanvasElement;

/** Element pictures the pass may hold: the elements live on one frame are a
 * handful of full-size pictures, and a play through the cut reaches every
 * element in it. */
const RASTER_TUNED = 48 * 2 ** 20;

/** What one element's pictures cost: one per word it lights, four bytes a
 * pixel, at the size of the frame they are drawn on. */
const entryBytes = (e: { byWord: Map<number, ImageBitmap> }): number => {
  let n = 0;
  for (const b of e.byWord.values()) n += b.width * b.height * 4;
  return n;
};

/** One element's pictures: whole-text ones by word slice and bar, and the
 * typing picture it is on. `pending` holds the keys being drawn, `want` the
 * typing picture it is waiting for, and `typedAt` when the pump last served
 * it, so elements typing at once take turns. */
interface RasterEntry {
  of: Overlay;
  byWord: Map<number, ImageBitmap>;
  pending: Set<number>;
  usedAt: number;
  last: ImageBitmap | null;
  want: Look | null;
  typedAt: number;
}

export class SubjectMaskCompositor {
  /** `publishes` marks the live preview's instance, the one whose matte the
   * DOM layer reads. */
  constructor(private publishes = false) {}

  private segmenter: Segmenter | null = null;
  private segKicked = false;
  /**
   * One picture per element, and one per emphasized word when the element
   * lights its words as they are said.
   *
   * Filed under the element's id and stamped with the element object the
   * picture was drawn from. The store hands back a new object for every edit,
   * so a stamp that no longer matches is a picture of an element that has since
   * changed: it closes and redraws. Keying on the object itself would say the
   * same thing, and would leave every superseded picture holding pixels outside
   * the heap that only `close()` gives back, with a keystroke's worth minted
   * per keystroke.
   *
   * `pending` rides in the entry so it is dropped whenever the pictures are.
   * It marks a draw already asked for, and a marker outliving the picture it
   * was asked for would leave the element blank for good.
   */
  private rasters = new Map<string, RasterEntry>();
  /** Typing pictures are painted one at a time on this surface: a typed
   * count is a picture shown for a frame or two, so it skips the PNG round
   * trip, and an element that types faster than the pictures land shows the
   * latest count it reached. */
  private typeSurface: HTMLCanvasElement | null = null;
  private typing = false;
  /** Typing pictures painted, the clock the pump takes turns on. */
  private typed = 0;
  private typeArgs: { w: number; h: number; assets: MediaAsset[] } = { w: 1, h: 1, assets: [] };
  /** Frames this pass has drawn, which is the clock the pictures are given
   * back on: an element drawn on the frame being composited is the one the
   * next frame will ask for again. */
  private pass = 0;
  private readonly releaseMemory = holdMemory("overlayRasters", () => {
    let n = 0;
    for (const e of this.rasters.values()) n += entryBytes(e);
    const typing = this.typeSurface ? this.typeSurface.width * this.typeSurface.height * 4 : 0;
    return n + this.fx.bytes() + typing;
  });
  private person: HTMLCanvasElement | null = null;
  /** Scratch for blurred and motion-blurred behind elements. */
  private fx = new ElementFx(createRasterCanvas);
  /** Where each picture's top-left sits in the frame: pictures are cropped to
   * the element's ink, however far past the frame it reaches, so a pose that
   * shrinks an oversize element keeps its edges. */
  private origins = new WeakMap<ImageBitmap, { x: number; y: number }>();
  /** The crop's mask scratch, made on first use. */
  private cropMask: HTMLCanvasElement | null = null;
  private small: HTMLCanvasElement | null = null;
  private mask: { at: number; alpha: HTMLCanvasElement | null } = { at: -1e9, alpha: null };
  /** A second matte slot for mid-stack clip masks: it snapshots the canvas as
   * it stands when the masked clip draws (the layers beneath it), so a
   * masked layer never reads its own trimmed pixels back. */
  private clipMatte: { at: number; alpha: HTMLCanvasElement | null } = { at: -1e9, alpha: null };

  /** Kick the (shared) segmenter load; safe to call every frame. */
  private ensureSegmenter() {
    if (this.segKicked) return;
    this.segKicked = true;
    void personSegmenter().then((s) => {
      this.segmenter = s;
    });
  }

  /** This element's entry, emptied first when the pictures in it were drawn
   * from an older version of the element. */
  private entryFor(o: Overlay) {
    let entry = this.rasters.get(o.id);
    if (entry && entry.of !== o) {
      for (const b of entry.byWord.values()) b.close();
      entry.byWord.clear();
      entry.pending.clear();
      entry.of = o;
      entry.last = null;
      entry.want = null;
    }
    if (!entry) {
      entry = {
        of: o,
        byWord: new Map(),
        pending: new Set(),
        usedAt: this.pass,
        last: null,
        want: null,
        typedAt: -1,
      };
      this.rasters.set(o.id, entry);
    }
    entry.usedAt = this.pass;
    return entry;
  }

  /**
   * Give back the pictures of the elements this pass has gone longest without
   * drawing, once what it holds is past its share.
   *
   * A picture is one full-size frame per element, so a play through a cut
   * whose elements come and go leaves the pass holding every one it has ever
   * drawn. What a frame is drawing was drawn on the frame before it, so the
   * live ones are the youngest and stay; an element the playhead has left
   * draws again from scratch when it is next reached.
   */
  private sweepRasters(): void {
    const cap = allowance("overlayRasters", RASTER_TUNED);
    let held = 0;
    for (const e of this.rasters.values()) held += entryBytes(e);
    if (held <= cap) return;
    for (const [id, e] of [...this.rasters].sort((a, b) => a[1].usedAt - b[1].usedAt)) {
      if (held <= cap) break;
      // The sweep runs at the head of a pass, before anything this frame is
      // stamped, so the frame's own pictures are the ones the pass before it
      // drew. Dropping those would close bitmaps the next draw asks for and
      // decode them again.
      if (e.usedAt >= this.pass - 1) continue;
      held -= entryBytes(e);
      for (const b of e.byWord.values()) b.close();
      this.rasters.delete(id);
    }
  }

  /**
   * The picture `look` names, or while it is being drawn the last one this
   * element landed, so a typing title moves on a frame late and never blinks
   * out.
   */
  private rasterFor(o: Overlay, look: Look, w: number, h: number, assets: MediaAsset[]): ImageBitmap | null {
    const entry = this.entryFor(o);
    const hit = entry.byWord.get(look.key);
    if (hit) return hit;
    if (look.key >= 0) {
      // Done typing: a typed count still queued is a picture nothing shows.
      entry.want = null;
      if (entry.pending.has(look.key)) return entry.last;
      entry.pending.add(look.key);
      // Neutral picture: position aside, the per-frame pose owns rotation and
      // opacity, so baking them here would apply each of them twice.
      void this.drawRaster(o, w, h, assets, look.key, look.word, look.caret).catch(() => {});
      return entry.last;
    }
    if (!entry.pending.has(look.key)) entry.want = look;
    this.typeArgs.w = w;
    this.typeArgs.h = h;
    this.typeArgs.assets = assets;
    this.pumpTyping();
    return entry.last;
  }

  /** The element as one picture shows it: its words as they stand across
   * the span the picture covers, `chars` of its text typed (-1 = all), its
   * numbers counted to `count`. */
  private pictureOf(o: Overlay, word: number, count?: number): Overlay {
    const spans = rasterSpans(o);
    const span = word >= 0 ? spans?.[word] : undefined;
    const at = span ? (span.start + span.end) / 2 : 0;
    const el = withFontCycle(applyWordDraw({ ...o, rotation: undefined, opacity: undefined }, at, spanOf(o)), at);
    if (count !== undefined && isTextOverlay(el)) {
      return { ...el, text: countText(el.text, count) };
    }
    return el;
  }

  /** One whole-text picture, kept under the element it came from. */
  private async drawRaster(
    o: Overlay,
    w: number,
    h: number,
    assets: MediaAsset[],
    key: number,
    word: number,
    caret: boolean
  ): Promise<void> {
    const surface = scratchCanvas();
    const origin = await paintElementCrop(surface, this.pictureOf(o, word), w, h, cutRenderEnv(assets), {
      t: 0,
      phase: caret ? CARET_LIT : undefined,
      maskScratch: (this.cropMask ??= scratchCanvas()),
    });
    const bmp = await snapshotRaster(surface);
    surface.width = surface.height = 1;
    this.origins.set(bmp, origin);
    this.land(o, key, bmp);
  }

  /** One typing picture, painted on the shared surface. */
  private async drawTyped(o: Overlay, w: number, h: number, assets: MediaAsset[], look: Look): Promise<void> {
    const surface = (this.typeSurface ??= scratchCanvas());
    // A typing picture keeps the whole text's layout and shows its typed head.
    // A disintegrating or crackling picture carries its erosion and arcs.
    const typing: PaintPhase | undefined = look.chars >= 0 ? { typed: look.chars } : undefined;
    const lived: PaintPhase = {
      ...(look.caret ? CARET_LIT : {}),
      ...typing,
      ...(look.erode !== undefined ? { erode: look.erode } : {}),
      ...(look.zap ? { zap: look.zap } : {}),
    };
    const origin = await paintElementCrop(surface, this.pictureOf(o, look.word, look.count), w, h, cutRenderEnv(assets), {
      t: 0,
      phase: Object.keys(lived).length > 0 ? lived : undefined,
      maskScratch: (this.cropMask ??= scratchCanvas()),
    });
    const bmp = await snapshotRaster(surface);
    this.origins.set(bmp, origin);
    this.land(o, look.key, bmp);
  }

  /** Paint the typing pictures elements are waiting on, one after another,
   * each at the latest count its element asked for. Elements typing at once
   * take turns: the one served longest ago goes next. */
  private pumpTyping(): void {
    if (this.typing) return;
    this.typing = true;
    void (async () => {
      try {
        for (;;) {
          let next: RasterEntry | null = null;
          for (const e of this.rasters.values()) {
            if (e.want && (!next || e.typedAt < next.typedAt)) next = e;
          }
          if (!next) break;
          const entry = next;
          const look = entry.want!;
          entry.want = null;
          entry.typedAt = this.typed++;
          entry.pending.add(look.key);
          const { w, h, assets } = this.typeArgs;
          await this.drawTyped(entry.of, w, h, assets, look).catch(() => entry.pending.delete(look.key));
        }
      } finally {
        this.typing = false;
      }
    })();
  }

  /** File a finished picture. Any typing picture it supersedes goes: an
   * element holds one typed count at a time, however long it types. */
  private land(o: Overlay, key: number, bmp: ImageBitmap): void {
    // The element may have been edited, or the pictures released, while this
    // one was being drawn; either way its entry is gone or stamped with a
    // different object, and this picture is of the wrong thing.
    const entry = this.rasters.get(o.id);
    if (!entry || entry.of !== o) {
      bmp.close();
      return;
    }
    entry.pending.delete(key);
    for (const [k, b] of entry.byWord) {
      if (k >= 0 || k === key) continue;
      b.close();
      entry.byWord.delete(k);
    }
    entry.byWord.get(key)?.close();
    entry.byWord.set(key, bmp);
    entry.last = bmp;
  }

  /** Export path: the picture every behind element needs at `t`, drawn
   * before the frame is, so no frame of a render shows a stale one. */
  async ready(overlays: Overlay[], w: number, h: number, assets: MediaAsset[], t: number): Promise<void> {
    for (const o of overlays) {
      if (!behindSubjectOverlay(o) || !drawable(o) || !overlayLiveAt(o, t)) continue;
      const tLocal = Math.max(0, t - o.start);
      const look = lookAt(o, tLocal, evalOverlayFrame(o, tLocal, w / h));
      if (this.entryFor(o).byWord.has(look.key)) continue;
      try {
        if (look.key < 0) await this.drawTyped(o, w, h, assets, look);
        else await this.drawRaster(o, w, h, assets, look.key, look.word, look.caret);
      } catch {
        // The overlay just draws in front when its raster is missing.
      }
    }
  }

  /** Give back every picture this pass is holding. The pass stays usable and
   * draws them again when it is next asked. */
  clear(): void {
    for (const e of this.rasters.values()) for (const b of e.byWord.values()) b.close();
    this.rasters.clear();
  }

  /** Give the pictures back and stop reporting what this pass holds. */
  dispose(): void {
    this.clear();
    this.fx.dispose();
    if (this.typeSurface) {
      this.typeSurface.width = 1;
      this.typeSurface.height = 1;
      this.typeSurface = null;
    }
    if (this.cropMask) {
      this.cropMask.width = this.cropMask.height = 1;
      this.cropMask = null;
    }
    this.releaseMemory();
  }

  /** Export path: everything resident before the first frame draws. */
  async prepare(overlays: Overlay[], w: number, h: number, assets: MediaAsset[]): Promise<void> {
    this.ensureSegmenter();
    await personSegmenter().then((s) => {
      this.segmenter = s;
    });
    const behind = overlays.filter((o) => behindSubjectOverlay(o) && drawable(o));
    await Promise.all(
      behind.filter((o) => divesAt(o.anim)).map((o) =>
        measureDiveFocus(o, w / h, cutRenderEnv(assets)).catch(() => {})
      )
    );
    await Promise.all(
      behind.flatMap((o) =>
        // An element that lights its words needs one picture per word, all of
        // them resident before the first frame draws.
        rasterSlices(o).map(async (word) => {
          const key = (word + 1) * 2;
          if (this.entryFor(o).byWord.has(key)) return;
          try {
            await this.drawRaster(o, w, h, assets, key, word, false);
          } catch {
            // The overlay just draws in front when its raster is missing.
          }
        })
      )
    );
  }

  /** Segment `source`'s current pixels into a person-alpha canvas, throttled
   * into `slot`. Returns null while the segmenter loads; a computed frame
   * with no person carries `alpha: null`. */
  private matteOf(
    source: HTMLCanvasElement | OffscreenCanvas,
    t: number,
    slot: { at: number; alpha: HTMLCanvasElement | null },
    minInterval: number
  ): SubjectMatte | null {
    this.ensureSegmenter();
    if (!this.segmenter) return null;
    if (slot.at > -1e8 && Math.abs(t - slot.at) < minInterval) return { alpha: slot.alpha };
    if (!this.small) this.small = scratchCanvas();
    const W = source.width;
    const H = source.height;
    const sw = SEG_WIDTH;
    const sh = Math.max(2, Math.round((SEG_WIDTH * H) / W));
    if (this.small.width !== sw || this.small.height !== sh) {
      this.small.width = sw;
      this.small.height = sh;
    }
    const sctx = this.small.getContext("2d", { willReadFrequently: true })!;
    sctx.clearRect(0, 0, sw, sh);
    sctx.drawImage(source, 0, 0, sw, sh);
    slot.at = t;
    slot.alpha = segmentSubjectAlpha(this.segmenter, this.small);
    return { alpha: slot.alpha };
  }

  /**
   * The matte a subject-masked video clip trims by, read mid-stack: the
   * canvas holds the layers beneath the clip at call time. Feed this as the
   * FrameCompositor's subject-matte provider.
   */
  clipMatteOf(
    canvas: HTMLCanvasElement | OffscreenCanvas,
    t: number,
    opts: { minMaskInterval?: number } = {}
  ): SubjectMatte | null {
    return this.matteOf(canvas, t, this.clipMatte, opts.minMaskInterval ?? 1 / 15);
  }

  /** Refresh the full-composite matte for front subject-masked elements,
   * with the video layers already on the canvas; the preview instance also
   * publishes it for the DOM layer. */
  publishMatte(
    canvas: HTMLCanvasElement | OffscreenCanvas,
    t: number,
    opts: { minMaskInterval?: number } = {}
  ): SubjectMatte | null {
    const res = this.matteOf(canvas, t, this.mask, opts.minMaskInterval ?? 1 / 15);
    if (this.publishes) {
      published = res ? { canvas: res.alpha, at: this.mask.at } : null;
    }
    return res;
  }

  /** Trim a stamped layer's pixels to the current matte (`invert` keeps the
   * outside), through `scratch`; feather blurs the matte edge. A computed
   * frame with no person clears a front layer (coverage is empty) and keeps
   * a behind layer whole; with no computed frame the layer stays whole. */
  applyMatte(
    target: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
    scratch: HTMLCanvasElement,
    invert: boolean,
    featherPx: number
  ): void {
    const matte = this.mask.alpha;
    const W = target.canvas.width;
    const H = target.canvas.height;
    if (!matte) {
      if (this.mask.at > -1e8 && !invert) target.clearRect(0, 0, W, H);
      return;
    }
    if (scratch.width !== W) scratch.width = W;
    if (scratch.height !== H) scratch.height = H;
    const sctx = scratch.getContext("2d")!;
    sctx.clearRect(0, 0, W, H);
    if (featherPx > 0 && "filter" in sctx) sctx.filter = `blur(${featherPx / 2}px)`;
    sctx.imageSmoothingEnabled = true;
    sctx.drawImage(matte, 0, 0, W, H);
    sctx.filter = "none";
    maskComposite(target, scratch, invert);
  }

  /** One stamped layer trimmed to the current matte: `draw` puts the stamp's
   * pixels (posed and all) onto a per-instance surface, then the matte
   * multiplies in at identity — the matte stays anchored to the frame while
   * the element moves under it. The surfaces live and die with this pass. */
  private stampSurface: HTMLCanvasElement | null = null;
  private stampScratch: HTMLCanvasElement | null = null;
  mattedStamp(
    o: { mask?: { invert?: boolean; feather?: number } },
    w: number,
    h: number,
    draw: (ctx: CanvasRenderingContext2D) => void
  ): CanvasImageSource {
    if (!this.stampSurface || !this.stampScratch) {
      this.stampSurface = scratchCanvas();
      this.stampScratch = scratchCanvas();
    }
    if (this.stampSurface.width !== w) this.stampSurface.width = w;
    if (this.stampSurface.height !== h) this.stampSurface.height = h;
    const ctx = this.stampSurface.getContext("2d")!;
    ctx.setTransform(1, 0, 0, 1, 0, 0);
    ctx.clearRect(0, 0, w, h);
    ctx.save();
    draw(ctx);
    ctx.restore();
    this.applyMatte(
      ctx,
      this.stampScratch,
      !!o.mask?.invert,
      (o.mask?.feather ?? 0) * (Math.min(w, h) / 1080)
    );
    return this.stampSurface;
  }

  /**
   * Run the behind pass on `canvas` (video layers already drawn): draw the
   * behind-tagged rasters, then the segmented person back over them. Also
   * refreshes and publishes the composite matte, so front subject elements
   * can read it even on frames with no behind element live. `minMaskInterval`
   * throttles segmentation for the live preview; pass 0 for exports.
   */
  draw(
    canvas: HTMLCanvasElement | OffscreenCanvas,
    overlays: Overlay[],
    assets: MediaAsset[],
    t: number,
    opts: { minMaskInterval?: number } = {}
  ): void {
    // A deleted element's picture has nothing left to draw it. Filing by id
    // means the entry outlives the element, so the document's own list is what
    // says which entries are still owed — checked only when there are more of
    // them than elements, which is the only way one can be stale.
    this.pass++;
    if (this.rasters.size > overlays.length) {
      const ids = new Set(overlays.map((o) => o.id));
      for (const [id, e] of this.rasters) {
        if (ids.has(id)) continue;
        for (const b of e.byWord.values()) b.close();
        this.rasters.delete(id);
      }
    }
    this.sweepRasters();
    const active = behindOverlaysAt(overlays, t);
    const wantsMatte = overlays.some(
      (o) => frontSubjectOverlay(o) && drawable(o) && overlayLiveAt(o, t)
    );
    if (active.length === 0 && !wantsMatte) return;
    const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
    if (!ctx) return;
    const W = canvas.width;
    const H = canvas.height;

    // The person source: the video pixels before any text lands on them.
    if (!this.person || this.person.width !== W || this.person.height !== H) {
      this.person = scratchCanvas();
      this.person.width = W;
      this.person.height = H;
    }
    const pctx = this.person.getContext("2d")!;
    pctx.globalCompositeOperation = "source-over";
    pctx.clearRect(0, 0, W, H);
    pctx.drawImage(canvas, 0, 0);

    // Segment (throttled) and publish before anything composites: between
    // refreshes the previous matte rides the new frame (the subject moves
    // little in 1/15s, and a lagging mask beats a stuttering preview).
    const alpha = this.publishMatte(canvas, t, opts)?.alpha ?? null;
    if (active.length === 0) return;

    // The behind elements, drawn straight onto the composite: a neutral
    // raster under the element's pose for this moment, exactly like the
    // export's stamped layers.
    const scale = Math.min(W, H) / 1080;
    for (const o of active) {
      const tLocal = Math.max(0, t - o.start);
      const ev = evalOverlayFrame(o, tLocal, W / H);
      const bmp = this.rasterFor(o, lookAt(o, tLocal, ev), W, H, assets);
      if (!bmp) continue;
      // One cached picture per element here, so a per-glyph ramp or loop runs
      // its motion over the whole box as a single letter would.
      const g = hasGlyphMotion(ev) ? glyphStateAt(ev, 0, 1) : null;
      const alphaOf = ev.opacity * (g?.alpha ?? 1);
      if (alphaOf <= 0.001) continue;
      const cx = o.x * W;
      const cy = o.y * H;
      // A blurred, streaking, darkening or tilted element poses into the
      // scratch alone and lands softened, dimmed or turned.
      const at = {
        x: ev.x * W + (ev.dx + (g?.dx ?? 0)) * scale,
        y: ev.y * H + (ev.dy + (g?.dy ?? 0)) * scale,
      };
      const target = this.fx.begin(ctx, W, H, elementLook(ev, scale, at)) as CanvasRenderingContext2D;
      target.save();
      target.globalAlpha = alphaOf;
      target.translate(at.x, at.y);
      target.rotate(((ev.rotation + (g?.rotate ?? 0)) * Math.PI) / 180);
      target.scale(ev.scale * (ev.sx ?? 1) * (g?.sx ?? 1), ev.scale * (ev.sy ?? 1) * (g?.sy ?? 1));
      target.translate(-cx, -cy);
      // A dive flies the same one picture in, under the view the other
      // renderers draw their type under.
      const focus = ev.dive && divesAt(o.anim) ? peekDiveFocus(o, W / H, cutRenderEnv(assets)) : undefined;
      if (focus) {
        const v = diveView(ev.dive!, focus, ev, o, { width: W, height: H, scale });
        target.translate(v.tx, v.ty);
        target.scale(v.s, v.s);
        target.translate(-v.fx, -v.fy);
      }
      const origin = this.origins.get(bmp);
      target.drawImage(bmp, origin?.x ?? 0, origin?.y ?? 0);
      target.restore();
      this.fx.end(ctx);
    }

    // The person back on top, its edge softened by the widest feather any
    // live behind element asks for.
    if (!alpha) return; // no person in shot: the elements stay in front
    const feather = Math.max(0, ...active.map((o) => (o.mask?.feather ?? 0) * scale));
    pctx.globalCompositeOperation = "destination-in";
    pctx.imageSmoothingEnabled = true;
    if (feather > 0 && "filter" in pctx) pctx.filter = `blur(${feather / 2}px)`;
    pctx.drawImage(alpha, 0, 0, W, H);
    pctx.filter = "none";
    pctx.globalCompositeOperation = "source-over";
    ctx.drawImage(this.person, 0, 0);
  }
}