Skip to content

Open source CapCut alternative.

TypeScript
8.6 KB
Raw Blame
"use client";

import { removalActive, retimeOf } from "@donkeycut/effects-kit";

/**
 * The person-matte video for the ffmpeg export path. The page — the only
 * place that can run segmentation — composes the video layers over the
 * subject window at a small size, segments each frame, and encodes the
 * person mask as a plain grayscale H.264 clip (white = person; no alpha
 * encoding, which browsers can't do reliably). The server scales it up, pads
 * it to the timeline, and every subject-masked consumer — behind elements,
 * front subject elements, subject-masked clips — trims by it. The matte
 * source leaves subject-masked clips out of its composite, so a trimmed clip
 * never reads its own pixels back.
 */

import {
  BufferTarget,
  CanvasSource,
  getFirstEncodableVideoCodec,
  Mp4OutputFormat,
  Output,
  Quality,
} from "mediabunny";
import { personSegmenter, segmentSubjectAlpha } from "./cutout";
import { FrameCompositor, MISSING_FRAME } from "./composite";
import { ensureClipLuts, sourceLookup } from "./lutBuild";
import { overlayPlan, trackZeroPlan } from "./framePlan";
import { ClipReader, renderFile, settleRenderColor, WORKING_VIDEO_CODECS } from "./exportRender";
import type { ExportDoc } from "./renderSnapshot";
import { createRasterCanvas } from "./raster";
import { getClipSpans } from "./store";
import { clipCovers, frameOf, isEffectOverlay, isTextOverlay, rectOf, subjectMasked, type ClipSpan, type MediaAsset, type VideoClip } from "./types";
import { liveReader } from "./liveReader";
import { cardMatteKey } from "./cameraCard";
import { matteStage } from "./matteAlpha";

/** Encoded mask rate — the server's fps filter duplicates frames up to the
 * output rate, and a person moves little in 1/15s. */
export const MASK_FPS = 15;
/** Mask frame short side; alphamerge scales it to the output. */
const MASK_SHORT = 480;

/** Timeline second inside a span mapped to its source second. */
const sourceTimeAt = (sp: ClipSpan, t: number) =>
  Math.max(sp.clip.in, Math.min(sp.clip.out, retimeOf(sp.clip).srcAt(Math.max(0, t - sp.start))));

/** A clip whose picture trims by the person matte (either direction). */
const subjectClip = (c: { mask?: { kind: string } }) => c.mask?.kind === "subject";

/**
 * Render the person-mask video covering every subject-masked consumer's
 * window (one union range): behind and front subject elements, and
 * subject-masked video clips. Null when nothing reads the matte, no person
 * segmenter is available, or the range is empty — the export simply proceeds
 * without the effect, matching the preview's degrade.
 */
export async function renderSubjectMask(
  doc: ExportDoc,
  duration: number
): Promise<{ blob: Blob; from: number } | null> {
  const windows: { start: number; end: number }[] = doc.overlays
    .filter(
      (o) =>
        subjectMasked(o) &&
        !o.hidden &&
        !isEffectOverlay(o) &&
        (!isTextOverlay(o) || !!o.text.trim()) &&
        o.start < duration
    )
    .map((o) => ({ start: o.start, end: o.end }));
  for (const track of [...new Set(doc.clips.map((c) => c.track))]) {
    for (const sp of getClipSpans(doc.clips, doc.assets, track)) {
      if (!subjectClip(sp.clip) || sp.clip.hidden || sp.start >= duration) continue;
      windows.push({ start: sp.start, end: sp.start + sp.len });
    }
  }
  if (windows.length === 0) return null;
  const from = Math.max(0, Math.min(...windows.map((w) => w.start)));
  const to = Math.min(duration, Math.max(...windows.map((w) => w.end)));
  if (to - from < 0.05) return null;
  const segmenter = await personSegmenter();
  if (!segmenter) return null;

  const frame = frameOf(doc.aspect);
  const scale = MASK_SHORT / Math.min(frame.w, frame.h);
  const even = (n: number) => 2 * Math.round((n * scale) / 2);
  const W = even(frame.w);
  const H = even(frame.h);
  const codec = await getFirstEncodableVideoCodec(WORKING_VIDEO_CODECS, { width: W, height: H });
  if (!codec) return null;

  // A small compositor: the video layers only, at mask resolution.
  const compose = createRasterCanvas(W, H);
  const comp = new FrameCompositor(compose);
  comp.colorMode = "exact";
  // The matte composites what the readers below read: every master, through
  // the recipe of that read.
  comp.sourceProvider = sourceLookup(doc.assets, renderFile);
  await settleRenderColor(doc.assets);
  await ensureClipLuts(doc.clips.map((c) => comp.recipeFor(c)));
  const mask = createRasterCanvas(W, H);
  const mctx = mask.getContext("2d") as CanvasRenderingContext2D;

  const spans = getClipSpans(doc.clips, doc.assets);
  const overlayTracks = [...new Set(doc.clips.filter((c) => c.track !== 0).map((c) => c.track))];
  const byTrack = new Map(overlayTracks.map((track) => [track, getClipSpans(doc.clips, doc.assets, track)]));
  const spanOfClip = new Map<string, ClipSpan>();
  for (const list of byTrack.values()) for (const sp of list) spanOfClip.set(sp.clip.id, sp);
  const readers = new Map<string, ClipReader>();
  const readerFor = (asset: MediaAsset) => {
    let r = readers.get(asset.id);
    if (!r) readers.set(asset.id, (r = liveReader(asset)));
    return r;
  };
  // Baked mattes ride along: a cutout keys its clip and a card pops its head
  // out above the card, so the segmenter sees the picture the preview draws.
  // Each matte is staged once per mask frame, before its clip draws.
  const mattes = new Map<string, ReturnType<typeof matteStage>>();
  const stageFor = (key: string, m: Parameters<typeof matteStage>[0]) => {
    if (!mattes.has(key)) mattes.set(key, matteStage(m, doc.assets));
    return mattes.get(key) ?? null;
  };
  const stageMattes = async (clip: VideoClip, srcT: number) => {
    if (removalActive(clip.removal)) await stageFor(clip.id, clip.removal?.matte)?.stage(srcT);
    if (clip.card?.popOut) await stageFor(cardMatteKey(clip.id), clip.card.matte)?.stage(srcT);
  };
  comp.removalMatteProvider = (clip) => mattes.get(clip.id)?.get() ?? null;
  comp.cardMatteProvider = (clip) => mattes.get(cardMatteKey(clip.id))?.get() ?? null;

  const output = new Output({ format: new Mp4OutputFormat(), target: new BufferTarget() });
  const video = new CanvasSource(mask, { codec, quality: new Quality({ bitrate: 1_000_000 }) });
  const frames = Math.max(1, Math.ceil((to - from) * MASK_FPS));
  output.addVideoTrack(video, { frameRate: MASK_FPS, maximumPacketCount: frames + 8 });

  try {
    await output.start();
    for (let i = 0; i < frames; i++) {
      const t = Math.min(to, from + i / MASK_FPS);
      comp.clear();
      // Subject-masked clips stay out of the matte's composite (their shape
      // masks still apply through the compositor): the matte drives their
      // trim, so reading their own pixels back would feed the mask itself.
      const master = spans.find((sp) => t >= sp.start && t < sp.start + sp.len);
      if (master && !master.clip.hidden && !subjectClip(master.clip)) {
        const plan = trackZeroPlan(master, spans, t);
        if (plan.backdrop && !subjectClip(plan.backdrop.span.clip)) {
          const f = await readerFor(plan.backdrop.span.asset).frameAt(plan.backdrop.at);
          await stageMattes(plan.backdrop.span.clip, plan.backdrop.at);
          comp.drawLayer(f, plan.backdrop.span.clip, false, 1, t);
        }
        const f = await readerFor(master.asset).frameAt(sourceTimeAt(master, t));
        await stageMattes(master.clip, sourceTimeAt(master, t));
        comp.drawLayer(f ?? MISSING_FRAME, master.clip, false, 1, t);
      }
      for (const layer of overlayPlan(overlayTracks, (track) => byTrack.get(track) ?? [], t)) {
        if (subjectClip(layer.clip)) continue;
        const span = spanOfClip.get(layer.clip.id);
        if (!span) continue;
        const f = await readerFor(layer.asset).frameAt(sourceTimeAt(span, t));
        await stageMattes(layer.clip, sourceTimeAt(span, t));
        comp.drawIntoRect(f, rectOf(layer.clip), clipCovers(layer.clip), layer.alpha, t, layer.zoom, layer.clip);
      }
      // Luma mask: black frame, the subject's alpha silhouette drawn white.
      mctx.globalCompositeOperation = "source-over";
      mctx.fillStyle = "#000000";
      mctx.fillRect(0, 0, W, H);
      const alpha = segmentSubjectAlpha(segmenter, compose as HTMLCanvasElement);
      if (alpha) mctx.drawImage(alpha, 0, 0, W, H);
      await video.add(i / MASK_FPS, 1 / MASK_FPS);
    }
    await output.finalize();
    const buffer = (output.target as BufferTarget).buffer;
    if (!buffer) return null;
    return { blob: new Blob([buffer], { type: "video/mp4" }), from };
  } catch {
    await output.cancel().catch(() => {});
    return null;
  } finally {
    for (const r of readers.values()) r.dispose();
    for (const m of mattes.values()) m?.dispose();
  }
}