Skip to content

Open source CapCut alternative.

TypeScript
10.1 KB
Raw Blame
"use client";

/**
 * Sticker cutouts: turn a picture into a transparent-background sticker.
 *
 * Two matting paths share one shape. People run through MediaPipe's
 * self-hosted person segmenter — free, on-device, offline (the same model
 * text-behind-speaker uses). Everything else goes to the hosted inference
 * surface: the vision model returns segmentation masks (box + base64 mask
 * PNG per subject) and the alpha is applied here. Both end in the same
 * finishing pass — crop to content, then an optional white die-cut outline —
 * which is the kit's, since shaping a matted picture needs no host.
 */

import {
  applyAlpha,
  canvasOf,
  cropToContent,
  keepLargestSubject,
  removeFlatBackdrop,
  withOutline,
} from "@donkeycut/effects-kit";
import { hostedPost } from "./hosted";
import { decodeRasterImage, rasterCanvasToDataUrl, rasterCanvasToPng } from "./raster";
import { geminiModelRoleNames } from "@/lib/inference/gemini-models";
import { visionFileset, withQuietWasmLogs } from "./mediapipe";

const PERSON_MODEL = "/mediapipe/selfie_segmenter.tflite";
/** Confidence above which a pixel counts as subject. */
const PERSON_THRESHOLD = 0.35;
type Segmenter = import("@mediapipe/tasks-vision").ImageSegmenter;

let segmenterOnce: Promise<Segmenter | null> | null = null;

/** Build the inference delegate at load time. TFLite defers it to the first
 * inference, which would otherwise land on the first playback frame that needs
 * a matte. */
const WARM_PX = 16;

function warmSegmenter(segmenter: Segmenter): void {
  const [canvas] = canvasOf(WARM_PX, WARM_PX);
  try {
    segmenter.segment(canvas).close();
  } catch {
    // A warm-up that fails is not a reason to withhold a working segmenter.
  }
}

/** The shared IMAGE-mode person segmenter, created on first use. Null when
 * the runtime assets are missing (segmentation quietly unavailable). */
export function personSegmenter(): Promise<Segmenter | null> {
  segmenterOnce ??= (async () => {
    try {
      const { ImageSegmenter } = await import("@mediapipe/tasks-vision");
      const fileset = await visionFileset();
      const segmenter = await withQuietWasmLogs(() =>
        ImageSegmenter.createFromOptions(fileset, {
          baseOptions: { modelAssetPath: PERSON_MODEL },
          runningMode: "IMAGE",
          outputCategoryMask: false,
          outputConfidenceMasks: true,
        })
      );
      warmSegmenter(segmenter);
      return segmenter;
    } catch {
      segmenterOnce = null;
      return null;
    }
  })();
  return segmenterOnce;
}

/**
 * Segment one frame and return the subject as an alpha canvas at the mask's
 * own (small) resolution — white where the person is, transparent elsewhere.
 * Scaling it over a frame with `destination-in` cuts the person out; drawing
 * it into a mask video encodes the same information as luma. Null when no
 * segmenter or no clear person. The `source` should be small (≤ ~512px) —
 * the model resamples anyway, and segmentation runs per preview frame.
 */
export function segmentSubjectAlpha(
  segmenter: Segmenter,
  source: HTMLCanvasElement
): HTMLCanvasElement | null {
  const result = segmenter.segment(source);
  try {
    // The person confidence rides the LAST mask: tasks-vision 1.x emits a
    // single foreground mask for the selfie model, and 0.10's
    // [background, person] pair ends on the person. Verified against the
    // bundled model — reading index 0 as background inverts every matte
    // under 1.x.
    const masks = result.confidenceMasks;
    const mask = masks?.[masks.length - 1];
    if (!mask) return null;
    const conf = mask.getAsFloat32Array();
    const mw = mask.width;
    const mh = mask.height;
    const [out, ctx] = canvasOf(mw, mh);
    const img = ctx.createImageData(mw, mh);
    let kept = 0;
    for (let i = 0; i < mw * mh; i++) {
      const subject = conf[i];
      const a =
        subject > PERSON_THRESHOLD
          ? Math.min(255, Math.round((subject - PERSON_THRESHOLD) * 4 * 255))
          : 0;
      img.data[i * 4] = 255;
      img.data[i * 4 + 1] = 255;
      img.data[i * 4 + 2] = 255;
      img.data[i * 4 + 3] = a;
      if (a > 128) kept++;
    }
    if (kept < mw * mh * 0.01) return null; // no meaningful person
    ctx.putImageData(img, 0, 0);
    return out;
  } finally {
    result.close();
  }
}

/** Person matting, fully on-device. Null when no person registers (callers
 * fall through to the hosted path). */
export async function personCutout(blob: Blob): Promise<HTMLCanvasElement | null> {
  const [img, segmenter] = await Promise.all([decodeRasterImage(blob), personSegmenter()]);
  if (!img || !segmenter) return null;
  const w = img.width;
  const h = img.height;
  const [canvas, ctx] = canvasOf(w, h);
  ctx.drawImage(img.source, 0, 0);
  const result = segmenter.segment(canvas);
  try {
    // Person confidence is the last mask — see segmentSubjectAlpha.
    const masks = result.confidenceMasks;
    const mask = masks?.[masks.length - 1];
    if (!mask) return null;
    const conf = mask.getAsFloat32Array();
    const mw = mask.width;
    const mh = mask.height;
    const ok = applyAlpha(ctx, w, h, (i) => {
      const x = Math.min(mw - 1, Math.round(((i % w) * mw) / w));
      const y = Math.min(mh - 1, Math.round((Math.floor(i / w) * mh) / h));
      const subject = conf[y * mw + x];
      return subject > PERSON_THRESHOLD ? Math.min(1, (subject - PERSON_THRESHOLD) * 4) : 0;
    });
    return ok ? canvas : null;
  } finally {
    result.close();
  }
}

/** Hosted matting for general subjects: the vision model returns segmentation
 * masks; the largest subject's mask becomes the alpha. */
export async function hostedCutout(blob: Blob): Promise<HTMLCanvasElement | null> {
  const img = await decodeRasterImage(blob);
  if (!img) return null;
  const w = img.width;
  const h = img.height;
  const [canvas, ctx] = canvasOf(w, h);
  ctx.drawImage(img.source, 0, 0);
  // Fit the upload to the inline budget; masks come back box-relative, so a
  // downscaled request still masks the full-size picture.
  const [small, sctx] = canvasOf(Math.min(1024, w), Math.round((Math.min(1024, w) * h) / w));
  sctx.drawImage(img.source, 0, 0, small.width, small.height);
  const dataUrl = await rasterCanvasToDataUrl(small, "image/jpeg", 0.85);
  const res = await hostedPost("/api/inference/responses", {
    donkeyProvider: "gemini",
    model: geminiModelRoleNames.chat,
    instructions:
      "You segment images. Output a JSON object {\"masks\": [...]} where each entry has \"box_2d\" ([y0, x0, y1, x1], normalized 0-1000), \"mask\" (a data: URI PNG probability mask for that box), and \"label\". Segment only the single main foreground subject.",
    response_format: { type: "json_object" },
    input: [
      {
        role: "user",
        content: [
          { type: "input_image", dataBase64: dataUrl.split(",")[1], mimeType: "image/jpeg" },
          { text: "Give the segmentation mask for the main foreground subject." },
        ],
      },
    ],
  });
  if (!res.ok) return null;
  const body = (await res.json()) as { output_text?: string };
  let masks: { box_2d?: number[]; mask?: string }[] = [];
  try {
    const parsed = JSON.parse(body.output_text ?? "") as
      | { masks?: typeof masks }
      | typeof masks;
    masks = Array.isArray(parsed) ? parsed : (parsed.masks ?? []);
  } catch {
    return null;
  }
  const best = masks
    .filter((m) => Array.isArray(m.box_2d) && m.box_2d.length === 4 && typeof m.mask === "string")
    .sort((a, b) => {
      const area = (box: number[]) => Math.max(0, box[2] - box[0]) * Math.max(0, box[3] - box[1]);
      return area(b.box_2d!) - area(a.box_2d!);
    })[0];
  if (!best) return null;
  const maskImg = await decodeRasterImage(await (await fetch(best.mask!)).blob().catch(() => new Blob()));
  if (!maskImg) return null;
  const [y0, x0, y1, x1] = best.box_2d!.map((v) => v / 1000);
  const [, mctx] = canvasOf(w, h);
  mctx.drawImage(maskImg.source, x0 * w, y0 * h, Math.max(1, (x1 - x0) * w), Math.max(1, (y1 - y0) * h));
  const maskData = mctx.getImageData(0, 0, w, h).data;
  const ok = applyAlpha(ctx, w, h, (i) => {
    const v = maskData[i * 4] / 255; // grayscale probability
    return v > 0.5 ? 1 : v > 0.25 ? (v - 0.25) * 4 : 0;
  });
  return ok ? canvas : null;
}

/** Knock out a flat studio backdrop — the shape a generated sticker arrives
 * in. Null when the picture has no such backdrop. */
export async function flatBackdropCutout(blob: Blob): Promise<HTMLCanvasElement | null> {
  const img = await decodeRasterImage(blob);
  if (!img) return null;
  const [canvas, ctx] = canvasOf(img.width, img.height);
  ctx.drawImage(img.source, 0, 0);
  return removeFlatBackdrop(canvas) ? canvas : null;
}

/**
 * The whole cutout pass, cheapest answer first: on-device person matting, then
 * the flat-backdrop knockout that generated stickers are made for, and only
 * then the hosted matte — the one that costs a round trip and credits. Ends in
 * crop-to-content and the optional die-cut outline. Null when no path finds a
 * subject, which is the caller's cue to keep the picture as it came.
 */
export async function makeStickerCutout(
  blob: Blob,
  opts: { outline?: boolean; plainBackdrop?: boolean } = {}
): Promise<Blob | null> {
  // A sticker we generated was asked for on a plain sweep, so the knockout is
  // both the cheapest answer and the one that knows the most. It goes first
  // there; a cat on a studio grey reads as a person often enough that letting
  // the segmenter answer first returns a mangled matte.
  const order = opts.plainBackdrop
    ? [flatBackdropCutout, personCutout, hostedCutout]
    : [personCutout, flatBackdropCutout, hostedCutout];
  let cut: HTMLCanvasElement | null = null;
  for (const attempt of order) {
    cut = await attempt(blob);
    if (cut) break;
  }
  if (!cut) return null;
  // A sticker is one thing. Strokes and specks the generator scattered around
  // it survive any matte — they are not backdrop — so the body is picked out
  // before the crop, which would otherwise frame the strays too.
  keepLargestSubject(cut);
  const cropped = cropToContent(cut);
  return rasterCanvasToPng(opts.outline === false ? cropped : withOutline(cropped));
}