Open source CapCut alternative.
TypeScript
"use client";
/**
* Sticker cutouts: turn a picture into a transparent-background sticker.
*
* Two matting paths share one shape. People run through MediaPipe's
* self-hosted person segmenter — free, on-device, offline (the same model
* text-behind-speaker uses). Everything else goes to the hosted inference
* surface: the vision model returns segmentation masks (box + base64 mask
* PNG per subject) and the alpha is applied here. Both end in the same
* finishing pass — crop to content, then an optional white die-cut outline —
* which is the kit's, since shaping a matted picture needs no host.
*/
import {
applyAlpha,
canvasOf,
cropToContent,
keepLargestSubject,
removeFlatBackdrop,
withOutline,
} from "@donkeycut/effects-kit";
import { hostedPost } from "./hosted";
import { decodeRasterImage, rasterCanvasToDataUrl, rasterCanvasToPng } from "./raster";
import { geminiModelRoleNames } from "@/lib/inference/gemini-models";
import { visionFileset, withQuietWasmLogs } from "./mediapipe";
const PERSON_MODEL = "/mediapipe/selfie_segmenter.tflite";
/** Confidence above which a pixel counts as subject. */
const PERSON_THRESHOLD = 0.35;
type Segmenter = import("@mediapipe/tasks-vision").ImageSegmenter;
let segmenterOnce: Promise<Segmenter | null> | null = null;
/** Build the inference delegate at load time. TFLite defers it to the first
* inference, which would otherwise land on the first playback frame that needs
* a matte. */
const WARM_PX = 16;
function warmSegmenter(segmenter: Segmenter): void {
const [canvas] = canvasOf(WARM_PX, WARM_PX);
try {
segmenter.segment(canvas).close();
} catch {
// A warm-up that fails is not a reason to withhold a working segmenter.
}
}
/** The shared IMAGE-mode person segmenter, created on first use. Null when
* the runtime assets are missing (segmentation quietly unavailable). */
export function personSegmenter(): Promise<Segmenter | null> {
segmenterOnce ??= (async () => {
try {
const { ImageSegmenter } = await import("@mediapipe/tasks-vision");
const fileset = await visionFileset();
const segmenter = await withQuietWasmLogs(() =>
ImageSegmenter.createFromOptions(fileset, {
baseOptions: { modelAssetPath: PERSON_MODEL },
runningMode: "IMAGE",
outputCategoryMask: false,
outputConfidenceMasks: true,
})
);
warmSegmenter(segmenter);
return segmenter;
} catch {
segmenterOnce = null;
return null;
}
})();
return segmenterOnce;
}
/**
* Segment one frame and return the subject as an alpha canvas at the mask's
* own (small) resolution — white where the person is, transparent elsewhere.
* Scaling it over a frame with `destination-in` cuts the person out; drawing
* it into a mask video encodes the same information as luma. Null when no
* segmenter or no clear person. The `source` should be small (≤ ~512px) —
* the model resamples anyway, and segmentation runs per preview frame.
*/
export function segmentSubjectAlpha(
segmenter: Segmenter,
source: HTMLCanvasElement
): HTMLCanvasElement | null {
const result = segmenter.segment(source);
try {
// The person confidence rides the LAST mask: tasks-vision 1.x emits a
// single foreground mask for the selfie model, and 0.10's
// [background, person] pair ends on the person. Verified against the
// bundled model — reading index 0 as background inverts every matte
// under 1.x.
const masks = result.confidenceMasks;
const mask = masks?.[masks.length - 1];
if (!mask) return null;
const conf = mask.getAsFloat32Array();
const mw = mask.width;
const mh = mask.height;
const [out, ctx] = canvasOf(mw, mh);
const img = ctx.createImageData(mw, mh);
let kept = 0;
for (let i = 0; i < mw * mh; i++) {
const subject = conf[i];
const a =
subject > PERSON_THRESHOLD
? Math.min(255, Math.round((subject - PERSON_THRESHOLD) * 4 * 255))
: 0;
img.data[i * 4] = 255;
img.data[i * 4 + 1] = 255;
img.data[i * 4 + 2] = 255;
img.data[i * 4 + 3] = a;
if (a > 128) kept++;
}
if (kept < mw * mh * 0.01) return null; // no meaningful person
ctx.putImageData(img, 0, 0);
return out;
} finally {
result.close();
}
}
/** Person matting, fully on-device. Null when no person registers (callers
* fall through to the hosted path). */
export async function personCutout(blob: Blob): Promise<HTMLCanvasElement | null> {
const [img, segmenter] = await Promise.all([decodeRasterImage(blob), personSegmenter()]);
if (!img || !segmenter) return null;
const w = img.width;
const h = img.height;
const [canvas, ctx] = canvasOf(w, h);
ctx.drawImage(img.source, 0, 0);
const result = segmenter.segment(canvas);
try {
// Person confidence is the last mask — see segmentSubjectAlpha.
const masks = result.confidenceMasks;
const mask = masks?.[masks.length - 1];
if (!mask) return null;
const conf = mask.getAsFloat32Array();
const mw = mask.width;
const mh = mask.height;
const ok = applyAlpha(ctx, w, h, (i) => {
const x = Math.min(mw - 1, Math.round(((i % w) * mw) / w));
const y = Math.min(mh - 1, Math.round((Math.floor(i / w) * mh) / h));
const subject = conf[y * mw + x];
return subject > PERSON_THRESHOLD ? Math.min(1, (subject - PERSON_THRESHOLD) * 4) : 0;
});
return ok ? canvas : null;
} finally {
result.close();
}
}
/** Hosted matting for general subjects: the vision model returns segmentation
* masks; the largest subject's mask becomes the alpha. */
export async function hostedCutout(blob: Blob): Promise<HTMLCanvasElement | null> {
const img = await decodeRasterImage(blob);
if (!img) return null;
const w = img.width;
const h = img.height;
const [canvas, ctx] = canvasOf(w, h);
ctx.drawImage(img.source, 0, 0);
// Fit the upload to the inline budget; masks come back box-relative, so a
// downscaled request still masks the full-size picture.
const [small, sctx] = canvasOf(Math.min(1024, w), Math.round((Math.min(1024, w) * h) / w));
sctx.drawImage(img.source, 0, 0, small.width, small.height);
const dataUrl = await rasterCanvasToDataUrl(small, "image/jpeg", 0.85);
const res = await hostedPost("/api/inference/responses", {
donkeyProvider: "gemini",
model: geminiModelRoleNames.chat,
instructions:
"You segment images. Output a JSON object {\"masks\": [...]} where each entry has \"box_2d\" ([y0, x0, y1, x1], normalized 0-1000), \"mask\" (a data: URI PNG probability mask for that box), and \"label\". Segment only the single main foreground subject.",
response_format: { type: "json_object" },
input: [
{
role: "user",
content: [
{ type: "input_image", dataBase64: dataUrl.split(",")[1], mimeType: "image/jpeg" },
{ text: "Give the segmentation mask for the main foreground subject." },
],
},
],
});
if (!res.ok) return null;
const body = (await res.json()) as { output_text?: string };
let masks: { box_2d?: number[]; mask?: string }[] = [];
try {
const parsed = JSON.parse(body.output_text ?? "") as
| { masks?: typeof masks }
| typeof masks;
masks = Array.isArray(parsed) ? parsed : (parsed.masks ?? []);
} catch {
return null;
}
const best = masks
.filter((m) => Array.isArray(m.box_2d) && m.box_2d.length === 4 && typeof m.mask === "string")
.sort((a, b) => {
const area = (box: number[]) => Math.max(0, box[2] - box[0]) * Math.max(0, box[3] - box[1]);
return area(b.box_2d!) - area(a.box_2d!);
})[0];
if (!best) return null;
const maskImg = await decodeRasterImage(await (await fetch(best.mask!)).blob().catch(() => new Blob()));
if (!maskImg) return null;
const [y0, x0, y1, x1] = best.box_2d!.map((v) => v / 1000);
const [, mctx] = canvasOf(w, h);
mctx.drawImage(maskImg.source, x0 * w, y0 * h, Math.max(1, (x1 - x0) * w), Math.max(1, (y1 - y0) * h));
const maskData = mctx.getImageData(0, 0, w, h).data;
const ok = applyAlpha(ctx, w, h, (i) => {
const v = maskData[i * 4] / 255; // grayscale probability
return v > 0.5 ? 1 : v > 0.25 ? (v - 0.25) * 4 : 0;
});
return ok ? canvas : null;
}
/** Knock out a flat studio backdrop — the shape a generated sticker arrives
* in. Null when the picture has no such backdrop. */
export async function flatBackdropCutout(blob: Blob): Promise<HTMLCanvasElement | null> {
const img = await decodeRasterImage(blob);
if (!img) return null;
const [canvas, ctx] = canvasOf(img.width, img.height);
ctx.drawImage(img.source, 0, 0);
return removeFlatBackdrop(canvas) ? canvas : null;
}
/**
* The whole cutout pass, cheapest answer first: on-device person matting, then
* the flat-backdrop knockout that generated stickers are made for, and only
* then the hosted matte — the one that costs a round trip and credits. Ends in
* crop-to-content and the optional die-cut outline. Null when no path finds a
* subject, which is the caller's cue to keep the picture as it came.
*/
export async function makeStickerCutout(
blob: Blob,
opts: { outline?: boolean; plainBackdrop?: boolean } = {}
): Promise<Blob | null> {
// A sticker we generated was asked for on a plain sweep, so the knockout is
// both the cheapest answer and the one that knows the most. It goes first
// there; a cat on a studio grey reads as a person often enough that letting
// the segmenter answer first returns a mangled matte.
const order = opts.plainBackdrop
? [flatBackdropCutout, personCutout, hostedCutout]
: [personCutout, flatBackdropCutout, hostedCutout];
let cut: HTMLCanvasElement | null = null;
for (const attempt of order) {
cut = await attempt(blob);
if (cut) break;
}
if (!cut) return null;
// A sticker is one thing. Strokes and specks the generator scattered around
// it survive any matte — they are not backdrop — so the body is picked out
// before the crop, which would otherwise frame the strays too.
keepLargestSubject(cut);
const cropped = cropToContent(cut);
return rasterCanvasToPng(opts.outline === false ? cropped : withOutline(cropped));
}