Open source CapCut alternative.
TypeScript
fix(site): person segmentation reads the model's foreground mask [rebuild]
David Le79ced6c62c31
diff --git a/site/packages/effects-kit/src/removal.ts b/site/packages/effects-kit/src/removal.tsindex 4666d099..7fcfe1cb 100644--- a/site/packages/effects-kit/src/removal.ts+++ b/site/packages/effects-kit/src/removal.ts@@ -78,8 +78,17 @@ export interface ClipRemoval {* covering the clip's trimmed source range, white keeps the pixel. `in` is* the source second the matte's own zero maps to, so a later retrim keeps* reading the right frames; the fingerprint records what the matte was- * baked from, so edits re-bake. */- matte?: { assetId: string; fingerprint: string; quality: "local" | "hq"; in: number };+ * baked from, so edits re-bake. `bake` is the local bake format that+ * produced a "local" matte — a local matte from another format re-bakes+ * (free), while hq mattes never read it, so a bake-format fix never+ * re-bills a hosted pass. */+ matte?: {+ assetId: string;+ fingerprint: string;+ quality: "local" | "hq";+ in: number;+ bake?: number;+ };/** Custom-mode selection. */seeds?: RemovalSeeds;/** The bake was explicitly started — the panel's Apply button or the chat@@ -130,6 +139,11 @@ export function removalFingerprint(return hash.toString(36);}+/** The local person-matte bake format. Bumped when the bake's output changes+ * meaning — format 2 fixed mattes baked with the segmenter's mask direction+ * read backwards — so stored local mattes from older formats re-bake. */+export const LOCAL_MATTE_BAKE = 2;+/** The baked-matte contract every producer and reader shares: encoded matte* rate (coverage moves little in 1/15s, and every consumer's fps filter or* nearest-frame read duplicates up to the output rate), the frame's shortdiff --git a/site/src/cut/lib/cutout.ts b/site/src/cut/lib/cutout.tsindex 625520e6..1b8bee96 100644--- a/site/src/cut/lib/cutout.ts+++ b/site/src/cut/lib/cutout.ts@@ -117,7 +117,13 @@ export function segmentSubjectAlpha(): HTMLCanvasElement | null {const result = segmenter.segment(source);try {- const mask = result.confidenceMasks?.[0];+ // The person confidence rides the LAST mask: tasks-vision 1.x emits a+ // single foreground mask for the selfie model, and 0.10's+ // [background, person] pair ends on the person. Verified against the+ // bundled model — reading index 0 as background inverts every matte+ // under 1.x.+ const masks = result.confidenceMasks;+ const mask = masks?.[masks.length - 1];if (!mask) return null;const conf = mask.getAsFloat32Array();const mw = mask.width;@@ -126,8 +132,7 @@ export function segmentSubjectAlpha(const img = ctx.createImageData(mw, mh);let kept = 0;for (let i = 0; i < mw * mh; i++) {- // The selfie model's first confidence mask is background; subject = 1-bg.- const subject = 1 - conf[i];+ const subject = conf[i];const a =subject > PERSON_THRESHOLD? Math.min(255, Math.round((subject - PERSON_THRESHOLD) * 4 * 255))@@ -157,16 +162,17 @@ export async function personCutout(blob: Blob): Promise<HTMLCanvasElement | nullctx.drawImage(img.source, 0, 0);const result = segmenter.segment(canvas);try {- const mask = result.confidenceMasks?.[0];+ // Person confidence is the last mask — see segmentSubjectAlpha.+ const masks = result.confidenceMasks;+ const mask = masks?.[masks.length - 1];if (!mask) return null;const conf = mask.getAsFloat32Array();const mw = mask.width;const mh = mask.height;- // The selfie model's first confidence mask is background; subject = 1-bg.const ok = applyAlpha(ctx, w, h, (i) => {const x = Math.min(mw - 1, Math.round(((i % w) * mw) / w));const y = Math.min(mh - 1, Math.round((Math.floor(i / w) * mh) / h));- const subject = 1 - conf[y * mw + x];+ const subject = conf[y * mw + x];return subject > PERSON_THRESHOLD ? Math.min(1, (subject - PERSON_THRESHOLD) * 4) : 0;});return ok ? canvas : null;diff --git a/site/src/cut/lib/removal/bakeJobs.ts b/site/src/cut/lib/removal/bakeJobs.tsindex 234606e2..450e69ef 100644--- a/site/src/cut/lib/removal/bakeJobs.ts+++ b/site/src/cut/lib/removal/bakeJobs.ts@@ -17,6 +17,7 @@*/import { create } from "zustand";+import { LOCAL_MATTE_BAKE } from "@donkeycut/effects-kit";import { importFileToProject } from "../media";import { useEditor } from "../store";import { removalFingerprint, type ClipRemoval, type VideoClip } from "../types";@@ -148,6 +149,13 @@ function owedFingerprint(clip: VideoClip | undefined): string | null {return removalFingerprint(clip.assetId, r);}+/** Whether a stored matte's pixels still mean what the renderer reads: hq+ * mattes always do, and a local matte must carry the current bake format —+ * an older one re-bakes for free. */+function matteFresh(matte: NonNullable<ClipRemoval["matte"]>): boolean {+ return matte.quality !== "local" || matte.bake === LOCAL_MATTE_BAKE;+}+/** Whether the stored matte's baked range still spans the clip's trims —* a matte wider than the clip plays is fine; one the trim reached past* wants a re-bake. Stills always cover. */@@ -174,7 +182,7 @@ function twinMatte(if (c.id === clipId) continue;const m = c.removal?.matte;if (!m || m.fingerprint !== fp || !s.assets.some((a) => a.id === m.assetId)) continue;- if (!matteCovers(m, clip)) continue;+ if (!matteFresh(m) || !matteCovers(m, clip)) continue;if (m.quality === "hq") return m;found ??= m;}@@ -225,7 +233,8 @@ export function ensureMatteBake(clipId: string, opts: { whileBrushing?: boolean// Settled: the stored matte matches the clip and its asset is still around// (an undo can revive a matte pointer whose asset was collected).const matte = r.matte;- const stored = matte?.fingerprint === fp && matteCovers(matte, clip) ? matte : null;+ const stored =+ matte?.fingerprint === fp && matteFresh(matte) && matteCovers(matte, clip) ? matte : null;// A split or a paste leaves two clips owing the same selection. A twin// matte another clip already carries is adopted whole — same asset, one// bake, one bill — as long as its baked range spans this clip's trims. A@@ -383,7 +392,13 @@ async function runBakeNow(const removal: ClipRemoval = {...latest.removal!,- matte: { assetId: matteAsset.id, fingerprint: fp, quality, in: baked.in },+ matte: {+ assetId: matteAsset.id,+ fingerprint: fp,+ quality,+ in: baked.in,+ ...(quality === "local" ? { bake: LOCAL_MATTE_BAKE } : {}),+ },};editor.updateClipTransient(clipId, { removal });// The matte this replaced may still be another clip's (a split, a paste);