Skip to content

Open source CapCut alternative.

TypeScript

fix(site): person segmentation reads the model's foreground mask [rebuild]

David Le79ced6c62c31
diff --git a/site/packages/effects-kit/src/removal.ts b/site/packages/effects-kit/src/removal.ts
index 4666d099..7fcfe1cb 100644
--- a/site/packages/effects-kit/src/removal.ts
+++ b/site/packages/effects-kit/src/removal.ts
@@ -78,8 +78,17 @@ export interface ClipRemoval {
* covering the clip's trimmed source range, white keeps the pixel. `in` is
* the source second the matte's own zero maps to, so a later retrim keeps
* reading the right frames; the fingerprint records what the matte was
- * baked from, so edits re-bake. */
- matte?: { assetId: string; fingerprint: string; quality: "local" | "hq"; in: number };
+ * baked from, so edits re-bake. `bake` is the local bake format that
+ * produced a "local" matte — a local matte from another format re-bakes
+ * (free), while hq mattes never read it, so a bake-format fix never
+ * re-bills a hosted pass. */
+ matte?: {
+ assetId: string;
+ fingerprint: string;
+ quality: "local" | "hq";
+ in: number;
+ bake?: number;
+ };
/** Custom-mode selection. */
seeds?: RemovalSeeds;
/** The bake was explicitly started — the panel's Apply button or the chat
@@ -130,6 +139,11 @@ export function removalFingerprint(
return hash.toString(36);
}
+/** The local person-matte bake format. Bumped when the bake's output changes
+ * meaning — format 2 fixed mattes baked with the segmenter's mask direction
+ * read backwards — so stored local mattes from older formats re-bake. */
+export const LOCAL_MATTE_BAKE = 2;
+
/** The baked-matte contract every producer and reader shares: encoded matte
* rate (coverage moves little in 1/15s, and every consumer's fps filter or
* nearest-frame read duplicates up to the output rate), the frame's short
diff --git a/site/src/cut/lib/cutout.ts b/site/src/cut/lib/cutout.ts
index 625520e6..1b8bee96 100644
--- a/site/src/cut/lib/cutout.ts
+++ b/site/src/cut/lib/cutout.ts
@@ -117,7 +117,13 @@ export function segmentSubjectAlpha(
): HTMLCanvasElement | null {
const result = segmenter.segment(source);
try {
- const mask = result.confidenceMasks?.[0];
+ // The person confidence rides the LAST mask: tasks-vision 1.x emits a
+ // single foreground mask for the selfie model, and 0.10's
+ // [background, person] pair ends on the person. Verified against the
+ // bundled model — reading index 0 as background inverts every matte
+ // under 1.x.
+ const masks = result.confidenceMasks;
+ const mask = masks?.[masks.length - 1];
if (!mask) return null;
const conf = mask.getAsFloat32Array();
const mw = mask.width;
@@ -126,8 +132,7 @@ export function segmentSubjectAlpha(
const img = ctx.createImageData(mw, mh);
let kept = 0;
for (let i = 0; i < mw * mh; i++) {
- // The selfie model's first confidence mask is background; subject = 1-bg.
- const subject = 1 - conf[i];
+ const subject = conf[i];
const a =
subject > PERSON_THRESHOLD
? Math.min(255, Math.round((subject - PERSON_THRESHOLD) * 4 * 255))
@@ -157,16 +162,17 @@ export async function personCutout(blob: Blob): Promise<HTMLCanvasElement | null
ctx.drawImage(img.source, 0, 0);
const result = segmenter.segment(canvas);
try {
- const mask = result.confidenceMasks?.[0];
+ // Person confidence is the last mask — see segmentSubjectAlpha.
+ const masks = result.confidenceMasks;
+ const mask = masks?.[masks.length - 1];
if (!mask) return null;
const conf = mask.getAsFloat32Array();
const mw = mask.width;
const mh = mask.height;
- // The selfie model's first confidence mask is background; subject = 1-bg.
const ok = applyAlpha(ctx, w, h, (i) => {
const x = Math.min(mw - 1, Math.round(((i % w) * mw) / w));
const y = Math.min(mh - 1, Math.round((Math.floor(i / w) * mh) / h));
- const subject = 1 - conf[y * mw + x];
+ const subject = conf[y * mw + x];
return subject > PERSON_THRESHOLD ? Math.min(1, (subject - PERSON_THRESHOLD) * 4) : 0;
});
return ok ? canvas : null;
diff --git a/site/src/cut/lib/removal/bakeJobs.ts b/site/src/cut/lib/removal/bakeJobs.ts
index 234606e2..450e69ef 100644
--- a/site/src/cut/lib/removal/bakeJobs.ts
+++ b/site/src/cut/lib/removal/bakeJobs.ts
@@ -17,6 +17,7 @@
*/
import { create } from "zustand";
+import { LOCAL_MATTE_BAKE } from "@donkeycut/effects-kit";
import { importFileToProject } from "../media";
import { useEditor } from "../store";
import { removalFingerprint, type ClipRemoval, type VideoClip } from "../types";
@@ -148,6 +149,13 @@ function owedFingerprint(clip: VideoClip | undefined): string | null {
return removalFingerprint(clip.assetId, r);
}
+/** Whether a stored matte's pixels still mean what the renderer reads: hq
+ * mattes always do, and a local matte must carry the current bake format —
+ * an older one re-bakes for free. */
+function matteFresh(matte: NonNullable<ClipRemoval["matte"]>): boolean {
+ return matte.quality !== "local" || matte.bake === LOCAL_MATTE_BAKE;
+}
+
/** Whether the stored matte's baked range still spans the clip's trims —
* a matte wider than the clip plays is fine; one the trim reached past
* wants a re-bake. Stills always cover. */
@@ -174,7 +182,7 @@ function twinMatte(
if (c.id === clipId) continue;
const m = c.removal?.matte;
if (!m || m.fingerprint !== fp || !s.assets.some((a) => a.id === m.assetId)) continue;
- if (!matteCovers(m, clip)) continue;
+ if (!matteFresh(m) || !matteCovers(m, clip)) continue;
if (m.quality === "hq") return m;
found ??= m;
}
@@ -225,7 +233,8 @@ export function ensureMatteBake(clipId: string, opts: { whileBrushing?: boolean
// Settled: the stored matte matches the clip and its asset is still around
// (an undo can revive a matte pointer whose asset was collected).
const matte = r.matte;
- const stored = matte?.fingerprint === fp && matteCovers(matte, clip) ? matte : null;
+ const stored =
+ matte?.fingerprint === fp && matteFresh(matte) && matteCovers(matte, clip) ? matte : null;
// A split or a paste leaves two clips owing the same selection. A twin
// matte another clip already carries is adopted whole — same asset, one
// bake, one bill — as long as its baked range spans this clip's trims. A
@@ -383,7 +392,13 @@ async function runBakeNow(
const removal: ClipRemoval = {
...latest.removal!,
- matte: { assetId: matteAsset.id, fingerprint: fp, quality, in: baked.in },
+ matte: {
+ assetId: matteAsset.id,
+ fingerprint: fp,
+ quality,
+ in: baked.in,
+ ...(quality === "local" ? { bake: LOCAL_MATTE_BAKE } : {}),
+ },
};
editor.updateClipTransient(clipId, { removal });
// The matte this replaced may still be another clip's (a split, a paste);