Skip to content

Open source CapCut alternative.

TypeScript

feat(site): the auto quality matte bakes on the hosted removal model

David Le6d5a477c8d96
diff --git a/site/src/cut/lib/removal/hostedBake.ts b/site/src/cut/lib/removal/hostedBake.ts
index f1f3bfac..9c645de5 100644
--- a/site/src/cut/lib/removal/hostedBake.ts
+++ b/site/src/cut/lib/removal/hostedBake.ts
@@ -2,12 +2,13 @@
/**
* The quality matte bake: the clip's trimmed segment goes to the hosted
- * tracker (credits), which propagates the subject across every frame; the
- * returned mask video is refined here against the source pixels with the
- * kit's guided filter and encoded as the final matte asset. Auto mode seeds
- * the tracker from the on-device person matte; custom mode seeds it from the
- * brush's stored prompts and paint. Driven by the page or the cloud worker —
- * the local engine only ever consumes the stored matte.
+ * matting service (credits) and the returned mask video is refined here
+ * against the source pixels with the kit's guided filter and encoded as the
+ * final matte asset. One model per job: auto mode sends the segment to the
+ * background-removal model, which masks the foreground subject on its own;
+ * custom mode sends it to the promptable tracker, seeded by the described
+ * subject and the brush's stored prompts and paint. Driven by the page or
+ * the cloud worker — the local engine only ever consumes the stored matte.
*
* A long clip tracks in parts: the span splits into pieces of at most
* `MATTE_MAX_S`, each rendered, uploaded, and tracked on its own upload
@@ -33,6 +34,7 @@ import { NO_CREDITS_MESSAGE } from "../credits";
import { ClipReader } from "../exportRender";
import { hostedPost } from "../hosted";
import { storeHostedBlob } from "../hostedBlobs";
+import { falMatteModels } from "@/lib/inference/matte-models";
import { createRasterCanvas, decodeRasterImageUrl, type RasterSurface } from "../raster";
import type { MediaAsset, RemovalSeeds, VideoClip } from "../types";
import { liveReader } from "../liveReader";
@@ -450,8 +452,10 @@ export async function hostedBakeMatte(
const nParts = Math.max(1, Math.ceil(totalFrames / maxFrames));
const perPart = Math.ceil(totalFrames / nParts);
- const painted = r.mode === "custom" && !!(r.seeds?.prompts.length || r.seeds?.paint?.length);
- const concept = r.mode === "custom" ? (r.subject ?? "").trim() : "person";
+ const custom = r.mode === "custom";
+ const painted = custom && !!(r.seeds?.prompts.length || r.seeds?.paint?.length);
+ const concept = custom ? (r.subject ?? "").trim() : "";
+ const model = custom ? falMatteModels.segmenter : falMatteModels.removal;
const gens = [...ticketParts(opts.resume, nParts, from, to)];
const ticket = (upto: number): HostedBakeTicket => ({
@@ -478,12 +482,13 @@ export async function hostedBakeMatte(
const segment = await renderSegment(srcReader, still, partFrom, partTo, frames, opts.signal, (f) =>
frac(0.02 + f * 0.13)
);
- // The words and the strokes ride together: auto mode and a described
- // subject send a text prompt, painted seeds send point prompts, and a
- // custom removal with both sends both — the segmenter's points refine
- // what the words detected. A painted part with no strokes of its own
- // takes the handoff points from the previous part's final mask, so the
- // picked selection crosses the seam. The route picks the segmenter.
+ // Custom mode's words and strokes ride together: a described subject
+ // sends a text prompt, painted seeds send point prompts, and a
+ // removal with both sends both — the segmenter's points refine what
+ // the words detected. A painted part with no strokes of its own
+ // takes the handoff points from the previous part's final mask, so
+ // the picked selection crosses the seam. Auto mode sends the segment
+ // alone — the removal model finds the subject itself.
let points = painted ? await seedPoints(r.seeds!, segment, segment.from) : [];
if (painted && points.length === 0 && carry.length > 0) {
points = carry.map((c) => ({
@@ -494,7 +499,7 @@ export async function hostedBakeMatte(
object: "carry",
}));
}
- if (!concept && points.length === 0) {
+ if (custom && !concept && points.length === 0) {
throw p === 0
? new Error("No selection to track.")
: new MatteTrackFailed("The selection was lost at a part boundary.");
@@ -511,7 +516,10 @@ export async function hostedBakeMatte(
const submit = await hostedPost("/api/inference/assets", {
kind: "matte",
- prompt: "Track the selected subject through the clip.",
+ model,
+ prompt: custom
+ ? "Track the selected subject through the clip."
+ : "Remove the clip's background.",
inputs: { video: { blobRef, blobField: "url", blobUrl: true, mimeType: "video/mp4" } },
parameters,
});
diff --git a/site/src/lib/credits/provider-pricing.ts b/site/src/lib/credits/provider-pricing.ts
index e9d0155a..faa4cb28 100644
--- a/site/src/lib/credits/provider-pricing.ts
+++ b/site/src/lib/credits/provider-pricing.ts
@@ -332,10 +332,13 @@ function geminiOmniCreditPricing(model: string): ProviderCreditPricing | undefin
// The matte segmenter bills upstream per 16 processed frames ($0.005 a
// chunk); the adapter counts the submitted segment's frames and charges the
-// same unit, generationCount = started chunks. Keyed by FalMatteModel, so
-// adding a segmenter id without a price fails the build.
+// same unit, generationCount = started chunks. Each model meters its own
+// chunk size (the adapter's matteChunkFrames): the segmenter per 16 frames,
+// the removal model per 30 frames with edge refinement on. Keyed by
+// FalMatteModel, so adding a matte id without a price fails the build.
const falMatteModelPricing: Record<FalMatteModel, ProviderCreditPricing> = {
[falMatteModels.segmenter]: { generationCostMicros: usdWithMargin("0.005") },
+ [falMatteModels.removal]: { generationCostMicros: usdWithMargin("0.0225") },
};
function falMatteCreditPricing(model: string): ProviderCreditPricing | undefined {
diff --git a/site/src/lib/inference/adapters/fal-matte.ts b/site/src/lib/inference/adapters/fal-matte.ts
index e03c165b..b6eb9d6f 100644
--- a/site/src/lib/inference/adapters/fal-matte.ts
+++ b/site/src/lib/inference/adapters/fal-matte.ts
@@ -16,21 +16,26 @@ import {
type StoredGenerationForProvider,
} from "@/lib/inference/providers";
-// The matte provider: a queue-hosted segmenter for kind="matte". One
-// call takes a clip segment (by URL — the route resolves the caller's stored
-// blob into a read link) plus a text description of what to keep, click
-// prompts, or both together — points sharing an object id refine what the
-// words detected — and returns a mask video. The prediction is async: submit
-// here, poll through refresh; the charge bills at submit, counted in the
-// segmenter's own unit.
+// The matte provider: queue-hosted matting for kind="matte", one model per
+// job. The segmenter takes a clip segment (by URL — the route resolves the
+// caller's stored blob into a read link) plus a text description of what to
+// keep, click prompts, or both together — points sharing an object id refine
+// what the words detected — and returns a mask video. The removal model
+// takes only the segment and masks the foreground subject whole, returning
+// the mask as a grayscale "alpha" h264 video. Predictions are async: submit
+// here, poll through refresh; the charge bills at submit, counted in each
+// model's own unit.
const providerID = "fal";
const QUEUE_BASE = "https://queue.fal.run";
-// The segmenter meters per 16 processed frames, so the charge counts the same
-// unit: the submitted segment is probed for its frame count and each started
-// chunk of 16 bills one generation at the per-chunk price.
-const CHUNK_FRAMES = 16;
+// Each model meters per processed-frame chunk of its own size, so the charge
+// counts the same unit: the submitted segment is probed for its frame count
+// and each started chunk bills one generation at the model's per-chunk price.
+const matteChunkFrames: Record<FalMatteModel, number> = {
+ [falMatteModels.segmenter]: 16,
+ [falMatteModels.removal]: 30,
+};
// The billable chunk count of the segment at `videoUrl` — the server's own
// count, so the charge never rides a client-supplied number. The frame count
@@ -40,7 +45,7 @@ const CHUNK_FRAMES = 16;
// billing path. An unreadable segment stops the submit.
const RATE_SAMPLE_PACKETS = 100;
-async function billableChunks(videoUrl: string): Promise<number> {
+async function billableChunks(videoUrl: string, chunkFrames: number): Promise<number> {
const input = new Input({ source: new UrlSource(videoUrl), formats: ALL_FORMATS });
try {
const track = await input.getPrimaryVideoTrack();
@@ -49,7 +54,7 @@ async function billableChunks(videoUrl: string): Promise<number> {
const rate = (await track.computePacketStats(RATE_SAMPLE_PACKETS)).averagePacketRate;
const frames = Math.round(duration * rate);
if (frames > 0) {
- return Math.ceil(frames / CHUNK_FRAMES);
+ return Math.ceil(frames / chunkFrames);
}
}
} catch {
@@ -73,28 +78,27 @@ export function createFalMatteProvider(
const token = environment.FAL_KEY?.trim();
const configured = Boolean(token);
const defaultModel: FalMatteModel = falMatteModels.segmenter;
+ const matteModels = Object.values(falMatteModels) as string[];
async function listModels(modalities: InferenceModality[]): Promise<InferenceModel[]> {
if (!modalities.includes("matte")) {
return [];
}
- return [
- {
- id: defaultModel,
- name: defaultModel,
- provider: providerID,
- inputModalities: ["video"],
- outputModalities: ["video"],
- contextLength: null,
- pricing: null,
- metadata: { provider: providerID },
- },
- ];
+ return matteModels.map((id) => ({
+ id,
+ name: id,
+ provider: providerID,
+ inputModalities: ["video"],
+ outputModalities: ["video"],
+ contextLength: null,
+ pricing: null,
+ metadata: { provider: providerID },
+ }));
}
- function resolveModel(requested?: string): string {
+ function resolveModel(requested?: string): FalMatteModel {
const model = requested?.trim() || defaultModel;
- if (model !== defaultModel) {
+ if (!matteModels.includes(model)) {
throw new InferenceProviderError("Unknown matte model.", {
statusCode: 400,
code: "unknown_matte_model",
@@ -108,7 +112,7 @@ export function createFalMatteProvider(
{ statusCode: 500, code: "matte_model_not_priced", details: { model } },
);
}
- return model;
+ return model as FalMatteModel;
}
async function api(url: string, init?: RequestInit): Promise<JsonValue> {
@@ -146,7 +150,7 @@ export function createFalMatteProvider(
}
// The route's pre-preflight count: the chunks this segment will bill, read
- // from the resolved segment itself.
+ // from the resolved segment itself in the resolved model's own unit.
async function assetGenerationCountFor(
request: AssetGenerationProviderRequest["request"],
): Promise<number> {
@@ -157,7 +161,7 @@ export function createFalMatteProvider(
code: "unsupported_asset_kind",
});
}
- return billableChunks(segmentUrl(request));
+ return billableChunks(segmentUrl(request), matteChunkFrames[resolveModel(request.model)]);
}
async function generateAsset({
@@ -187,26 +191,44 @@ export function createFalMatteProvider(
const parameters = toJsonObject(request.parameters ?? {});
const prompt = matteConceptPrompt(parameters.prompt);
const points = mattePointPrompts(parameters.points);
- if (!prompt && points.length === 0) {
- throw new InferenceProviderError("A matte needs a text prompt or point prompts.", {
- statusCode: 400,
- code: "empty_matte_prompts",
- });
- }
- // The segmenter takes object ids as integers; each named brush object
- // maps to a stable one in prompt order.
- const objectIds = new Map<string, number>();
- const objectId = (name: string) => {
- const held = objectIds.get(name);
- if (held !== undefined) return held;
- const next = objectIds.size + 1;
- objectIds.set(name, next);
- return next;
- };
- const submitted = await api(`${QUEUE_BASE}/${model}`, {
- method: "POST",
- body: JSON.stringify({
+ let body: Record<string, unknown>;
+ if (model === falMatteModels.removal) {
+ // The removal model finds the foreground subject on its own; prompts
+ // have nowhere to land, so a prompted request is a caller bug.
+ if (prompt || points.length > 0) {
+ throw new InferenceProviderError("The removal model takes no selection prompts.", {
+ statusCode: 400,
+ code: "matte_prompts_unsupported",
+ });
+ }
+ body = {
+ video_url: videoUrl,
+ // h264 returns the mask alone as a grayscale "alpha" video — the
+ // matte contract's shape; vp9 would fold it into an alpha channel
+ // WebCodecs cannot read back.
+ output_codec: "h264",
+ refine_foreground_edges: true,
+ subject_is_person: true,
+ };
+ } else {
+ if (!prompt && points.length === 0) {
+ throw new InferenceProviderError("A matte needs a text prompt or point prompts.", {
+ statusCode: 400,
+ code: "empty_matte_prompts",
+ });
+ }
+ // The segmenter takes object ids as integers; each named brush object
+ // maps to a stable one in prompt order.
+ const objectIds = new Map<string, number>();
+ const objectId = (name: string) => {
+ const held = objectIds.get(name);
+ if (held !== undefined) return held;
+ const next = objectIds.size + 1;
+ objectIds.set(name, next);
+ return next;
+ };
+ body = {
video_url: videoUrl,
// The mask video, without the source picture under it.
apply_mask: false,
@@ -222,7 +244,12 @@ export function createFalMatteProvider(
})),
}
: {}),
- }),
+ };
+ }
+
+ const submitted = await api(`${QUEUE_BASE}/${model}`, {
+ method: "POST",
+ body: JSON.stringify(body),
});
const requestId =
isJsonObject(submitted) && typeof submitted.request_id === "string"
@@ -328,7 +355,10 @@ export function createFalMatteProvider(
};
}
-// The finished run's mask video URL.
+// The finished run's mask video URL. The segmenter returns one video object;
+// the removal model returns a file list where the mask is the entry named
+// "alpha" (verified against the live endpoint — h264 returns the alpha video
+// alone, at the segment's own size and rate).
function maskUrl(result: JsonValue): string | null {
if (!isJsonObject(result)) {
return null;
@@ -337,5 +367,16 @@ function maskUrl(result: JsonValue): string | null {
if (isJsonObject(video) && typeof video.url === "string" && video.url.startsWith("https://")) {
return video.url;
}
+ if (Array.isArray(video)) {
+ const files = video.filter(
+ (f): f is { url: string; file_name?: string } =>
+ isJsonObject(f) && typeof f.url === "string" && f.url.startsWith("https://"),
+ );
+ const alpha = files.find((f) =>
+ `${typeof f.file_name === "string" ? f.file_name : ""} ${f.url}`.toLowerCase().includes("alpha"),
+ );
+ if (alpha) return alpha.url;
+ if (files.length === 1) return files[0].url;
+ }
return null;
}
diff --git a/site/src/lib/inference/matte-models.ts b/site/src/lib/inference/matte-models.ts
index 73176868..2e1247e2 100644
--- a/site/src/lib/inference/matte-models.ts
+++ b/site/src/lib/inference/matte-models.ts
@@ -1,13 +1,17 @@
-// The hosted matte (video background-removal) model ids, hardcoded like the
-// Gemini registries: configuration is code, and pricing keys off these exact
-// strings.
+// The hosted matte model ids, hardcoded like the Gemini registries:
+// configuration is code, and pricing keys off these exact strings.
-// The segmenter: one call takes a clip plus a text description, click
-// prompts, or both together (points refine what the words detected), and
-// returns the mask video (apply_mask: false renders the white-on-black mask;
-// the sibling video-rle endpoint returns RLE JSON with no video output).
+// Two matte jobs, one model each. The segmenter takes a clip plus a text
+// description, click prompts, or both together (points refine what the words
+// detected) and returns the mask video (apply_mask: false renders the
+// white-on-black mask; the sibling video-rle endpoint returns RLE JSON with
+// no video output). The removal model takes only the clip — no prompts — and
+// masks the foreground subject whole; with output_codec "h264" it returns
+// the mask as a grayscale "alpha" video at the segment's own size and rate,
+// which is the matte contract's exact shape.
export const falMatteModels = {
segmenter: "fal-ai/sam-3/video",
+ removal: "veed/video-background-removal",
} as const;
export type FalMatteModel = (typeof falMatteModels)[keyof typeof falMatteModels];