Open source CapCut alternative.
TypeScript
feat(site): the auto quality matte bakes on the hosted removal model
David Le6d5a477c8d96
diff --git a/site/src/cut/lib/removal/hostedBake.ts b/site/src/cut/lib/removal/hostedBake.tsindex f1f3bfac..9c645de5 100644--- a/site/src/cut/lib/removal/hostedBake.ts+++ b/site/src/cut/lib/removal/hostedBake.ts@@ -2,12 +2,13 @@/*** The quality matte bake: the clip's trimmed segment goes to the hosted- * tracker (credits), which propagates the subject across every frame; the- * returned mask video is refined here against the source pixels with the- * kit's guided filter and encoded as the final matte asset. Auto mode seeds- * the tracker from the on-device person matte; custom mode seeds it from the- * brush's stored prompts and paint. Driven by the page or the cloud worker —- * the local engine only ever consumes the stored matte.+ * matting service (credits) and the returned mask video is refined here+ * against the source pixels with the kit's guided filter and encoded as the+ * final matte asset. One model per job: auto mode sends the segment to the+ * background-removal model, which masks the foreground subject on its own;+ * custom mode sends it to the promptable tracker, seeded by the described+ * subject and the brush's stored prompts and paint. Driven by the page or+ * the cloud worker — the local engine only ever consumes the stored matte.** A long clip tracks in parts: the span splits into pieces of at most* `MATTE_MAX_S`, each rendered, uploaded, and tracked on its own upload@@ -33,6 +34,7 @@ import { NO_CREDITS_MESSAGE } from "../credits";import { ClipReader } from "../exportRender";import { hostedPost } from "../hosted";import { storeHostedBlob } from "../hostedBlobs";+import { falMatteModels } from "@/lib/inference/matte-models";import { createRasterCanvas, decodeRasterImageUrl, type RasterSurface } from "../raster";import type { MediaAsset, RemovalSeeds, VideoClip } from "../types";import { liveReader } from "../liveReader";@@ -450,8 +452,10 @@ export async function hostedBakeMatte(const nParts = Math.max(1, Math.ceil(totalFrames / maxFrames));const perPart = Math.ceil(totalFrames / nParts);- const painted = r.mode === "custom" && !!(r.seeds?.prompts.length || r.seeds?.paint?.length);- const concept = r.mode === "custom" ? (r.subject ?? "").trim() : "person";+ const custom = r.mode === "custom";+ const painted = custom && !!(r.seeds?.prompts.length || r.seeds?.paint?.length);+ const concept = custom ? (r.subject ?? "").trim() : "";+ const model = custom ? falMatteModels.segmenter : falMatteModels.removal;const gens = [...ticketParts(opts.resume, nParts, from, to)];const ticket = (upto: number): HostedBakeTicket => ({@@ -478,12 +482,13 @@ export async function hostedBakeMatte(const segment = await renderSegment(srcReader, still, partFrom, partTo, frames, opts.signal, (f) =>frac(0.02 + f * 0.13));- // The words and the strokes ride together: auto mode and a described- // subject send a text prompt, painted seeds send point prompts, and a- // custom removal with both sends both — the segmenter's points refine- // what the words detected. A painted part with no strokes of its own- // takes the handoff points from the previous part's final mask, so the- // picked selection crosses the seam. The route picks the segmenter.+ // Custom mode's words and strokes ride together: a described subject+ // sends a text prompt, painted seeds send point prompts, and a+ // removal with both sends both — the segmenter's points refine what+ // the words detected. A painted part with no strokes of its own+ // takes the handoff points from the previous part's final mask, so+ // the picked selection crosses the seam. Auto mode sends the segment+ // alone — the removal model finds the subject itself.let points = painted ? await seedPoints(r.seeds!, segment, segment.from) : [];if (painted && points.length === 0 && carry.length > 0) {points = carry.map((c) => ({@@ -494,7 +499,7 @@ export async function hostedBakeMatte(object: "carry",}));}- if (!concept && points.length === 0) {+ if (custom && !concept && points.length === 0) {throw p === 0? new Error("No selection to track."): new MatteTrackFailed("The selection was lost at a part boundary.");@@ -511,7 +516,10 @@ export async function hostedBakeMatte(const submit = await hostedPost("/api/inference/assets", {kind: "matte",- prompt: "Track the selected subject through the clip.",+ model,+ prompt: custom+ ? "Track the selected subject through the clip."+ : "Remove the clip's background.",inputs: { video: { blobRef, blobField: "url", blobUrl: true, mimeType: "video/mp4" } },parameters,});diff --git a/site/src/lib/credits/provider-pricing.ts b/site/src/lib/credits/provider-pricing.tsindex e9d0155a..faa4cb28 100644--- a/site/src/lib/credits/provider-pricing.ts+++ b/site/src/lib/credits/provider-pricing.ts@@ -332,10 +332,13 @@ function geminiOmniCreditPricing(model: string): ProviderCreditPricing | undefin// The matte segmenter bills upstream per 16 processed frames ($0.005 a// chunk); the adapter counts the submitted segment's frames and charges the-// same unit, generationCount = started chunks. Keyed by FalMatteModel, so-// adding a segmenter id without a price fails the build.+// same unit, generationCount = started chunks. Each model meters its own+// chunk size (the adapter's matteChunkFrames): the segmenter per 16 frames,+// the removal model per 30 frames with edge refinement on. Keyed by+// FalMatteModel, so adding a matte id without a price fails the build.const falMatteModelPricing: Record<FalMatteModel, ProviderCreditPricing> = {[falMatteModels.segmenter]: { generationCostMicros: usdWithMargin("0.005") },+ [falMatteModels.removal]: { generationCostMicros: usdWithMargin("0.0225") },};function falMatteCreditPricing(model: string): ProviderCreditPricing | undefined {diff --git a/site/src/lib/inference/adapters/fal-matte.ts b/site/src/lib/inference/adapters/fal-matte.tsindex e03c165b..b6eb9d6f 100644--- a/site/src/lib/inference/adapters/fal-matte.ts+++ b/site/src/lib/inference/adapters/fal-matte.ts@@ -16,21 +16,26 @@ import {type StoredGenerationForProvider,} from "@/lib/inference/providers";-// The matte provider: a queue-hosted segmenter for kind="matte". One-// call takes a clip segment (by URL — the route resolves the caller's stored-// blob into a read link) plus a text description of what to keep, click-// prompts, or both together — points sharing an object id refine what the-// words detected — and returns a mask video. The prediction is async: submit-// here, poll through refresh; the charge bills at submit, counted in the-// segmenter's own unit.+// The matte provider: queue-hosted matting for kind="matte", one model per+// job. The segmenter takes a clip segment (by URL — the route resolves the+// caller's stored blob into a read link) plus a text description of what to+// keep, click prompts, or both together — points sharing an object id refine+// what the words detected — and returns a mask video. The removal model+// takes only the segment and masks the foreground subject whole, returning+// the mask as a grayscale "alpha" h264 video. Predictions are async: submit+// here, poll through refresh; the charge bills at submit, counted in each+// model's own unit.const providerID = "fal";const QUEUE_BASE = "https://queue.fal.run";-// The segmenter meters per 16 processed frames, so the charge counts the same-// unit: the submitted segment is probed for its frame count and each started-// chunk of 16 bills one generation at the per-chunk price.-const CHUNK_FRAMES = 16;+// Each model meters per processed-frame chunk of its own size, so the charge+// counts the same unit: the submitted segment is probed for its frame count+// and each started chunk bills one generation at the model's per-chunk price.+const matteChunkFrames: Record<FalMatteModel, number> = {+ [falMatteModels.segmenter]: 16,+ [falMatteModels.removal]: 30,+};// The billable chunk count of the segment at `videoUrl` — the server's own// count, so the charge never rides a client-supplied number. The frame count@@ -40,7 +45,7 @@ const CHUNK_FRAMES = 16;// billing path. An unreadable segment stops the submit.const RATE_SAMPLE_PACKETS = 100;-async function billableChunks(videoUrl: string): Promise<number> {+async function billableChunks(videoUrl: string, chunkFrames: number): Promise<number> {const input = new Input({ source: new UrlSource(videoUrl), formats: ALL_FORMATS });try {const track = await input.getPrimaryVideoTrack();@@ -49,7 +54,7 @@ async function billableChunks(videoUrl: string): Promise<number> {const rate = (await track.computePacketStats(RATE_SAMPLE_PACKETS)).averagePacketRate;const frames = Math.round(duration * rate);if (frames > 0) {- return Math.ceil(frames / CHUNK_FRAMES);+ return Math.ceil(frames / chunkFrames);}}} catch {@@ -73,28 +78,27 @@ export function createFalMatteProvider(const token = environment.FAL_KEY?.trim();const configured = Boolean(token);const defaultModel: FalMatteModel = falMatteModels.segmenter;+ const matteModels = Object.values(falMatteModels) as string[];async function listModels(modalities: InferenceModality[]): Promise<InferenceModel[]> {if (!modalities.includes("matte")) {return [];}- return [- {- id: defaultModel,- name: defaultModel,- provider: providerID,- inputModalities: ["video"],- outputModalities: ["video"],- contextLength: null,- pricing: null,- metadata: { provider: providerID },- },- ];+ return matteModels.map((id) => ({+ id,+ name: id,+ provider: providerID,+ inputModalities: ["video"],+ outputModalities: ["video"],+ contextLength: null,+ pricing: null,+ metadata: { provider: providerID },+ }));}- function resolveModel(requested?: string): string {+ function resolveModel(requested?: string): FalMatteModel {const model = requested?.trim() || defaultModel;- if (model !== defaultModel) {+ if (!matteModels.includes(model)) {throw new InferenceProviderError("Unknown matte model.", {statusCode: 400,code: "unknown_matte_model",@@ -108,7 +112,7 @@ export function createFalMatteProvider({ statusCode: 500, code: "matte_model_not_priced", details: { model } },);}- return model;+ return model as FalMatteModel;}async function api(url: string, init?: RequestInit): Promise<JsonValue> {@@ -146,7 +150,7 @@ export function createFalMatteProvider(}// The route's pre-preflight count: the chunks this segment will bill, read- // from the resolved segment itself.+ // from the resolved segment itself in the resolved model's own unit.async function assetGenerationCountFor(request: AssetGenerationProviderRequest["request"],): Promise<number> {@@ -157,7 +161,7 @@ export function createFalMatteProvider(code: "unsupported_asset_kind",});}- return billableChunks(segmentUrl(request));+ return billableChunks(segmentUrl(request), matteChunkFrames[resolveModel(request.model)]);}async function generateAsset({@@ -187,26 +191,44 @@ export function createFalMatteProvider(const parameters = toJsonObject(request.parameters ?? {});const prompt = matteConceptPrompt(parameters.prompt);const points = mattePointPrompts(parameters.points);- if (!prompt && points.length === 0) {- throw new InferenceProviderError("A matte needs a text prompt or point prompts.", {- statusCode: 400,- code: "empty_matte_prompts",- });- }- // The segmenter takes object ids as integers; each named brush object- // maps to a stable one in prompt order.- const objectIds = new Map<string, number>();- const objectId = (name: string) => {- const held = objectIds.get(name);- if (held !== undefined) return held;- const next = objectIds.size + 1;- objectIds.set(name, next);- return next;- };- const submitted = await api(`${QUEUE_BASE}/${model}`, {- method: "POST",- body: JSON.stringify({+ let body: Record<string, unknown>;+ if (model === falMatteModels.removal) {+ // The removal model finds the foreground subject on its own; prompts+ // have nowhere to land, so a prompted request is a caller bug.+ if (prompt || points.length > 0) {+ throw new InferenceProviderError("The removal model takes no selection prompts.", {+ statusCode: 400,+ code: "matte_prompts_unsupported",+ });+ }+ body = {+ video_url: videoUrl,+ // h264 returns the mask alone as a grayscale "alpha" video — the+ // matte contract's shape; vp9 would fold it into an alpha channel+ // WebCodecs cannot read back.+ output_codec: "h264",+ refine_foreground_edges: true,+ subject_is_person: true,+ };+ } else {+ if (!prompt && points.length === 0) {+ throw new InferenceProviderError("A matte needs a text prompt or point prompts.", {+ statusCode: 400,+ code: "empty_matte_prompts",+ });+ }+ // The segmenter takes object ids as integers; each named brush object+ // maps to a stable one in prompt order.+ const objectIds = new Map<string, number>();+ const objectId = (name: string) => {+ const held = objectIds.get(name);+ if (held !== undefined) return held;+ const next = objectIds.size + 1;+ objectIds.set(name, next);+ return next;+ };+ body = {video_url: videoUrl,// The mask video, without the source picture under it.apply_mask: false,@@ -222,7 +244,12 @@ export function createFalMatteProvider(})),}: {}),- }),+ };+ }++ const submitted = await api(`${QUEUE_BASE}/${model}`, {+ method: "POST",+ body: JSON.stringify(body),});const requestId =isJsonObject(submitted) && typeof submitted.request_id === "string"@@ -328,7 +355,10 @@ export function createFalMatteProvider(};}-// The finished run's mask video URL.+// The finished run's mask video URL. The segmenter returns one video object;+// the removal model returns a file list where the mask is the entry named+// "alpha" (verified against the live endpoint — h264 returns the alpha video+// alone, at the segment's own size and rate).function maskUrl(result: JsonValue): string | null {if (!isJsonObject(result)) {return null;@@ -337,5 +367,16 @@ function maskUrl(result: JsonValue): string | null {if (isJsonObject(video) && typeof video.url === "string" && video.url.startsWith("https://")) {return video.url;}+ if (Array.isArray(video)) {+ const files = video.filter(+ (f): f is { url: string; file_name?: string } =>+ isJsonObject(f) && typeof f.url === "string" && f.url.startsWith("https://"),+ );+ const alpha = files.find((f) =>+ `${typeof f.file_name === "string" ? f.file_name : ""} ${f.url}`.toLowerCase().includes("alpha"),+ );+ if (alpha) return alpha.url;+ if (files.length === 1) return files[0].url;+ }return null;}diff --git a/site/src/lib/inference/matte-models.ts b/site/src/lib/inference/matte-models.tsindex 73176868..2e1247e2 100644--- a/site/src/lib/inference/matte-models.ts+++ b/site/src/lib/inference/matte-models.ts@@ -1,13 +1,17 @@-// The hosted matte (video background-removal) model ids, hardcoded like the-// Gemini registries: configuration is code, and pricing keys off these exact-// strings.+// The hosted matte model ids, hardcoded like the Gemini registries:+// configuration is code, and pricing keys off these exact strings.-// The segmenter: one call takes a clip plus a text description, click-// prompts, or both together (points refine what the words detected), and-// returns the mask video (apply_mask: false renders the white-on-black mask;-// the sibling video-rle endpoint returns RLE JSON with no video output).+// Two matte jobs, one model each. The segmenter takes a clip plus a text+// description, click prompts, or both together (points refine what the words+// detected) and returns the mask video (apply_mask: false renders the+// white-on-black mask; the sibling video-rle endpoint returns RLE JSON with+// no video output). The removal model takes only the clip — no prompts — and+// masks the foreground subject whole; with output_codec "h264" it returns+// the mask as a grayscale "alpha" video at the segment's own size and rate,+// which is the matte contract's exact shape.export const falMatteModels = {segmenter: "fal-ai/sam-3/video",+ removal: "veed/video-background-removal",} as const;export type FalMatteModel = (typeof falMatteModels)[keyof typeof falMatteModels];