Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

feat(camera): show the camera background on the crop thumbnail

EtienneLescoteab7eb380e04
diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs
index 6c2c8a768..13ce87614 100644
--- a/crates/compositor-view-napi/src/lib.rs
+++ b/crates/compositor-view-napi/src/lib.rs
@@ -87,6 +87,56 @@ pub fn segmentation_runtime_available() -> bool {
openscreen_compositor::segmentation::runtime_available()
}
+pub struct SegmentFrameTask {
+ model_path: String,
+ rgba: Vec<u8>,
+}
+
+impl Task for SegmentFrameTask {
+ type Output = Vec<u8>;
+ type JsValue = Buffer;
+
+ fn compute(&mut self) -> Result<Self::Output> {
+ let rgb: Vec<u8> = self
+ .rgba
+ .chunks_exact(4)
+ .flat_map(|p| [p[0], p[1], p[2]])
+ .collect();
+ // ponytail: une session ONNX chargée par appel (quelques dizaines de ms), parce qu'il
+ // n'y a qu'un appel par caméra et par ouverture du panneau. À garder en cache si un
+ // appelant se met à segmenter en continu.
+ let mut segmenter = openscreen_compositor::segmentation::Segmenter::load(
+ std::path::Path::new(&self.model_path),
+ )
+ .map_err(|e| Error::from_reason(format!("{e:#}")))?;
+ segmenter
+ .run(&rgb)
+ .map(<[u8]>::to_vec)
+ .map_err(|e| Error::from_reason(format!("{e:#}")))
+ }
+
+ fn resolve(&mut self, _env: Env, mask: Self::Output) -> Result<Self::JsValue> {
+ Ok(mask.into())
+ }
+}
+
+/// Masque du sujet pour UNE image, sans vue : la vignette de recadrage du panneau caméra s'en
+/// sert pour montrer le fond choisi sur sa propre frame.
+///
+/// `rgba` est une frame RGBA8 déjà réduite à la taille du modèle (`segmentation::MODEL_WIDTH` x
+/// `MODEL_HEIGHT`), soit le `getImageData` d'un canvas tel quel. Le retour a le format de
+/// `Segmenter::run` : un octet par pixel, 0 = fond, 255 = sujet.
+///
+/// `AsyncTask`, comme `remux_seekable` : charger la session prend plus longtemps que l'inférence,
+/// et ce temps n'a rien à faire sur le thread principal de Node.
+#[napi]
+pub fn segment_frame(model_path: String, rgba: Buffer) -> AsyncTask<SegmentFrameTask> {
+ AsyncTask::new(SegmentFrameTask {
+ model_path,
+ rgba: rgba.to_vec(),
+ })
+}
+
#[napi]
pub fn create_view(
rect: CompositorViewRect,
diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts
index 2bf0e04c4..e25ea048b 100644
--- a/electron/ipc/nativeBridge.ts
+++ b/electron/ipc/nativeBridge.ts
@@ -372,6 +372,10 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) {
return createSuccessResponse(requestId, {
support: compositorViewService.probeSegmentation(),
});
+ case "segmentFrame":
+ return createSuccessResponse(requestId, {
+ mask: await compositorViewService.segmentFrame(request.payload.rgba),
+ });
case "setRect":
compositorViewService.setRect(request.payload.id, request.payload.rect);
return createSuccessResponse(requestId, { ok: true });
diff --git a/electron/native-bridge/services/compositorViewService.ts b/electron/native-bridge/services/compositorViewService.ts
index 918b6ea43..aa9774e07 100644
--- a/electron/native-bridge/services/compositorViewService.ts
+++ b/electron/native-bridge/services/compositorViewService.ts
@@ -556,6 +556,20 @@ export class CompositorViewService {
return resolveSceneAssetPath(SEGMENTATION_MODEL_ASSET) ? "ready" : "no-model";
}
+ /** Subject mask for one camera frame, no view needed — see `segmentFrame` in the addon.
+ * `null` when the addon (or this version of it) or the model is missing. */
+ async segmentFrame(rgba: Uint8Array): Promise<Uint8Array | null> {
+ const addon = this.ensureAddon();
+ const modelPath = resolveSceneAssetPath(SEGMENTATION_MODEL_ASSET);
+ if (!addon?.segmentFrame || !modelPath) {
+ return null;
+ }
+ return addon.segmentFrame(
+ modelPath,
+ Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength),
+ );
+ }
+
/** Allocates an offscreen compositor view sized to `rect.width`x`rect.height`.
* `rect.x` / `rect.y` are vestigial (ignored native-side) — the renderer
* keeps them on the wire so the existing `CompositorViewRect` shape stays
diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts
index 2ac90e08e..1365846c7 100644
--- a/electron/native/compositor-view/addon.d.ts
+++ b/electron/native/compositor-view/addon.d.ts
@@ -123,6 +123,11 @@ export interface CompositorViewAddon {
* `--dir` build has none either. */
segmentationRuntimeAvailable(): boolean;
+ /** Subject mask for ONE frame, outside any view — the crop thumbnail shows the camera
+ * background with it. `rgba` is already at the model's size (256x144 RGBA8); resolves one
+ * byte per pixel, 0 = background, 255 = subject. Optional: an older `.node` predates it. */
+ segmentFrame?(modelPath: string, rgba: Buffer): Promise<Buffer>;
+
/** Allocates an offscreen compositor view sized to `rect.width`x`rect.height` (the
* target preview resolution; `rect.x` / `rect.y` are vestigial and ignored native-side).
* No HWND/native-window-handle is passed: there's no OS window to parent to. The
diff --git a/src/components/ai-edition/NewEditorShell.module.css b/src/components/ai-edition/NewEditorShell.module.css
index 6050c1a80..d200dc755 100644
--- a/src/components/ai-edition/NewEditorShell.module.css
+++ b/src/components/ai-edition/NewEditorShell.module.css
@@ -350,7 +350,7 @@
var(--scene-foliage-d) 100%);
/* ponytail: the wallpaper card is a floating, rounded surface (per the v4
design's preview frame) — rounded corners + a layered drop shadow lift it
- off the scene gradient. The custom background (buildFrameStyle) fills this
+ off the scene gradient. The custom background (wallpaperStyle) fills this
card; the video sits on top with its own padding so the wallpaper shows
around it. */
border-radius: 16px;
diff --git a/src/components/ai-edition/PreviewCanvas.tsx b/src/components/ai-edition/PreviewCanvas.tsx
index fe2a1eef4..f9ca4e1e8 100644
--- a/src/components/ai-edition/PreviewCanvas.tsx
+++ b/src/components/ai-edition/PreviewCanvas.tsx
@@ -57,7 +57,7 @@ import {
type WebcamCompositeLayout,
} from "@/lib/compositeLayout";
import { webcamAnchorAt } from "@/lib/projectDefaults";
-import { classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper";
+import { wallpaperStyle } from "@/lib/wallpaper";
import { getCssClipPath } from "@/lib/webcamMaskShapes";
import { computeCameraFullscreenProgress } from "@/lib/zoomMath/cameraFullscreenUtils";
import { AnnotationLayer } from "./AnnotationLayer";
@@ -321,7 +321,7 @@ export function PreviewCanvas(props: PreviewCanvasProps) {
// The stage hosting the interactive overlays is the CONTENT rect, and a frame never changes
// it: the compositor keeps the footage the same size with or without a frame and grows the
// frame outward, so every handle (annotations, privacy blur, zoom focus) sits on `layout`.
- const frameStyle = useMemo(() => buildFrameStyle(settings), [settings]);
+ const frameStyle = useMemo(() => wallpaperStyle(settings.wallpaper), [settings.wallpaper]);
const screenStyle = useMemo(
() => buildScreenStyle(layout, settings, frameSize),
[layout, settings, frameSize],
@@ -509,38 +509,6 @@ export function PreviewCanvas(props: PreviewCanvasProps) {
);
}
-// ponytail: resolve `settings.wallpaper` to an actual CSS background. Image
-// wallpapers must go through resolveImageWallpaperUrl (→ getAssetPath): in the
-// packaged Electron app the renderer loads over file://, where a bare
-// `/wallpapers/foo.jpg` points at the filesystem root and 404s, so the custom
-// background silently failed to paint (worked in the http dev server only).
-// classifyWallpaper also handles color-function (rgb/hsl/…) and every gradient
-// variant, which the old ad-hoc startsWith checks missed.
-function resolveWallpaperImageUrl(imagePath: string): string | null {
- try {
- return resolveImageWallpaperUrl(imagePath);
- } catch {
- return null;
- }
-}
-
-// Canvas: wallpaper only — no padding, no shadow.
-function buildFrameStyle(
- settings: ReturnType<typeof useEditorSettings>["settings"],
-): React.CSSProperties {
- const w = classifyWallpaper(settings.wallpaper);
- if (w.kind === "color") return { backgroundColor: w.value };
- if (w.kind === "gradient") return { backgroundImage: w.value, backgroundSize: "cover" };
- const url = resolveWallpaperImageUrl(w.path);
- if (!url) return {};
- return {
- backgroundImage: `url(${url})`,
- backgroundSize: "cover",
- backgroundPosition: "center",
- backgroundRepeat: "no-repeat",
- };
-}
-
// Screen stage: rectangle from the composite layout, converted to percentages
// of the canvas. Positions/sizes the interactive overlays (ZoomFocusOverlay,
// AnnotationLayer) hosted inside it and the CSS-hidden <video> (decode/clock
diff --git a/src/components/ai-edition/RightPanes.layout.test.tsx b/src/components/ai-edition/RightPanes.layout.test.tsx
index 4a9ff4071..cef8a13b0 100644
--- a/src/components/ai-edition/RightPanes.layout.test.tsx
+++ b/src/components/ai-edition/RightPanes.layout.test.tsx
@@ -5,7 +5,7 @@
// the saved preference is left untouched on disk and the help popover says so.
import "@testing-library/jest-dom";
-import { cleanup, fireEvent, render, screen, within } from "@testing-library/react";
+import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { I18nProvider } from "@/contexts/I18nContext";
@@ -14,6 +14,15 @@ import { type AxcutDocument, createEmptyDocument } from "@/lib/ai-edition/schema
import { useProjectStore } from "@/lib/ai-edition/store/projectStore";
import { LayoutPane } from "./RightPanes";
+const segmentation = vi.hoisted(() => ({ supported: false, segmentCameraFrame: vi.fn() }));
+vi.mock("@/native/hooks/useSegmentationSupport", () => ({
+ useCanSegmentCamera: () => segmentation.supported,
+}));
+vi.mock("@/native/compositorViewClient", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@/native/compositorViewClient")>()),
+ segmentCameraFrame: segmentation.segmentCameraFrame,
+}));
+
function seedProject(hasCamera: boolean): AxcutDocument {
const base = createEmptyDocument({ projectId: "project_layout", title: "Layout" });
return {
@@ -154,6 +163,72 @@ describe("LayoutPane webcam framing picture", () => {
});
});
+// The thumbnail shows the camera background the preview draws. The promise that makes it
+// cheap: nothing runs while the background is the camera's own, and the frame is segmented
+// once, every later change of mode, blur or wallpaper being CSS over the same mask.
+describe("LayoutPane webcam framing background", () => {
+ beforeEach(() => {
+ segmentation.supported = true;
+ segmentation.segmentCameraFrame.mockResolvedValue(new Uint8Array(256 * 144).fill(255));
+ // jsdom has no 2D canvas; the thumbnail only draws into one and reads it back.
+ vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
+ drawImage: vi.fn(),
+ getImageData: (_x: number, _y: number, w: number, h: number) => ({
+ data: new Uint8ClampedArray(w * h * 4),
+ }),
+ createImageData: (w: number, h: number) => ({ data: new Uint8ClampedArray(w * h * 4) }),
+ putImageData: vi.fn(),
+ } as unknown as CanvasRenderingContext2D);
+ vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue("data:image/png;base64,AA");
+ });
+ afterEach(() => {
+ segmentation.supported = false;
+ segmentation.segmentCameraFrame.mockReset();
+ vi.restoreAllMocks();
+ });
+
+ const withBackground = (webcamBackgroundMode: string) => {
+ const doc = seedProject(true);
+ return { ...doc, legacyEditor: { ...doc.legacyEditor, webcamBackgroundMode } };
+ };
+ /** Loads the thumbnail's frame the way the browser would: metadata, then the seek. */
+ const showFrame = () => {
+ const video = document.querySelector('[class*="framingVideo"]') as HTMLVideoElement;
+ Object.defineProperty(video, "videoWidth", { configurable: true, value: 640 });
+ Object.defineProperty(video, "videoHeight", { configurable: true, value: 480 });
+ fireEvent.loadedMetadata(video);
+ fireEvent.seeked(video);
+ return video;
+ };
+ const backdrop = () =>
+ document.querySelector('[class*="framingBox"] > div[aria-hidden="true"]') as HTMLElement | null;
+
+ it("segments nothing while the camera keeps its own background", () => {
+ renderLayout(withBackground("none"));
+ const video = showFrame();
+
+ expect(segmentation.segmentCameraFrame).not.toHaveBeenCalled();
+ expect(video.style.maskImage).toBe("");
+ expect(backdrop()).toBeNull();
+ });
+
+ it("cuts the subject out once, then restyles the background around it", async () => {
+ renderLayout(withBackground("blur"));
+ const video = showFrame();
+
+ await waitFor(() => expect(video.style.maskImage).toContain("data:image/png"));
+ expect(backdrop()?.style.filter).toMatch(/^blur\(/);
+
+ fireEvent.click(screen.getByRole("button", { name: "Custom" }));
+ expect(backdrop()?.style.filter).toBe("");
+ fireEvent.click(screen.getByRole("button", { name: "Cutout" }));
+ expect(backdrop()).toBeNull();
+ expect(video.style.maskImage).toContain("data:image/png");
+
+ expect(segmentation.segmentCameraFrame).toHaveBeenCalledTimes(1);
+ });
+});
+
describe("LayoutPane picture-in-picture camera", () => {
const stored = () => useProjectStore.getState().document?.legacyEditor as Record<string, unknown>;
diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx
index 2ba359d49..7bfa6992a 100644
--- a/src/components/ai-edition/RightPanes.tsx
+++ b/src/components/ai-edition/RightPanes.tsx
@@ -45,7 +45,11 @@ import { toast } from "sonner";
import defaultCursorPreviewUrl from "@/assets/cursors/Cursor=Default.svg";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import { toFileUrl } from "@/components/video-editor/projectPersistence";
-import { WALLPAPER_MOTIONS, type WallpaperMotion } from "@/components/video-editor/types";
+import {
+ WALLPAPER_MOTIONS,
+ type WallpaperMotion,
+ type WebcamBackgroundMode,
+} from "@/components/video-editor/types";
import { useI18n, useScopedT } from "@/contexts/I18nContext";
import { resolveCaptionLane } from "@/lib/ai-edition/captions/settings";
import { collapseTracksToPills, trackGroupId } from "@/lib/ai-edition/document/audioTracks";
@@ -122,8 +126,14 @@ import {
resolveImageWallpaperUrl,
WALLPAPER_PATHS,
WALLPAPER_THUMB_PATHS,
+ wallpaperStyle,
} from "@/lib/wallpaper";
import { isNativeCompositorActive, setNativeParam } from "@/native";
+import {
+ SEGMENTATION_HEIGHT,
+ SEGMENTATION_WIDTH,
+ segmentCameraFrame,
+} from "@/native/compositorViewClient";
import { ROUNDNESS_SLIDER_MAX_PX } from "@/native/paramUnits";
import { wallpaperAcceptsMotion } from "@/native/sceneDescription";
import {
@@ -3228,6 +3238,15 @@ export function LayoutPane() {
src={cameraSrc}
crop={webcamCrop}
pan={cropPan}
+ background={
+ canSegmentCamera && settings.webcamBackgroundMode !== "none"
+ ? {
+ mode: settings.webcamBackgroundMode,
+ blurIntensity: settings.webcamBlurIntensity,
+ wallpaper: settings.webcamWallpaper,
+ }
+ : null
+ }
disabled={layoutControlsDisabled}
hint={ts("layout.webcamFramingDrag")}
onFrameLive={setCropFrame}
@@ -3242,6 +3261,35 @@ const MIN_CROP_SIZE = 1 / 3;
const FRAME_CORNERS = ["nw", "ne", "sw", "se"] as const;
type FrameCorner = (typeof FRAME_CORNERS)[number];
+/** The thumbnail's frame, cut out once: a subject mask to lay over the video, and a small copy
+ * of the frame for the blur mode to blur. `null` when this machine cannot segment. */
+async function cutOutSubject(
+ video: HTMLVideoElement,
+): Promise<{ mask: string; backdrop: string } | null> {
+ const canvas = document.createElement("canvas");
+ const ctx = canvas.getContext("2d");
+ if (!ctx || video.videoWidth === 0) return null;
+ // Blurred, a small copy is as good as the full frame. It keeps the camera's shape.
+ canvas.width = SEGMENTATION_WIDTH;
+ canvas.height = Math.round((SEGMENTATION_WIDTH * video.videoHeight) / video.videoWidth);
+ ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
+ const backdrop = canvas.toDataURL("image/jpeg");
+ // The model sees the whole frame squeezed to its input size, as the compositor feeds it.
+ canvas.width = SEGMENTATION_WIDTH;
+ canvas.height = SEGMENTATION_HEIGHT;
+ ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
+ const pixels = ctx.getImageData(0, 0, canvas.width, canvas.height).data;
+ const mask = await segmentCameraFrame(new Uint8Array(pixels.buffer));
+ if (!mask) return null;
+ // Stretched back over the video by `mask-size: 100% 100%`, as the shader samples it.
+ const alpha = ctx.createImageData(canvas.width, canvas.height);
+ mask.forEach((value, i) => {
+ alpha.data[i * 4 + 3] = value;
+ });
+ ctx.putImageData(alpha, 0, 0);
+ return { mask: canvas.toDataURL(), backdrop };
+}
+
/** Où la webcam cadre, réglé à la main : une vignette de la caméra, le cadre gardé posé dessus.
* Le cadre EST le réglage : on le glisse pour choisir ce qu'il montre, on tire un coin pour
* zoomer, comme n'importe quel outil de recadrage. Il garde la forme de la caméra, que la
@@ -3257,6 +3305,7 @@ function WebcamFraming({
src,
crop,
pan,
+ background,
disabled,
hint,
onFrameLive,
@@ -3267,12 +3316,45 @@ function WebcamFraming({
src: string | null;
crop: { x: number; y: number; width: number; height: number };
pan: { x: number; y: number };
+ /** The camera background to show on the thumbnail, `null` for the raw camera. */
+ background: {
+ mode: Exclude<WebcamBackgroundMode, "none">;
+ blurIntensity: number;
+ wallpaper: string;
+ } | null;
disabled: boolean;
hint: string;
onFrameLive: (size: number, pan: { x: number; y: number }) => void;
onCommit: () => void;
}) {
const boxRef = useRef<HTMLDivElement | null>(null);
+ const videoRef = useRef<HTMLVideoElement | null>(null);
+ // The source whose chosen frame is on screen, the one the cutout is taken from.
+ const [seekedSrc, setSeekedSrc] = useState<string | null>(null);
+ // Segmented once per source: the mask depends on the picture alone, so changing the mode,
+ // the blur or the wallpaper afterwards is plain CSS over it. Nothing runs while the
+ // background is "none".
+ const [cutout, setCutout] = useState<{ src: string; mask: string; backdrop: string } | null>(
+ null,
+ );
+ const wantsCutout = background !== null;
+ useEffect(() => {
+ const video = videoRef.current;
+ if (!wantsCutout || !video || src === null || seekedSrc !== src) return;
+ let current = true;
+ cutOutSubject(video)
+ .then((result) => {
+ if (current && result) setCutout({ src, ...result });
+ })
+ .catch((err) => console.warn("[webcam-framing] segmentation failed:", err));
+ return () => {
+ current = false;
+ };
+ }, [wantsCutout, src, seekedSrc]);
+ const shown = background && cutout?.src === src ? cutout : null;
+ // The shader's blur radius (`blur_webcam_bg`) as a CSS deviation, at the scale of a typical
+ // PiP bubble: the thumbnail cannot know the real one, so this is close, not exact.
+ const blurPx = background ? (background.blurIntensity * 22 + 1.5) / 4 : 0;
const gestureRef = useRef<
| { kind: "move"; x: number; y: number; pan: { x: number; y: number } }
| { kind: "resize"; anchor: { x: number; y: number }; corner: FrameCorner }
@@ -3345,11 +3427,32 @@ function WebcamFraming({
return (
<div className={styles.framing}>
<div ref={boxRef} className={styles.framingBox} style={{ aspectRatio: aspect }}>
+ {shown && background?.mode === "blur" ? (
+ // Oversized by the blur's reach, so its faded edge falls outside the box.
+ <div
+ aria-hidden="true"
+ style={{
+ position: "absolute",
+ inset: -2 * blurPx,
+ background: `center / cover no-repeat url(${shown.backdrop})`,
+ filter: `blur(${blurPx}px)`,
+ }}
+ />
+ ) : shown && background?.mode === "custom" ? (
+ <div
+ aria-hidden="true"
+ style={{ position: "absolute", inset: 0, ...wallpaperStyle(background.wallpaper) }}
+ />
+ ) : null}
{src ? (
<video
+ ref={videoRef}
className={styles.framingVideo}
src={src}
- style={{ visibility: ready ? "visible" : "hidden" }}
+ style={{
+ visibility: ready ? "visible" : "hidden",
+ ...(shown ? { maskImage: `url(${shown.mask})`, maskSize: "100% 100%" } : {}),
+ }}
muted
playsInline
preload="metadata"
@@ -3362,6 +3465,7 @@ function WebcamFraming({
// A frame from the take rather than its first one, which is often black.
video.currentTime = Math.min(1, (video.duration || 0) / 2);
}}
+ onSeeked={() => setSeekedSrc(src)}
/>
) : null}
<div
diff --git a/src/lib/wallpaper.ts b/src/lib/wallpaper.ts
index 5bf57e2d2..1db7e91fd 100644
--- a/src/lib/wallpaper.ts
+++ b/src/lib/wallpaper.ts
@@ -1,3 +1,4 @@
+import type { CSSProperties } from "react";
import { getAssetPath } from "@/lib/assetPath";
export const WALLPAPER_COUNT = 18;
@@ -55,6 +56,31 @@ export function classifyWallpaper(value: string): WallpaperClassification {
return { kind: "color", value: trimmed };
}
+/**
+ * `value` as a CSS background that fills its box, for the preview frame and the camera crop
+ * thumbnail. Image wallpapers must go through `resolveImageWallpaperUrl` (→ getAssetPath): in
+ * the packaged Electron app the renderer loads over file://, where a bare `/wallpapers/foo.jpg`
+ * points at the filesystem root and 404s, so the background silently failed to paint (worked in
+ * the http dev server only). An image that does not resolve paints nothing.
+ */
+export function wallpaperStyle(value: string): CSSProperties {
+ const w = classifyWallpaper(value);
+ if (w.kind === "color") return { backgroundColor: w.value };
+ if (w.kind === "gradient") return { backgroundImage: w.value, backgroundSize: "cover" };
+ let url: string;
+ try {
+ url = resolveImageWallpaperUrl(w.path);
+ } catch {
+ return {};
+ }
+ return {
+ backgroundImage: `url(${url})`,
+ backgroundSize: "cover",
+ backgroundPosition: "center",
+ backgroundRepeat: "no-repeat",
+ };
+}
+
function extractTrailingGradient(value: string): string | null {
const match = value.match(/,\s*((?:repeating-)?(?:linear|radial|conic)-gradient\(.*\))\s*$/);
return match?.[1] ?? null;
diff --git a/src/native/compositorViewClient.ts b/src/native/compositorViewClient.ts
index 3898a4f44..793521086 100644
--- a/src/native/compositorViewClient.ts
+++ b/src/native/compositorViewClient.ts
@@ -58,6 +58,21 @@ export async function probeSegmentationSupport(): Promise<SegmentationSupport> {
}
}
+/** The segmentation model's input size (`segmentation::MODEL_WIDTH` x `MODEL_HEIGHT`). */
+export const SEGMENTATION_WIDTH = 256;
+export const SEGMENTATION_HEIGHT = 144;
+
+/** Subject mask for one camera frame: `rgba` is SEGMENTATION_WIDTH x SEGMENTATION_HEIGHT
+ * RGBA8, the mask one byte per pixel, 255 = subject. `null` when the addon cannot do it. */
+export async function segmentCameraFrame(rgba: Uint8Array): Promise<Uint8Array | null> {
+ const { mask } = await requireNativeBridgeData<{ mask: Uint8Array | null }>({
+ domain: "compositor",
+ action: "segmentFrame",
+ payload: { rgba },
+ });
+ return mask;
+}
+
export function createCompositorView(
rect: CompositorViewRect,
sources?: { screenPath?: string; webcamPath?: string; cursorPath?: string },
diff --git a/src/native/contracts.ts b/src/native/contracts.ts
index 6b1cec409..a2115e7f6 100644
--- a/src/native/contracts.ts
+++ b/src/native/contracts.ts
@@ -719,6 +719,13 @@ export type NativeBridgeRequest =
action: "probeSegmentation";
requestId?: string;
}
+ | {
+ domain: "compositor";
+ action: "segmentFrame";
+ /** One camera frame, 256x144 RGBA8 — the segmentation model's input size. */
+ payload: { rgba: Uint8Array };
+ requestId?: string;
+ }
| {
domain: "compositor";
action: "setRect";