Code-first video & image, rendered natively on WebGPU.
/**
* A `withVision({ mode: "matte" })` layer of a clip that `comp.vision` already
* analyses reuses the clip's per-frame subject instead of segmenting again.
*/
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import {
CLIP_SUBJECT,
type InstanceMask,
trimapFromMask,
type VisionRunner,
} from "@framefields/vision";
import { describe, expect, it } from "vitest";
import { type ClipVision, seedClipVision } from "./clip-vision.js";
import { Composition, HeadlessMediaRenderer, Layer } from "./index.js";
import { decodePngPixels, getPixel } from "./test-utils/pixels.js";
const FPS = 30;
const SIZE = 200;
function stage(background = "#000000") {
return new Composition({
width: SIZE,
height: SIZE,
fps: FPS,
durationFrames: 10,
backgroundColor: background,
});
}
/** A 10-frame solid green clip. */
async function greenClip(): Promise<string> {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "ff-clip-matte-"));
const file = path.join(dir, "green.mp4");
const c = stage();
c.add(
Layer.box({
position: "absolute",
x: 0,
y: 0,
width: SIZE,
height: SIZE,
background: "#00ff00",
}),
);
const r = await c.renderVideo({ outputPath: file, quality: "lossless" });
await r.cleanup?.();
return file;
}
/** Subject: the left half of the frame. */
function leftHalf(): Uint8Array {
const mask = new Uint8Array(SIZE * SIZE);
for (let y = 0; y < SIZE; y++) mask.fill(255, y * SIZE, y * SIZE + SIZE / 2);
return mask;
}
const cutout = (clip: string, vision: Record<string, unknown> = {}) =>
Layer.video(clip, {
position: "absolute",
x: 0,
y: 0,
width: SIZE,
height: SIZE,
fit: "cover",
muted: true,
}).withVision({
mode: "matte",
// Unreachable: any segmentation pass during render would fail.
baseUrl: "http://127.0.0.1:9/",
...vision,
});
describe("matte reuses comp.vision", () => {
it("fails clearly when an explicit prompted mask cannot align with the layer aspect", async () => {
const file = await greenClip();
const c = stage();
const layer = Layer.video(file, {
width: SIZE,
height: SIZE / 2,
fit: "cover",
muted: true,
}).withVision({ mode: "matte" });
c.add(layer);
const clip = c.vision(layer, {
cacheDir: false,
sam2: {
prompts: [{ frame: 0, points: [{ x: 0.25, y: 0.5, label: 1 }] }],
},
});
seedClipVision(
clip,
{ width: SIZE, height: SIZE },
Array.from({ length: 40 }, () => ({
kp: null,
mask: null,
subject: leftHalf(),
})),
);
await expect(c.renderFrame({ frame: 0 })).rejects.toThrow(/aspect ratio/);
});
it("renders an authoritative absent subject as empty instead of holding an earlier mask", async () => {
const file = await greenClip();
const c = stage("#0000ff");
const layer = cutout(file);
c.add(layer);
const clip = c.vision(layer, { cacheDir: false, smoothing: 0 });
seedClipVision(
clip,
{ width: SIZE, height: SIZE },
Array.from({ length: 40 }, (_, frame) => ({
kp: null,
mask: null,
subject: frame === 0 ? leftHalf() : null,
})),
);
const renderer = new HeadlessMediaRenderer();
await c.renderFrame({ frame: 0, renderer });
const image = await decodePngPixels(
await c.renderFrame({ frame: 1, renderer }),
);
expect(getPixel(image, 50, 100).b).toBeGreaterThan(240);
expect(getPixel(image, 50, 100).g).toBeLessThan(15);
});
it("follows a clip's subject while its media plays in reverse", async () => {
const file = await greenClip();
const c = stage("#0000ff");
const layer = Layer.video(file, {
position: "absolute",
x: 0,
y: 0,
width: SIZE,
height: SIZE,
durationFrames: 2,
muted: true,
timeRemap: { kind: "rate", sourceStartSec: 2 / FPS, rate: -1 },
}).withVision({ mode: "matte", baseUrl: "http://127.0.0.1:9/" });
c.add(layer);
const clip = c.vision(layer, { cacheDir: false, smoothing: 0 });
const right = leftHalf().map((v) => 255 - v);
seedClipVision(
clip,
{ width: SIZE, height: SIZE },
Array.from({ length: 40 }, (_, i) => ({
kp: null,
mask: null,
subject: i === 2 ? right : leftHalf(),
})),
);
const first = await decodePngPixels(await c.renderFrame({ frame: 0 }));
const next = await decodePngPixels(await c.renderFrame({ frame: 1 }));
expect(getPixel(first, 150, 100).g).toBeGreaterThan(240);
expect(getPixel(next, 50, 100).g).toBeGreaterThan(240);
});
it("uses mapped source time to select the rendered matte", async () => {
const file = await greenClip();
const c = stage("#0000ff");
const layer = Layer.video(file, {
position: "absolute",
x: 0,
y: 0,
width: SIZE,
height: SIZE,
durationFrames: 4,
muted: true,
timeRemap: {
kind: "keyframes",
keyframes: [
{ frame: 0, sourceSec: 1 / FPS },
{ frame: 4, sourceSec: 1 / FPS },
],
},
}).withVision({ mode: "matte", baseUrl: "http://127.0.0.1:9/" });
c.add(layer);
const clip = c.vision(layer, { cacheDir: false, smoothing: 0 });
const right = leftHalf().map((v) => 255 - v);
seedClipVision(
clip,
{ width: SIZE, height: SIZE },
Array.from({ length: 40 }, (_, i) => ({
kp: null,
mask: null,
subject: i === 0 ? leftHalf() : right,
})),
);
const img = await decodePngPixels(await c.renderFrame({ frame: 0 }));
expect(getPixel(img, 50, 100).b).toBeGreaterThan(240);
expect(getPixel(img, 150, 100).g).toBeGreaterThan(240);
});
it("links each remapped layer to its own vision clock", () => {
const c = stage();
const first = Layer.video("clip.mp4", {
durationFrames: 10,
timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },
}).withVision({ mode: "matte" });
const second = Layer.video("clip.mp4", {
durationFrames: 10,
timeRemap: { kind: "rate", sourceStartSec: 1, rate: 1 },
}).withVision({ mode: "matte" });
c.add([first, second]);
const a = c.vision(first, { cacheDir: false });
const b = c.vision(second, { cacheDir: false });
(c as unknown as { linkClipLayers(): void }).linkClipLayers();
expect((first.vision as { [CLIP_SUBJECT]?: unknown })[CLIP_SUBJECT]).toBe(
a,
);
expect((second.vision as { [CLIP_SUBJECT]?: unknown })[CLIP_SUBJECT]).toBe(
b,
);
});
it("cuts the layer out with the clip's cached subject, from the first frame", async () => {
const file = await greenClip();
const c = stage("#0000ff");
const clip = c.vision(file, { cacheDir: false, smoothing: 0 });
seedClipVision(
clip,
{ width: SIZE, height: SIZE },
Array.from({ length: 40 }, () => ({
kp: null,
mask: null,
subject: leftHalf(),
})),
);
c.add(cutout(file));
const renderer = new HeadlessMediaRenderer();
for (const f of [0, 1, 5]) {
const img = await decodePngPixels(
await c.renderFrame({ frame: f, renderer }),
);
// Without reuse frame 0 has no subject yet (inference runs one frame behind).
const inside = getPixel(img, 50, 100);
expect(inside.g).toBeGreaterThan(240);
expect(inside.b).toBeLessThan(15);
const outside = getPixel(img, 150, 100);
expect(outside.b).toBeGreaterThan(240);
expect(outside.g).toBeLessThan(15);
}
});
it("links only layers whose settings give the clip's subject", async () => {
const c = stage();
c.vision("clip.mp4", { cacheDir: false });
const same = cutout("clip.mp4");
const classes = cutout("clip.mp4", { classes: ["person"] });
const keyed = cutout("clip.mp4", { keyBackground: true });
const other = cutout("other.mp4");
c.add([same, classes, keyed, other]);
(c as unknown as { linkClipLayers(): void }).linkClipLayers();
const linked = (n: { vision?: object }) =>
CLIP_SUBJECT in (n.vision as object);
expect(linked(same as never)).toBe(true);
expect(linked(classes as never)).toBe(false);
expect(linked(keyed as never)).toBe(false);
expect(linked(other as never)).toBe(false);
});
});
/** Stands in for ViTMatte: every pixel of the trimap's unknown band gets alpha 128. */
function stubMatting(clip: ClipVision): { calls: number } {
const count = { calls: 0 };
const runner: Pick<VisionRunner, "refineMatte" | "close"> = {
refineMatte: async (image, mask, options) => {
count.calls++;
const trimap = trimapFromMask(
mask,
image.width,
image.height,
options?.band ?? 4,
);
return {
mask: trimap.data,
width: image.width,
height: image.height,
nearest: trimap.nearest,
};
},
close: () => undefined,
};
(clip as unknown as { runner: typeof runner }).runner = runner;
return count;
}
const maskAt = (clip: ClipVision, frame: number, id?: number) =>
(
clip as unknown as {
maskAt(frame: number, id?: number): InstanceMask | undefined;
}
).maskAt(frame, id);
describe("matting", () => {
/** Track 1: x < 50; track 2: x ≥ 150. */
function twoTracks(): Uint8Array {
const labels = new Uint8Array(SIZE * SIZE);
for (let y = 0; y < SIZE; y++) {
labels.fill(1, y * SIZE, y * SIZE + 50);
labels.fill(2, y * SIZE + 150, y * SIZE + SIZE);
}
return labels;
}
const prompted = (cacheDir: string) => ({
cacheDir,
matting: { band: 4 },
sam2: { prompts: [{ frame: 0, points: [{ x: 0.1, y: 0.5, label: 1 }] }] },
});
it("gives each track its own soft edge, and caches mattes apart from masks", async () => {
const file = await greenClip();
const cacheDir = await fs.mkdtemp(path.join(os.tmpdir(), "ff-matting-"));
const seed = (clip: ClipVision) =>
seedClipVision(
clip,
{ width: SIZE, height: SIZE },
Array.from({ length: 10 }, () => ({
kp: null,
mask: null,
tracks: twoTracks(),
})),
);
const clip = seed(stage().vision(file, prompted(cacheDir)));
const stub = stubMatting(clip);
await clip.ready(0, 3);
expect(stub.calls).toBe(4);
expect(clip.isReady(3)).toBe(true);
expect(clip.isReady(4)).toBe(false);
const row = 100 * SIZE;
const subject = clip.subject(0, FPS)?.mask;
expect(subject?.[row + 10]).toBe(255);
expect(subject?.[row + 52]).toBe(128);
expect(subject?.[row + 100]).toBe(0);
// The band past track 1's edge belongs to track 1 only, and likewise for track 2.
expect(maskAt(clip, 0, 1)?.mask[row + 52]).toBe(128);
expect(maskAt(clip, 0, 1)?.mask[row + 148]).toBe(0);
expect(maskAt(clip, 0, 2)?.mask[row + 148]).toBe(128);
expect(maskAt(clip, 0, 2)?.mask[row + 52]).toBe(0);
// A second run reads the mattes from disk.
const again = seed(stage().vision(file, prompted(cacheDir)));
const rerun = stubMatting(again);
await again.ready(0, 3);
expect(rerun.calls).toBe(0);
expect(again.subject(0, FPS)?.mask[row + 52]).toBe(128);
// Without matting the same clip keeps its hard masks.
const hard = seed(
stage().vision(file, { ...prompted(cacheDir), matting: false }),
);
await hard.ready(0, 3);
expect(hard.subject(0, FPS)?.mask[row + 52]).toBe(0);
});
it("cuts a matte layer out with the soft edge", async () => {
const file = await greenClip();
const c = stage("#0000ff");
const layer = cutout(file);
c.add(layer);
const clip = c.vision(layer, {
cacheDir: false,
smoothing: 0,
matting: { band: 4 },
});
seedClipVision(
clip,
{ width: SIZE, height: SIZE },
Array.from({ length: 40 }, () => ({
kp: null,
mask: null,
subject: leftHalf(),
})),
);
stubMatting(clip);
const image = await decodePngPixels(await c.renderFrame({ frame: 0 }));
expect(getPixel(image, 50, 100).g).toBeGreaterThan(240);
const edge = getPixel(image, 101, 100);
expect(edge.g).toBeGreaterThan(90);
expect(edge.g).toBeLessThan(170);
expect(edge.b).toBeGreaterThan(90);
expect(getPixel(image, 150, 100).b).toBeGreaterThan(240);
});
});