Code-first video & image, rendered natively on WebGPU.
feat(vision): matte layers reuse comp.vision results for the same clip
okanasl303c410b6bad
diff --git a/.changeset/clip-vision-matte.md b/.changeset/clip-vision-matte.mdnew file mode 100644index 00000000..9c0f836b--- /dev/null+++ b/.changeset/clip-vision-matte.md@@ -0,0 +1,7 @@+---+"framefields": minor+"@framefields/vision": minor+"@framefields/node-vision": minor+---++`withVision({ mode: "matte" })` (and `mask` / `crop`) on a video layer now reuses `comp.vision(clip)`'s per-frame results when the composition already analyses that clip, instead of segmenting every frame a second time during render. The clip keeps each frame's full-resolution subject mask (merged the same way as the matte), persisted beside its other results in the vision cache, and the cutout uses the mask for the frame being drawn, so it no longer runs one frame behind the plate. Layers whose settings would give a different subject (`classes`, `keyBackground`, `matteSource: "selfie"`, a different `variant` / `confidence` / `maskThreshold` / `featherRadius`, or an attached vision bundle) keep their own segmentation, and so do frames the clip has not analysed.diff --git a/nodes/node-vision/src/renderers/webgpu-renderer.ts b/nodes/node-vision/src/renderers/webgpu-renderer.tsindex 9fedcb3d..fb2ddda2 100644--- a/nodes/node-vision/src/renderers/webgpu-renderer.ts+++ b/nodes/node-vision/src/renderers/webgpu-renderer.ts@@ -1,6 +1,8 @@/// <reference types="webgpu" />import type { WebGPUNodeRenderer } from "@framefields/node-sdk";import {+ CLIP_SUBJECT,+ type ClipSubjectSource,type InstanceMask,maskBounds,matchPoseToTracks,@@ -623,12 +625,28 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {op.enableMatte === true || (isMatteMode && op.matteSource === "selfie");const wantsPose = op.enablePose === true || mode === "skeleton";- // 2. Results are cached per source frame (selfie mattes and poses are not,+ // 2. A `comp.vision` clip that already analysed this source frame gives the+ // subject for THIS frame's pixels: no readback, no inference. Undefined+ // means it has nothing for the frame, so the node segments it itself.+ const clipSubject =+ isMatteMode &&+ !visionBundle &&+ !wantsPose &&+ !wantsSelfie &&+ op.keyBackground !== true+ ? clipSubjectFor(effectKey, frameIdx, fps, targetWidth, targetHeight)+ : undefined;+ const fromClip = clipSubject !== undefined;++ // 3. Results are cached per source frame (selfie mattes and poses are not,// so nodes that need them always run inference). A frame analysed before// is composited with its own pixels; otherwise inference runs on the// frame the child texture still holds, one frame behind.const cacheable =- !wantsSelfie && !wantsPose && (wantsDetection || wantsSegmentation);+ !fromClip &&+ !wantsSelfie &&+ !wantsPose &&+ (wantsDetection || wantsSegmentation);const source = cacheable? cacheSource(nodeKeyStr, childMedia, op, targetWidth, targetHeight): "";@@ -648,7 +666,7 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {? heldFrameByNode.get(nodeKeyStr): undefined;let cached = lookup(frameIdx);- const exact = cached !== undefined;+ const exact = fromClip || cached !== undefined;if (!cached) cached = lookup(heldFrame);// Keep the held frame's picture on the GPU: the matte of that frame is@@ -663,12 +681,12 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {]);}- // 3. The held frame, when it still needs inference: copied now (submitted+ // 4. The held frame, when it still needs inference: copied now (submitted// before this frame redraws the child) and analysed on the GPU. Pixels// come back to the CPU only for the opt-in background key.let frame: VisionGpuImage | undefined;let framePixels: Uint8ClampedArray | Uint8Array | null = null;- if (hasPreviousFrame && !cached) {+ if (hasPreviousFrame && !cached && !fromClip) {const analysis = analysisTexture(ctx.device, nodeKeyStr, childTex);const copy = ctx.device.createCommandEncoder({label: "vision_analysis_copy",@@ -690,7 +708,7 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {}}- // 4. Draw THIS frame's child into the persistent texture (recorded into the+ // 5. Draw THIS frame's child into the persistent texture (recorded into the// shared encoder, submitted at end of frame, read back next call).const childView = childTex.createView();const clearPass = ctx.renderer.beginFrame(@@ -713,7 +731,7 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {);heldFrameByNode.set(nodeKeyStr, frameIdx);- // 5. Results: from the cache, or inference on the held frame's pixels+ // 6. Results: from the clip, the cache, or inference on the held frame's pixels// (models download lazily on first use of each task).let frameObjects: readonly TrackedObject[] = [];let frameMasks: readonly InstanceMask[] = [];@@ -721,7 +739,9 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {let currentPoseRes: PoseResult | undefined;let subject: InstanceMask | undefined;- if (cached) {+ if (fromClip) {+ subject = selectSubject(clipSubject ?? undefined, nodeKeyStr, frameIdx);+ } else if (cached) {frameObjects = cached.objects;if (wantsDetection) {visionBundle?.setObjectResult(frameIdx, {@@ -841,7 +861,7 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {}}- // 6. Output to destination texture based on mode+ // 7. Output to destination texture based on mode// Mask-family modes: the picture the results belong to, cut out on the GPU.if (isMatteMode) {@@ -1031,6 +1051,27 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {outPass.end();};+/**+ * The subject a linked `comp.vision` clip holds for a source frame (see+ * CLIP_SUBJECT). Its mask is in source pixels; the composite samples it by+ * uv, so it fits any target of the same aspect. Other shapes fall back.+ */+function clipSubjectFor(+ effect: object | undefined,+ frame: number,+ fps: number,+ width: number,+ height: number,+): InstanceMask | null | undefined {+ const source = (effect as Record<symbol, unknown> | undefined)?.[+ CLIP_SUBJECT+ ] as ClipSubjectSource | undefined;+ const subject = source?.subject(frame, fps);+ if (!subject) return subject;+ const aspect = subject.width / subject.height / (width / height);+ return Math.abs(aspect - 1) < 0.01 ? subject : undefined;+}+/** Adapts a Selfie Segmenter matte to the instance-mask shape the composite path takes. */function personMatteAsMask(matte: PersonMatte | undefined,diff --git a/packages/framefields/src/clip-vision-matte.test.ts b/packages/framefields/src/clip-vision-matte.test.tsnew file mode 100644index 00000000..13a234f1--- /dev/null+++ b/packages/framefields/src/clip-vision-matte.test.ts@@ -0,0 +1,116 @@+/**+ * A `withVision({ mode: "matte" })` layer of a clip that `comp.vision` already+ * analyses reuses the clip's per-frame subject instead of segmenting again.+ */+import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { CLIP_SUBJECT } from "@framefields/vision";+import { describe, expect, it } from "vitest";+import { seedClipVision } from "./clip-vision.js";+import { Composition, HeadlessMediaRenderer, Layer } from "./index.js";+import { decodePngPixels, getPixel } from "./test-utils/pixels.js";++const FPS = 30;+const SIZE = 200;++function stage(background = "#000000") {+ return new Composition({+ width: SIZE,+ height: SIZE,+ fps: FPS,+ durationFrames: 10,+ backgroundColor: background,+ });+}++/** A 10-frame solid green clip. */+async function greenClip(): Promise<string> {+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "ff-clip-matte-"));+ const file = path.join(dir, "green.mp4");+ const c = stage();+ c.add(+ Layer.box({+ position: "absolute",+ x: 0,+ y: 0,+ width: SIZE,+ height: SIZE,+ background: "#00ff00",+ }),+ );+ const r = await c.renderVideo({ outputPath: file, quality: "lossless" });+ await r.cleanup?.();+ return file;+}++/** Subject: the left half of the frame. */+function leftHalf(): Uint8Array {+ const mask = new Uint8Array(SIZE * SIZE);+ for (let y = 0; y < SIZE; y++) mask.fill(255, y * SIZE, y * SIZE + SIZE / 2);+ return mask;+}++const cutout = (clip: string, vision: Record<string, unknown> = {}) =>+ Layer.video(clip, {+ position: "absolute",+ x: 0,+ y: 0,+ width: SIZE,+ height: SIZE,+ fit: "cover",+ muted: true,+ }).withVision({+ mode: "matte",+ // Unreachable: any segmentation pass during render would fail.+ baseUrl: "http://127.0.0.1:9/",+ ...vision,+ });++describe("matte reuses comp.vision", () => {+ it("cuts the layer out with the clip's cached subject, from the first frame", async () => {+ const file = await greenClip();+ const c = stage("#0000ff");+ const clip = c.vision(file, { cacheDir: false, smoothing: 0 });+ seedClipVision(+ clip,+ { width: SIZE, height: SIZE },+ Array.from({ length: 40 }, () => ({+ kp: null,+ mask: null,+ subject: leftHalf(),+ })),+ );+ c.add(cutout(file));+ const renderer = new HeadlessMediaRenderer();+ for (const f of [0, 1, 5]) {+ const img = await decodePngPixels(+ await c.renderFrame({ frame: f, renderer }),+ );+ // Without reuse frame 0 has no subject yet (inference runs one frame behind).+ const inside = getPixel(img, 50, 100);+ expect(inside.g).toBeGreaterThan(240);+ expect(inside.b).toBeLessThan(15);+ const outside = getPixel(img, 150, 100);+ expect(outside.b).toBeGreaterThan(240);+ expect(outside.g).toBeLessThan(15);+ }+ });++ it("links only layers whose settings give the clip's subject", async () => {+ const c = stage();+ c.vision("clip.mp4", { cacheDir: false });+ const same = cutout("clip.mp4");+ const classes = cutout("clip.mp4", { classes: ["person"] });+ const keyed = cutout("clip.mp4", { keyBackground: true });+ const other = cutout("other.mp4");+ c.add([same, classes, keyed, other]);+ (c as unknown as { linkClipMattes(): void }).linkClipMattes();+ const linked = (n: { vision?: object }) =>+ CLIP_SUBJECT in (n.vision as object);+ expect(linked(same as never)).toBe(true);+ expect(linked(classes as never)).toBe(false);+ expect(linked(keyed as never)).toBe(false);+ expect(linked(other as never)).toBe(false);+ });+});diff --git a/packages/framefields/src/clip-vision.test.ts b/packages/framefields/src/clip-vision.test.tsindex 8ba828d0..2ef6a822 100644--- a/packages/framefields/src/clip-vision.test.ts+++ b/packages/framefields/src/clip-vision.test.ts@@ -67,6 +67,25 @@ describe("ClipVision", () => {expect(edge.gx).toBeLessThan(0);});+ it("gives render-time mattes the full-resolution subject per source frame", () => {+ const subject = new Uint8Array(160 * 80);+ subject.fill(200, 0, 160 * 10);+ const c = clipVisionFromResults(24, { width: 160, height: 80, cell: 4 }, [+ { kp: null, mask: null, subject },+ { kp: null, mask: null, subject: null },+ { kp: null, mask: null },+ ]);+ const s = c.subject(0, 24);+ expect(s).toMatchObject({ width: 160, height: 80, category: "person" });+ expect(s?.mask).toEqual(subject);+ expect(s?.area).toBe(Math.round((200 * 160 * 10) / 255));+ // Source frame 2 at 48 fps is clip frame 1: analysed, nobody there.+ expect(c.subject(2, 48)).toBeNull();+ // Not analysed with subjects, or between clip frames: the caller segments.+ expect(c.subject(2, 24)).toBeUndefined();+ expect(c.subject(1, 48)).toBeUndefined();+ });+it("explains a read of a frame that was never analysed", () => {expect(() => clip(2).person.point("leftWrist", 30)).toThrow(/not analysed yet/,diff --git a/packages/framefields/src/clip-vision.ts b/packages/framefields/src/clip-vision.tsindex f57e17b8..2541978e 100644--- a/packages/framefields/src/clip-vision.ts+++ b/packages/framefields/src/clip-vision.ts@@ -1,5 +1,11 @@import { type FrameSignal, Signal } from "@framefields/core";-import { COCO17_KEYPOINT_NAMES, VisionRunner } from "@framefields/vision";+import {+ type ClipSubjectSource,+ COCO17_KEYPOINT_NAMES,+ type InstanceMask,+ mergeSubjectMask,+ VisionRunner,+} from "@framefields/vision";import { mediaDecoderCache } from "@framefields/webgpu-renderers";import type { VisionProps } from "./effects/index.js";@@ -48,6 +54,15 @@ interface FrameResult {mask: Uint8Array | null;}+/** A frame's merged subject at source resolution, run-length encoded. */+interface SubjectResult {+ category: string;+ area: number;+ /** (value, run length) pairs over the source-sized 0–255 mask. */+ values: Uint8Array;+ lengths: Uint32Array;+}+interface Transform {sx: number;sy: number;@@ -70,15 +85,17 @@ const CACHE_VERSION = 1;* export fill the cache as they play; nothing has to be analysed up front.** Results depend only on the clip and settings, so they persist on disk in- * Node: a second run, or a render after a preview, reuses them.+ * Node: a second run, or a render after a preview, reuses them. A matte-mode+ * `withVision` layer of the same clip reuses them too (`keepSubjects`).*/-export class ClipVision {+export class ClipVision implements ClipSubjectSource {readonly source: string;readonly fps: number;readonly startFrame: number;readonly trimStartSec: number;readonly smoothing: number;- private readonly options: ClipVisionOptions;+ /** @internal */+ readonly options: ClipVisionOptions;private readonly results = new Map<number, FrameResult>();private sourceWidth = 0;private sourceHeight = 0;@@ -89,9 +106,14 @@ export class ClipVision {private runner?: VisionRunner;private cacheLoaded?: Promise<void>;private unsaved = 0;+ private unsavedSubjects = 0;private readonly blurred = new Map<number, Float32Array | null>();private readonly smoothed = new Map<string, [number, number, number]>();private personView?: PersonTrack;+ /** Full-resolution subject masks for render-time mattes (see `keepSubjects`). */+ private readonly subjects = new Map<number, SubjectResult | null>();+ private wantSubjects = false;+ private subjectsLoaded?: Promise<void>;/** @internal Use `comp.vision(source, options)`. */constructor(source: string, fps: number, options: ClipVisionOptions = {}) {@@ -128,6 +150,39 @@ export class ClipVision {return true;}+ /**+ * @internal Also keep each frame's full-resolution subject mask, merged+ * like the Vision effect's matte, so a matte layer of this clip reuses it+ * instead of segmenting the frame again (framefields links them).+ */+ keepSubjects(): void {+ this.wantSubjects = true;+ }++ /**+ * @internal The subject at a source frame of the clip (`frame` at `fps`),+ * for a render-time matte: undefined until that frame is analysed with+ * subjects kept, null when it holds no subject.+ */+ subject(frame: number, fps: number): InstanceMask | null | undefined {+ const c = (frame * this.fps) / fps;+ if (Math.abs(c - Math.round(c)) > 1e-6) return undefined;+ const s = this.subjects.get(Math.round(c));+ if (s === undefined) return s;+ if (s === null || !this.sourceWidth) return null;+ const width = this.sourceWidth;+ const height = this.sourceHeight;+ return {+ category: s.category,+ mask: decodeRuns(s, width * height),+ width,+ height,+ area: s.area,+ coverage: s.area / (width * height),+ detectionIndex: -1,+ };+ }+/** Composition frame → clip frame (fractional). */clipFrame(frame: number): number {return frame - this.startFrame + this.trimStartSec * this.fps;@@ -142,11 +197,19 @@ export class ClipVision {];}+ private analysed(c: number): boolean {+ return this.results.has(c) && (!this.wantSubjects || this.subjects.has(c));+ }+private async compute(from: number, to: number): Promise<void> {this.cacheLoaded ??= this.loadCache();await this.cacheLoaded;+ if (this.wantSubjects) {+ this.subjectsLoaded ??= this.loadSubjects();+ await this.subjectsLoaded;+ }let missing = false;- for (let c = from; c <= to; c++) if (!this.results.has(c)) missing = true;+ for (let c = from; c <= to; c++) if (!this.analysed(c)) missing = true;if (!missing) return;const wantPose = this.options.pose ?? true;@@ -167,16 +230,19 @@ export class ClipVision {"clip-vision",);for (let c = from; c <= to; c++) {- if (this.results.has(c)) continue;+ if (this.analysed(c)) continue;const image = await decodedPixels(await decoder.getFrame(c / this.fps));if (!image) {this.results.set(c, { kp: null, mask: null });+ if (this.wantSubjects) this.subjects.set(c, null);continue;}if (!this.sourceWidth) this.setSize(image.width, image.height);- let kp: Float32Array | null = null;- let mask: Uint8Array | null = null;- if (wantPose) {+ // Analysed before without subjects: only segment it again.+ const known = this.results.get(c);+ let kp: Float32Array | null = known?.kp ?? null;+ let mask: Uint8Array | null = known?.mask ?? null;+ if (wantPose && !known) {const person = (await this.runner.pose(image)).people[0];if (person) {kp = new Float32Array(51);@@ -185,8 +251,26 @@ export class ClipVision {}}}- if (wantMask) {- const silhouette = personSilhouette(await this.runner.segment(image));+ const seg =+ (wantMask && !known) || this.wantSubjects+ ? await this.runner.segment(image)+ : undefined;+ if (seg && this.wantSubjects) {+ const subject = mergeSubjectMask(seg.masks);+ this.subjects.set(+ c,+ subject+ ? {+ category: subject.category,+ area: subject.area,+ ...encodeRuns(subject.mask),+ }+ : null,+ );+ this.unsavedSubjects++;+ }+ if (seg && wantMask && !known) {+ const silhouette = personSilhouette(seg);if (silhouette) {mask = new Uint8Array(this.cellW * this.cellH);downsampleMean(@@ -201,9 +285,12 @@ export class ClipVision {}}this.results.set(c, { kp, mask });- if (++this.unsaved >= 24) await this.saveCache();+ if (!known) this.unsaved++;+ if (this.unsaved >= 24) await this.saveCache();+ if (this.unsavedSubjects >= 24) await this.saveSubjects();}if (this.unsaved > 0) await this.saveCache();+ if (this.unsavedSubjects > 0) await this.saveSubjects();}private setSize(width: number, height: number): void {@@ -384,6 +471,88 @@ export class ClipVision {);fs.renameSync(tmp, file);}++ /** Subjects live beside the results: large, and only matte layers read them. */+ private async subjectsPath(): Promise<string | null> {+ const file = await this.cachePath();+ return file ? file.replace(/\.json$/, ".subjects.json") : null;+ }++ private async loadSubjects(): Promise<void> {+ const file = await this.subjectsPath();+ if (!file) return;+ try {+ const fs = await import("node:fs");+ if (!fs.existsSync(file)) return;+ const json = JSON.parse(fs.readFileSync(file, "utf8")) as {+ width: number;+ height: number;+ frames: Record<+ string,+ {+ category: string;+ area: number;+ values: string;+ lengths: string;+ } | null+ >;+ };+ if (this.sourceWidth && json.width !== this.sourceWidth) return;+ if (!this.sourceWidth) this.setSize(json.width, json.height);+ for (const [c, s] of Object.entries(json.frames)) {+ const lengths = Buffer.from(s?.lengths ?? "", "base64");+ this.subjects.set(+ Number(c),+ s && {+ category: s.category,+ area: s.area,+ values: new Uint8Array(Buffer.from(s.values, "base64")),+ lengths: new Uint32Array(+ lengths.buffer.slice(+ lengths.byteOffset,+ lengths.byteOffset + lengths.byteLength,+ ),+ ),+ },+ );+ }+ } catch {+ // A damaged cache is recomputed.+ }+ }++ private async saveSubjects(): Promise<void> {+ this.unsavedSubjects = 0;+ const file = await this.subjectsPath();+ if (!file || !this.sourceWidth) return;+ const frames: Record<+ string,+ { category: string; area: number; values: string; lengths: string } | null+ > = {};+ for (const [c, s] of this.subjects) {+ frames[c] = s && {+ category: s.category,+ area: s.area,+ values: Buffer.from(s.values).toString("base64"),+ lengths: Buffer.from(+ s.lengths.buffer,+ s.lengths.byteOffset,+ s.lengths.byteLength,+ ).toString("base64"),+ };+ }+ const fs = await import("node:fs");+ const tmp = `${file}.${process.pid}.tmp`;+ fs.writeFileSync(+ tmp,+ JSON.stringify({+ width: this.sourceWidth,+ height: this.sourceHeight,+ frames,+ }),+ );+ fs.renameSync(tmp, file);+ }}/**@@ -628,6 +797,42 @@ function downsampleMean(}}+/** A mask as (value, run length) pairs: long flat runs with soft edges. */+function encodeRuns(mask: Uint8Array): {+ values: Uint8Array;+ lengths: Uint32Array;+} {+ let runs = 0;+ for (let i = 1; i <= mask.length; i++)+ if (i === mask.length || mask[i] !== mask[i - 1]) runs++;+ const values = new Uint8Array(runs);+ const lengths = new Uint32Array(runs);+ let r = 0;+ let start = 0;+ for (let i = 1; i <= mask.length; i++) {+ if (i === mask.length || mask[i] !== mask[start]) {+ values[r] = mask[start];+ lengths[r++] = i - start;+ start = i;+ }+ }+ return { values, lengths };+}++function decodeRuns(+ runs: { values: Uint8Array; lengths: Uint32Array },+ size: number,+): Uint8Array {+ const out = new Uint8Array(size);+ let at = 0;+ for (let r = 0; r < runs.values.length; r++) {+ const end = Math.min(size, at + runs.lengths[r]);+ if (runs.values[r] !== 0) out.fill(runs.values[r], at, end);+ at = end;+ }+ return out;+}+/** Separable box blur to 0–1, so gradients reach a little past the edge. */function boxBlur(src: Uint8Array,@@ -668,11 +873,18 @@ function boxBlur(return dst;}+type SeedFrame = {+ kp: number[] | null;+ mask: Uint8Array | null;+ /** Source-sized subject mask for render-time mattes; null = no subject. */+ subject?: Uint8Array | null;+};+/** @internal Test hook: a clip whose results are given instead of computed. */export function clipVisionFromResults(fps: number,size: { width: number; height: number; cell: number },- frames: Array<{ kp: number[] | null; mask: Uint8Array | null }>,+ frames: SeedFrame[],options: ClipVisionOptions = {},): ClipVision {const clip = new ClipVision("memory", fps, {@@ -680,9 +892,19 @@ export function clipVisionFromResults(maskCell: size.cell,cacheDir: false,});+ return seedClipVision(clip, size, frames);+}++/** @internal Test hook: gives a clip's per-frame results instead of computing them. */+export function seedClipVision(+ clip: ClipVision,+ size: { width: number; height: number },+ frames: SeedFrame[],+): ClipVision {const c = clip as unknown as {setSize(w: number, h: number): void;results: Map<number, FrameResult>;+ subjects: Map<number, SubjectResult | null>;};c.setSize(size.width, size.height);frames.forEach((f, i) => {@@ -690,6 +912,17 @@ export function clipVisionFromResults(kp: f.kp ? Float32Array.from(f.kp) : null,mask: f.mask,});+ if (f.subject === undefined) return;+ let area = 0;+ for (const v of f.subject ?? []) area += v;+ c.subjects.set(+ i,+ f.subject && {+ category: "person",+ area: Math.round(area / 255),+ ...encodeRuns(f.subject),+ },+ );});return clip;}diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.tsindex 9239e8b0..998bb36b 100644--- a/packages/framefields/src/index.ts+++ b/packages/framefields/src/index.ts@@ -162,6 +162,7 @@ import {} from "@framefields/core";import {analyzeSequence as analyzeVisionFrames,+ CLIP_SUBJECT,type LandmarkCoordinateSignals,type MaskTrackSignals,type ObjectTrackSignals,@@ -1179,11 +1180,80 @@ export class Composition {* frame — physics, particles — always find every frame before it.*/public async prepareVision(frame: number): Promise<void> {+ this.linkClipMattes();await Promise.all([...this._clipVision.values()].map((clip) => clip.ready(0, frame)),);}+ /**+ * Points matte-family `withVision` layers at the `comp.vision` clip they+ * show, so the cutout reuses that clip's per-frame subject instead of+ * segmenting the frame again during render. Only layers whose settings+ * would give the same subject are linked; the rest keep their own pass.+ */+ private linkClipMattes(): void {+ if (this._clipVision.size === 0) return;+ const walk = (node: unknown): void => {+ if (!node || typeof node !== "object") return;+ const n = node as {+ kind?: string;+ dataType?: string;+ inputHandleId?: unknown;+ vision?: unknown;+ children?: unknown;+ };+ if (+ n.kind === "media" &&+ n.dataType === "Video" &&+ typeof n.inputHandleId === "string" &&+ n.vision instanceof Vision &&+ !(CLIP_SUBJECT in n.vision)+ ) {+ const clip = this.clipForMatte(n.inputHandleId, n.vision);+ if (clip) {+ clip.keepSubjects();+ Object.defineProperty(n.vision, CLIP_SUBJECT, {+ value: clip,+ enumerable: false,+ configurable: true,+ });+ }+ }+ if (Array.isArray(n.children)) for (const c of n.children) walk(c);+ };+ for (const child of this.children) walk(child);+ }++ private clipForMatte(source: string, v: Vision): ClipVision | undefined {+ const mode = v.mode ?? "passthrough";+ if (mode !== "matte" && mode !== "mask" && mode !== "crop") return;+ // Anything that changes the subject, or needs results the clip lacks.+ if (+ v.matteSource === "selfie" ||+ v.keyBackground === true ||+ v.classes !== undefined ||+ v.enablePose === true ||+ v.enableMatte === true ||+ v.visionBundle !== undefined ||+ (v.maskThreshold ?? 0.5) !== 0.5 ||+ v.featherRadius !== undefined+ ) {+ return;+ }+ for (const clip of this._clipVision.values()) {+ const o = clip.options;+ if (+ clip.source === source &&+ (o.variant ?? "s") === (v.variant ?? "s") &&+ (o.confidence ?? 0.3) === (v.confidence ?? 0.3)+ ) {+ return clip;+ }+ }+ return undefined;+ }+/*** One-shot, ffmpeg-free vision analysis of a media source.*diff --git a/packages/vision/src/index.ts b/packages/vision/src/index.tsindex dcd330d8..a60109da 100644--- a/packages/vision/src/index.ts+++ b/packages/vision/src/index.ts@@ -13,6 +13,7 @@ export * from "./runner/canonical-baselines.js";export * from "./runner/vision-runner.js";export * from "./runtime/node-webgpu-provider.js";export * from "./runtime/session-provider.js";+export * from "./segmentation/clip-subject.js";export * from "./segmentation/subject.js";export * from "./signals/vision-bundle.js";export * from "./spatial/camera-space-transformer.js";diff --git a/packages/vision/src/segmentation/clip-subject.ts b/packages/vision/src/segmentation/clip-subject.tsnew file mode 100644index 00000000..41471e8c--- /dev/null+++ b/packages/vision/src/segmentation/clip-subject.ts@@ -0,0 +1,20 @@+import type { InstanceMask } from "../types.js";++/**+ * Where a render-time Vision effect finds subject masks someone already+ * computed for its clip (`comp.vision(clip)` in framefields), so a matte layer+ * reuses them instead of segmenting the frame again. Set on the Vision effect+ * instance under this key; the renderer reads it from the effect.+ */+export const CLIP_SUBJECT: unique symbol = Symbol.for(+ "framefields.vision.clipSubject",+) as never;++export interface ClipSubjectSource {+ /**+ * The merged subject mask for a source frame of the clip (`frame` at `fps`),+ * in source pixels; `null` when the frame was analysed and holds no subject;+ * `undefined` when it was not analysed, so the caller segments it itself.+ */+ subject(frame: number, fps: number): InstanceMask | null | undefined;+}diff --git a/plugins/framefields/skills/framefields-effects/SKILL.md b/plugins/framefields/skills/framefields-effects/SKILL.mdindex 13ff997e..afd9d0bf 100644--- a/plugins/framefields/skills/framefields-effects/SKILL.md+++ b/plugins/framefields/skills/framefields-effects/SKILL.md@@ -178,7 +178,7 @@ Inference runs on the GPU through `onnxruntime-web`'s WebGPU backend (an optiona### How it behaves at render time-- **One-frame delay.** Vision reads the layer's *previous* rendered frame, so the very first frame has no results (a cutout renders transparent) and masks trail the plate by one frame. This is invisible at normal playback speed.+- **One-frame delay.** Vision reads the layer's *previous* rendered frame, so the very first frame has no results (a cutout renders transparent) and masks trail the plate by one frame. This is invisible at normal playback speed. A matte of a clip the composition also reads with `comp.vision` has neither (see Clip Vision below).- **Render vision frames in order.** `renderVideo` does this for you. For stills, render at least two consecutive frames with the same renderer and keep the last one. Jumping straight to a later frame (or using a frame grid) cuts out pixels from whatever frame was rendered before it, which shows up as a ghosted double of the subject.- **Subject choice.** `mask` / `matte` / `crop` use the largest person; with no person, the most confident instance. Instances overlapping the subject and no more than twice its size are merged in (a dress, a held instrument) — large containers around it (a tunnel, a window frame) are not. `classes` limits what the subject can be, not what merges into it: COCO has no clothing class, so a flowing dress often comes back as a "kite" or "umbrella" and still joins a `classes: ["person"]` cutout.- Each vision layer tracks objects independently; track ids from two layers are unrelated.@@ -256,6 +256,7 @@ await dancer.ready(0); // analyse a frame (or a range: ready(0, 120)- The composition analyses each frame's clips just before drawing it, in every render path and the preview, and a live physics world (`world.live(comp)`) before stepping. Nothing runs up front; a preview starts at once.- Results persist in Node (`~/.cache/framefields/vision`, or `$FRAMEFIELDS_VISION_CACHE_DIR`; `cacheDir: false` to opt out), keyed by the file and settings, so later runs and renders after a preview are instant.- Options: `startFrame` / `trimStartSec` (where the clip plays), `pose`, `mask`, `maskCell` (silhouette grid, default 8 px), `smoothing` (frames, default 2), `variant`. When the person is lost the last pose is held for up to a second.+- A `withVision({ mode: "matte" })` cutout (or `mask` / `crop`) of the same clip reuses these results: the composition hands it the clip's subject for the frame being drawn, so it is segmented once, not again at render, and the cutout lines up with the plate on the same frame. Layers with `classes`, `keyBackground`, `matteSource: "selfie"` or other model settings than the clip keep their own pass.- Analysis costs about 200 ms per frame on CPU (segmentation dominates); on WebGPU it is faster. In a live preview the playhead waits for frames not analysed yet.### 6. Analyze Before Authoring