Code-first video & image, rendered natively on WebGPU.
feat(vision): skeleton layers draw the pose matched to a comp.vision clip
okanasla1e37b8659b5
diff --git a/.changeset/skeleton-clip-pose.md b/.changeset/skeleton-clip-pose.mdnew file mode 100644index 00000000..20e7bd01--- /dev/null+++ b/.changeset/skeleton-clip-pose.md@@ -0,0 +1,7 @@+---+"@framefields/vision": minor+"@framefields/node-vision": minor+"framefields": minor+---++A `withVision({ mode: "skeleton" })` layer of a clip that `comp.vision` analyses now draws the pose that clip matched to its subject, for the frame on screen, instead of running its own pass and drawing the first person detected. A skeleton layer of a SAM clip needs `pose: true` on the clip and fails clearly without it. Skeleton mode draws only the skeleton: a frame without a pose is empty rather than showing the footage. SAM clip caches written before this change claimed pose they did not track; they are invalidated, so SAM clips re-analyse once.diff --git a/nodes/node-vision/src/renderers/webgpu-renderer.ts b/nodes/node-vision/src/renderers/webgpu-renderer.tsindex 055bc4fa..43c5d8b1 100644--- a/nodes/node-vision/src/renderers/webgpu-renderer.ts+++ b/nodes/node-vision/src/renderers/webgpu-renderer.ts@@ -8,6 +8,7 @@ import {matchPoseToTracks,mergeSubjectMask,type PersonMatte,+ type PoseKeypoint,type PoseResult,PoseSkeletonRenderer,readTexturePixels,@@ -644,7 +645,20 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {targetHeight,): undefined;- const fromClip = clipSubject !== undefined;+ // A skeleton of a `comp.vision` clip draws the pose matched to its subject.+ const clipPose =+ mode === "skeleton" && !visionBundle+ ? clipPoseFor(+ effectKey,+ props.sourceTimeSec === undefined+ ? frameIdx+ : props.sourceTimeSec * fps,+ fps,+ targetWidth,+ targetHeight,+ )+ : undefined;+ const fromClip = clipSubject !== undefined || clipPose !== undefined;// 3. Results are cached per source frame (selfie mattes and poses are not,// so nodes that need them always run inference). A frame analysed before@@ -933,30 +947,34 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {if (!sharedSkeletonRenderer) {sharedSkeletonRenderer = new PoseSkeletonRenderer(ctx.device);}- const poseRes = visionBundle?.getPoseResult(frameIdx) ?? currentPoseRes;- const person = poseRes?.people[0];- if (person && person.keypoints.length > 0) {- const skelTex = sharedSkeletonRenderer.renderToTexture(person.keypoints, {+ const keypoints =+ clipPose !== undefined+ ? clipPose+ : (visionBundle?.getPoseResult(frameIdx) ?? currentPoseRes)?.people[0]+ ?.keypoints;+ // Only the skeleton: a frame without a pose stays empty, never the footage.+ const skelPass = ctx.renderer.beginFrame(+ encoder,+ targetView,+ { r: 0, g: 0, b: 0, a: 0 },+ targetWidth,+ targetHeight,+ "clear",+ );+ if (keypoints && keypoints.length > 0) {+ const skelTex = sharedSkeletonRenderer.renderToTexture(keypoints, {width: targetWidth,height: targetHeight,});- const skelPass = ctx.renderer.beginFrame(- encoder,- targetView,- { r: 0, g: 0, b: 0, a: 0 },- targetWidth,- targetHeight,- "clear",- );ctx.renderer.drawTexture(skelPass, skelTex, {x: 0,y: 0,width: targetWidth,height: targetHeight,});- skelPass.end();- return;}+ skelPass.end();+ return;}if (mode === "boxes" || mode === "tracking") {@@ -1074,18 +1092,51 @@ function clipSubjectFor(width: number,height: number,): InstanceMask | null | undefined {- const source = (effect as Record<symbol, unknown> | undefined)?.[- CLIP_SUBJECT- ] as ClipSubjectSource | undefined;+ const source = linkedClip(effect);const subject = source?.subject(frame, fps);- if (!subject) return subject;- const aspect = subject.width / subject.height / (width / height);- if (Math.abs(aspect - 1) < 0.01) return subject;- if (source?.exact)+ if (!subject || !source) return subject;+ return fitsTarget(source, subject.width / subject.height, width / height)+ ? subject+ : undefined;+}++/**+ * The pose a linked `comp.vision` clip matched to its subject for a source+ * frame, normalized to the source; it fits targets of the same aspect.+ */+function clipPoseFor(+ effect: object | undefined,+ frame: number,+ fps: number,+ width: number,+ height: number,+): readonly PoseKeypoint[] | null | undefined {+ const source = linkedClip(effect);+ const pose = source?.pose?.(frame, fps);+ if (!pose || !source) return pose === null ? null : undefined;+ return fitsTarget(source, pose.width / pose.height, width / height)+ ? pose.keypoints+ : undefined;+}++function linkedClip(effect: object | undefined): ClipSubjectSource | undefined {+ return (effect as Record<symbol, unknown> | undefined)?.[CLIP_SUBJECT] as+ | ClipSubjectSource+ | undefined;+}++/** Source results map onto a target of the same aspect; a prompted clip must not fall back. */+function fitsTarget(+ source: ClipSubjectSource,+ sourceAspect: number,+ targetAspect: number,+): boolean {+ if (Math.abs(sourceAspect / targetAspect - 1) < 0.01) return true;+ if (source.exact)throw new Error("A prompted clip matte requires the source aspect ratio. Apply layout cropping after the source matte.",);- return undefined;+ return false;}/** Adapts a Selfie Segmenter matte to the instance-mask shape the composite path takes. */diff --git a/packages/framefields/src/clip-vision-matte.test.ts b/packages/framefields/src/clip-vision-matte.test.tsindex 073bee61..6551f150 100644--- a/packages/framefields/src/clip-vision-matte.test.ts+++ b/packages/framefields/src/clip-vision-matte.test.ts@@ -198,7 +198,7 @@ describe("matte reuses comp.vision", () => {c.add([first, second]);const a = c.vision(first, { cacheDir: false });const b = c.vision(second, { cacheDir: false });- (c as unknown as { linkClipMattes(): void }).linkClipMattes();+ (c as unknown as { linkClipLayers(): void }).linkClipLayers();expect((first.vision as { [CLIP_SUBJECT]?: unknown })[CLIP_SUBJECT]).toBe(a,);@@ -243,7 +243,7 @@ describe("matte reuses comp.vision", () => {const keyed = cutout("clip.mp4", { keyBackground: true });const other = cutout("other.mp4");c.add([same, classes, keyed, other]);- (c as unknown as { linkClipMattes(): void }).linkClipMattes();+ (c as unknown as { linkClipLayers(): void }).linkClipLayers();const linked = (n: { vision?: object }) =>CLIP_SUBJECT in (n.vision as object);expect(linked(same as never)).toBe(true);diff --git a/packages/framefields/src/clip-vision-skeleton.test.ts b/packages/framefields/src/clip-vision-skeleton.test.tsnew file mode 100644index 00000000..918b2814--- /dev/null+++ b/packages/framefields/src/clip-vision-skeleton.test.ts@@ -0,0 +1,147 @@+/**+ * A `withVision({ mode: "skeleton" })` layer of a clip that `comp.vision`+ * analyses draws the pose the clip matched to its subject, not the first+ * person a render-time pass detects.+ */+import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { CLIP_SUBJECT } from "@framefields/vision";+import { describe, expect, it } from "vitest";+import { seedClipVision } from "./clip-vision.js";+import { Composition, HeadlessMediaRenderer, Layer } from "./index.js";+import { decodePngPixels, getPixel } from "./test-utils/pixels.js";++const FPS = 30;+const SIZE = 200;+const SAM3 = { prompts: [{ frame: 0, text: "person" }] };++function stage() {+ return new Composition({+ width: SIZE,+ height: SIZE,+ fps: FPS,+ durationFrames: 10,+ backgroundColor: "#000000",+ });+}++/** A 10-frame solid green clip. */+async function greenClip(): Promise<string> {+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "ff-clip-skeleton-"));+ const file = path.join(dir, "green.mp4");+ const c = stage();+ c.add(+ Layer.box({+ position: "absolute",+ x: 0,+ y: 0,+ width: SIZE,+ height: SIZE,+ background: "#00ff00",+ }),+ );+ const r = await c.renderVideo({ outputPath: file, quality: "lossless" });+ await r.cleanup?.();+ return file;+}++/** A standing person in the left half: 17 COCO keypoints as [x, y, visibility], source px. */+function leftPerson(): number[] {+ const points: [number, number][] = [+ [50, 30], // nose+ [46, 26],+ [54, 26], // eyes+ [42, 28],+ [58, 28], // ears+ [40, 60],+ [60, 60], // shoulders+ [40, 100],+ [60, 100], // elbows+ [40, 130],+ [60, 130], // wrists+ [42, 120],+ [58, 120], // hips+ [42, 155],+ [58, 155], // knees+ [42, 190],+ [58, 190], // ankles+ ];+ return points.flatMap(([x, y]) => [x, y, 1]);+}++const skeleton = (clip: string) =>+ Layer.video(clip, {+ position: "absolute",+ x: 0,+ y: 0,+ width: SIZE,+ height: SIZE,+ fit: "cover",+ muted: true,+ }).withVision({+ mode: "skeleton",+ // Unreachable: any pose pass during render would fail.+ baseUrl: "http://127.0.0.1:9/",+ });++function litPixels(+ image: Awaited<ReturnType<typeof decodePngPixels>>,+ x0: number,+ x1: number,+): number {+ let lit = 0;+ for (let y = 0; y < SIZE; y += 2)+ for (let x = x0; x < x1; x += 2) {+ const p = getPixel(image, x, y);+ if (p.r + p.g + p.b > 60) lit++;+ }+ return lit;+}++describe("skeleton reuses comp.vision", () => {+ it("draws the clip's matched pose from the first frame, and nothing where it has none", async () => {+ const file = await greenClip();+ const c = stage();+ const layer = skeleton(file);+ c.add(layer);+ const clip = c.vision(layer, { cacheDir: false, sam3: SAM3, pose: true });+ seedClipVision(+ clip,+ { width: SIZE, height: SIZE },+ Array.from({ length: 40 }, (_, frame) => ({+ kp: frame === 1 ? null : leftPerson(),+ mask: null,+ subject: null,+ })),+ );+ const renderer = new HeadlessMediaRenderer();+ const first = await decodePngPixels(+ await c.renderFrame({ frame: 0, renderer }),+ );+ expect(litPixels(first, 0, 100)).toBeGreaterThan(20);+ // Only the skeleton: the green footage never shows through.+ expect(litPixels(first, 100, SIZE)).toBe(0);+ const empty = await decodePngPixels(+ await c.renderFrame({ frame: 1, renderer }),+ );+ expect(litPixels(empty, 0, SIZE)).toBe(0);+ });++ it("links skeleton layers only to clips that track pose", () => {+ const c = stage();+ const plain = skeleton("clip.mp4");+ c.add(plain);+ c.vision("clip.mp4", { cacheDir: false, pose: false });+ (c as unknown as { linkClipLayers(): void }).linkClipLayers();+ expect(CLIP_SUBJECT in (plain.vision as object)).toBe(false);++ const d = stage();+ const prompted = skeleton("clip.mp4");+ d.add(prompted);+ d.vision("clip.mp4", { cacheDir: false, sam3: SAM3 });+ expect(() =>+ (d as unknown as { linkClipLayers(): void }).linkClipLayers(),+ ).toThrow(/needs `pose: true`/);+ });+});diff --git a/packages/framefields/src/clip-vision.ts b/packages/framefields/src/clip-vision.tsindex 80ffc0f9..8a77ab88 100644--- a/packages/framefields/src/clip-vision.ts+++ b/packages/framefields/src/clip-vision.ts@@ -6,6 +6,7 @@ import {Signal,} from "@framefields/core";import {+ type ClipPose,type ClipSubjectSource,COCO17_KEYPOINT_NAMES,type InstanceMask,@@ -27,7 +28,7 @@ import type { VisionProps } from "./effects/index.js";export type PoseKeypointName = (typeof COCO17_KEYPOINT_NAMES)[number];export interface ClipVisionOptions {- /** Prompted SAM 2.1 (Tiny or Small) video segmentation. Pose is opt-in in this mode. */+ /** Prompted SAM 2.1 (Tiny or Small) video segmentation. Pose is opt-in in this mode, as with sam3. */sam2?: Sam2Options;/** SAM 3 point/box or text selection with temporal tracking. Mutually exclusive with sam2. */sam3?: Sam3Options;@@ -43,7 +44,10 @@ export interface ClipVisionOptions {durationFrames?: number;/** Analysis frame rate (default: the composition's). */fps?: number;- /** Track 17 COCO keypoints (default true). */+ /**+ * Track 17 COCO keypoints (default true; opt-in with sam2/sam3, where each frame's pose+ * is the detected person whose keypoints fall most inside the selection).+ */pose?: boolean;/** Track the person's silhouette, dress and held things included (default true). */mask?: boolean;@@ -341,17 +345,8 @@ export class ClipVision implements ClipSubjectSource {* subjects kept, null when it holds no subject.*/subject(frame: number, fps: number): InstanceMask | null | undefined {- const c = (frame * this.fps) / fps;- if (- !this.exact &&- !this.options.timeRemap &&- Math.abs(c - Math.round(c)) > 1e-6- )- return undefined;- const sourceFrame =- this.options.timeRemap || this.exact- ? Math.floor(c + 1e-6)- : Math.round(c);+ const sourceFrame = this.sourceFrame(frame, fps);+ if (sourceFrame === undefined) return undefined;const s = this.subjects.get(sourceFrame);if (s === undefined) return s;if (s === null || !this.sourceWidth) return null;@@ -368,6 +363,54 @@ export class ClipVision implements ClipSubjectSource {};}+ /**+ * @internal The pose matched to the subject at a source frame of the clip+ * (`frame` at `fps`), normalized to the source size, for a render-time+ * skeleton: undefined until that frame and its smoothing window are+ * analysed, null when no pose matched there.+ */+ pose(frame: number, fps: number): ClipPose | null | undefined {+ const c = this.sourceFrame(frame, fps);+ if (c === undefined || !this.tracksPose || !this.sourceWidth) return;+ for (let g = c - this.smoothing; g <= c + this.smoothing; g++)+ if (!this.results.has(Math.max(0, g))) return undefined;+ if (!this.results.get(c)?.kp) return null;+ const width = this.sourceWidth;+ const height = this.sourceHeight;+ return {+ width,+ height,+ keypoints: Array.from({ length: 17 }, (_, k) => {+ const [x, y, visibility] = this.keypoint(k, c);+ return { x: x / width, y: y / height, visibility };+ }),+ };+ }++ /** Whether frames carry keypoints: on by default, opt-in for prompted (SAM) clips. */+ get tracksPose(): boolean {+ return this.exact+ ? this.options.pose === true+ : (this.options.pose ?? true);+ }++ /**+ * The analysed source frame a render-time read at `frame` (at `fps`) maps to;+ * undefined for an unprompted clip read between its analysed frames.+ */+ private sourceFrame(frame: number, fps: number): number | undefined {+ const c = (frame * this.fps) / fps;+ if (+ !this.exact &&+ !this.options.timeRemap &&+ Math.abs(c - Math.round(c)) > 1e-6+ )+ return undefined;+ return this.options.timeRemap || this.exact+ ? Math.floor(c + 1e-6)+ : Math.round(c);+ }+/** Composition frame → clip frame (fractional). */clipFrame(frame: number): number {if (this.options.timeRemap)@@ -446,7 +489,7 @@ export class ClipVision implements ClipSubjectSource {await this.computeSam3(to);return;}- const wantPose = this.options.pose ?? true;+ const wantPose = this.tracksPose;const wantMask = this.options.mask ?? true;this.runner ??= VisionRunner.create({modelsDir: this.options.modelsDir,@@ -562,7 +605,7 @@ export class ClipVision implements ClipSubjectSource {frame,prompt?.points,);- const kp = this.options.pose+ const kp = this.tracksPose? await this.matchedPose(image, result.mask): null;this.storeSamFrame(frame, result.width, result.height, [result], kp);@@ -615,7 +658,7 @@ export class ClipVision implements ClipSubjectSource {if (!this.sourceWidth) this.setSize(image.width, image.height);const prompt = settings.prompts.find((item) => item.frame === frame);const result = await this.sam3Tracker.process(image, frame, prompt);- const kp = this.options.pose+ const kp = this.tracksPose? await this.matchedPose(image, unionMask(result.objects, image)): null;this.storeSamFrame(@@ -930,13 +973,14 @@ export class ClipVision implements ClipSubjectSource {stat.size,stat.mtimeMs,this.fps,- o.pose ?? true,+ this.tracksPose,o.mask ?? true,this.cell,o.variant ?? "s",o.confidence ?? null,- // Prompted subjects are track-ID label maps (format 2).- ...(o.sam2 ? [[sam?.sam2Revision(o.sam2.model), o.sam2, 2]] : []),+ // Prompted subjects are track-ID label maps (format 2); format 3 drops+ // entries keyed as tracking pose when SAM clips left it off by default.+ ...(o.sam2 ? [[sam?.sam2Revision(o.sam2.model), o.sam2, 3]] : []),...(o.sam3 && sam? [[@@ -944,7 +988,7 @@ export class ClipVision implements ClipSubjectSource {await sam.sam3Precision(undefined, o.sam3.precision),),o.sam3,- 2,+ 3,],]: []),diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.tsindex cccb7a7b..840b02d9 100644--- a/packages/framefields/src/index.ts+++ b/packages/framefields/src/index.ts@@ -1335,19 +1335,20 @@ export class Composition {* frame — physics, particles — always find every frame before it.*/public async prepareVision(frame: number): Promise<void> {- this.linkClipMattes();+ this.linkClipLayers();await Promise.all([...this._clipVision.values()].map((clip) => clip.ready(0, frame)),);}/**- * Points matte-family `withVision` layers at the `comp.vision` clip they- * show, so the cutout reuses that clip's per-frame subject instead of- * segmenting the frame again during render. Only layers whose settings- * would give the same subject are linked; the rest keep their own pass.+ * Points matte-family and skeleton `withVision` layers at the `comp.vision`+ * clip they show, so the cutout reuses that clip's per-frame subject and the+ * skeleton draws its matched pose, instead of analysing the frame again+ * during render. Only layers whose settings would give the same subject are+ * linked; the rest keep their own pass.*/- private linkClipMattes(): void {+ private linkClipLayers(): void {if (this._clipVision.size === 0) return;let layers:| Map<@@ -1386,13 +1387,13 @@ export class Composition {}),);const layer = layers.get(n.id);- const clip = this.clipForMatte(n.src ?? n.inputHandleId, n.vision, {+ const clip = this.clipForLayer(n.src ?? n.inputHandleId, n.vision, {trimStartSec: n.trimStartSec,startFrame: layer?.startFrame ?? n.startFrame,timeRemap: layer?.timeRemap,});if (clip) {- clip.keepSubjects();+ if (n.vision.mode !== "skeleton") clip.keepSubjects();Object.defineProperty(n.vision, CLIP_SUBJECT, {value: clip,enumerable: false,@@ -1405,7 +1406,8 @@ export class Composition {for (const child of this.children) walk(child);}- private clipForMatte(+ /** The `comp.vision` clip a matte or skeleton layer of the same source reads instead of analysing. */+ private clipForLayer(source: string,v: Vision,layer: {@@ -1415,17 +1417,20 @@ export class Composition {},): ClipVision | undefined {const mode = v.mode ?? "passthrough";- if (mode !== "matte" && mode !== "mask" && mode !== "crop") return;+ const skeleton = mode === "skeleton";+ if (mode !== "matte" && mode !== "mask" && mode !== "crop" && !skeleton)+ return;// Anything that changes the subject, or needs results the clip lacks.if (- v.matteSource === "selfie" ||- v.keyBackground === true ||v.classes !== undefined ||- v.enablePose === true ||v.enableMatte === true ||v.visionBundle !== undefined ||- (v.maskThreshold ?? 0.5) !== 0.5 ||- v.featherRadius !== undefined+ (!skeleton &&+ (v.matteSource === "selfie" ||+ v.keyBackground === true ||+ v.enablePose === true ||+ (v.maskThreshold ?? 0.5) !== 0.5 ||+ v.featherRadius !== undefined))) {return;}@@ -1439,6 +1444,14 @@ export class Composition {(o.variant ?? "s") === (v.variant ?? "s") &&(o.confidence ?? 0.3) === (v.confidence ?? 0.3)) {+ // A skeleton of a selected person must draw that person, never whoever is detected first.+ if (skeleton && !clip.tracksPose) {+ if (clip.exact)+ throw new Error(+ "A skeleton layer of a prompted comp.vision clip needs `pose: true` on that clip.",+ );+ continue;+ }return clip;}}diff --git a/packages/vision/src/segmentation/clip-subject.ts b/packages/vision/src/segmentation/clip-subject.tsindex cc11edae..a6f63e8f 100644--- a/packages/vision/src/segmentation/clip-subject.ts+++ b/packages/vision/src/segmentation/clip-subject.ts@@ -1,10 +1,11 @@-import type { InstanceMask } from "../types.js";+import type { InstanceMask, PoseKeypoint } from "../types.js";/*** Where a render-time Vision effect finds subject masks someone already* computed for its clip (`comp.vision(clip)` in framefields), so a matte layer- * reuses them instead of segmenting the frame again. Set on the Vision effect- * instance under this key; the renderer reads it from the effect.+ * reuses them instead of segmenting the frame again, and a skeleton layer draws+ * the pose matched to that subject instead of the first person it detects. Set+ * on the Vision effect instance under this key; the renderer reads it from the effect.*/export const CLIP_SUBJECT: unique symbol = Symbol.for("framefields.vision.clipSubject",@@ -19,4 +20,18 @@ export interface ClipSubjectSource {* `undefined` when it was not analysed, so the caller segments it itself.*/subject(frame: number, fps: number): InstanceMask | null | undefined;+ /**+ * The subject's pose for a source frame; `null` when the frame was analysed+ * and holds no matched pose; `undefined` when it was not analysed (or the+ * clip tracks no pose).+ */+ pose?(frame: number, fps: number): ClipPose | null | undefined;+}++export interface ClipPose {+ /** 17 COCO keypoints, normalized to the source size. */+ readonly keypoints: readonly PoseKeypoint[];+ /** Source size in pixels, for fitting the pose to a target. */+ readonly width: number;+ readonly height: number;}