Skip to content

Code-first video & image, rendered natively on WebGPU.

feat(vision): skeleton layers draw the pose matched to a comp.vision clip

okanasla1e37b8659b5
diff --git a/.changeset/skeleton-clip-pose.md b/.changeset/skeleton-clip-pose.md
new file mode 100644
index 00000000..20e7bd01
--- /dev/null
+++ b/.changeset/skeleton-clip-pose.md
@@ -0,0 +1,7 @@
+---
+"@framefields/vision": minor
+"@framefields/node-vision": minor
+"framefields": minor
+---
+
+A `withVision({ mode: "skeleton" })` layer of a clip that `comp.vision` analyses now draws the pose that clip matched to its subject, for the frame on screen, instead of running its own pass and drawing the first person detected. A skeleton layer of a SAM clip needs `pose: true` on the clip and fails clearly without it. Skeleton mode draws only the skeleton: a frame without a pose is empty rather than showing the footage. SAM clip caches written before this change claimed pose they did not track; they are invalidated, so SAM clips re-analyse once.
diff --git a/nodes/node-vision/src/renderers/webgpu-renderer.ts b/nodes/node-vision/src/renderers/webgpu-renderer.ts
index 055bc4fa..43c5d8b1 100644
--- a/nodes/node-vision/src/renderers/webgpu-renderer.ts
+++ b/nodes/node-vision/src/renderers/webgpu-renderer.ts
@@ -8,6 +8,7 @@ import {
matchPoseToTracks,
mergeSubjectMask,
type PersonMatte,
+ type PoseKeypoint,
type PoseResult,
PoseSkeletonRenderer,
readTexturePixels,
@@ -644,7 +645,20 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {
targetHeight,
)
: undefined;
- const fromClip = clipSubject !== undefined;
+ // A skeleton of a `comp.vision` clip draws the pose matched to its subject.
+ const clipPose =
+ mode === "skeleton" && !visionBundle
+ ? clipPoseFor(
+ effectKey,
+ props.sourceTimeSec === undefined
+ ? frameIdx
+ : props.sourceTimeSec * fps,
+ fps,
+ targetWidth,
+ targetHeight,
+ )
+ : undefined;
+ const fromClip = clipSubject !== undefined || clipPose !== undefined;
// 3. Results are cached per source frame (selfie mattes and poses are not,
// so nodes that need them always run inference). A frame analysed before
@@ -933,30 +947,34 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {
if (!sharedSkeletonRenderer) {
sharedSkeletonRenderer = new PoseSkeletonRenderer(ctx.device);
}
- const poseRes = visionBundle?.getPoseResult(frameIdx) ?? currentPoseRes;
- const person = poseRes?.people[0];
- if (person && person.keypoints.length > 0) {
- const skelTex = sharedSkeletonRenderer.renderToTexture(person.keypoints, {
+ const keypoints =
+ clipPose !== undefined
+ ? clipPose
+ : (visionBundle?.getPoseResult(frameIdx) ?? currentPoseRes)?.people[0]
+ ?.keypoints;
+ // Only the skeleton: a frame without a pose stays empty, never the footage.
+ const skelPass = ctx.renderer.beginFrame(
+ encoder,
+ targetView,
+ { r: 0, g: 0, b: 0, a: 0 },
+ targetWidth,
+ targetHeight,
+ "clear",
+ );
+ if (keypoints && keypoints.length > 0) {
+ const skelTex = sharedSkeletonRenderer.renderToTexture(keypoints, {
width: targetWidth,
height: targetHeight,
});
- const skelPass = ctx.renderer.beginFrame(
- encoder,
- targetView,
- { r: 0, g: 0, b: 0, a: 0 },
- targetWidth,
- targetHeight,
- "clear",
- );
ctx.renderer.drawTexture(skelPass, skelTex, {
x: 0,
y: 0,
width: targetWidth,
height: targetHeight,
});
- skelPass.end();
- return;
}
+ skelPass.end();
+ return;
}
if (mode === "boxes" || mode === "tracking") {
@@ -1074,18 +1092,51 @@ function clipSubjectFor(
width: number,
height: number,
): InstanceMask | null | undefined {
- const source = (effect as Record<symbol, unknown> | undefined)?.[
- CLIP_SUBJECT
- ] as ClipSubjectSource | undefined;
+ const source = linkedClip(effect);
const subject = source?.subject(frame, fps);
- if (!subject) return subject;
- const aspect = subject.width / subject.height / (width / height);
- if (Math.abs(aspect - 1) < 0.01) return subject;
- if (source?.exact)
+ if (!subject || !source) return subject;
+ return fitsTarget(source, subject.width / subject.height, width / height)
+ ? subject
+ : undefined;
+}
+
+/**
+ * The pose a linked `comp.vision` clip matched to its subject for a source
+ * frame, normalized to the source; it fits targets of the same aspect.
+ */
+function clipPoseFor(
+ effect: object | undefined,
+ frame: number,
+ fps: number,
+ width: number,
+ height: number,
+): readonly PoseKeypoint[] | null | undefined {
+ const source = linkedClip(effect);
+ const pose = source?.pose?.(frame, fps);
+ if (!pose || !source) return pose === null ? null : undefined;
+ return fitsTarget(source, pose.width / pose.height, width / height)
+ ? pose.keypoints
+ : undefined;
+}
+
+function linkedClip(effect: object | undefined): ClipSubjectSource | undefined {
+ return (effect as Record<symbol, unknown> | undefined)?.[CLIP_SUBJECT] as
+ | ClipSubjectSource
+ | undefined;
+}
+
+/** Source results map onto a target of the same aspect; a prompted clip must not fall back. */
+function fitsTarget(
+ source: ClipSubjectSource,
+ sourceAspect: number,
+ targetAspect: number,
+): boolean {
+ if (Math.abs(sourceAspect / targetAspect - 1) < 0.01) return true;
+ if (source.exact)
throw new Error(
"A prompted clip matte requires the source aspect ratio. Apply layout cropping after the source matte.",
);
- return undefined;
+ return false;
}
/** Adapts a Selfie Segmenter matte to the instance-mask shape the composite path takes. */
diff --git a/packages/framefields/src/clip-vision-matte.test.ts b/packages/framefields/src/clip-vision-matte.test.ts
index 073bee61..6551f150 100644
--- a/packages/framefields/src/clip-vision-matte.test.ts
+++ b/packages/framefields/src/clip-vision-matte.test.ts
@@ -198,7 +198,7 @@ describe("matte reuses comp.vision", () => {
c.add([first, second]);
const a = c.vision(first, { cacheDir: false });
const b = c.vision(second, { cacheDir: false });
- (c as unknown as { linkClipMattes(): void }).linkClipMattes();
+ (c as unknown as { linkClipLayers(): void }).linkClipLayers();
expect((first.vision as { [CLIP_SUBJECT]?: unknown })[CLIP_SUBJECT]).toBe(
a,
);
@@ -243,7 +243,7 @@ describe("matte reuses comp.vision", () => {
const keyed = cutout("clip.mp4", { keyBackground: true });
const other = cutout("other.mp4");
c.add([same, classes, keyed, other]);
- (c as unknown as { linkClipMattes(): void }).linkClipMattes();
+ (c as unknown as { linkClipLayers(): void }).linkClipLayers();
const linked = (n: { vision?: object }) =>
CLIP_SUBJECT in (n.vision as object);
expect(linked(same as never)).toBe(true);
diff --git a/packages/framefields/src/clip-vision-skeleton.test.ts b/packages/framefields/src/clip-vision-skeleton.test.ts
new file mode 100644
index 00000000..918b2814
--- /dev/null
+++ b/packages/framefields/src/clip-vision-skeleton.test.ts
@@ -0,0 +1,147 @@
+/**
+ * A `withVision({ mode: "skeleton" })` layer of a clip that `comp.vision`
+ * analyses draws the pose the clip matched to its subject, not the first
+ * person a render-time pass detects.
+ */
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { CLIP_SUBJECT } from "@framefields/vision";
+import { describe, expect, it } from "vitest";
+import { seedClipVision } from "./clip-vision.js";
+import { Composition, HeadlessMediaRenderer, Layer } from "./index.js";
+import { decodePngPixels, getPixel } from "./test-utils/pixels.js";
+
+const FPS = 30;
+const SIZE = 200;
+const SAM3 = { prompts: [{ frame: 0, text: "person" }] };
+
+function stage() {
+ return new Composition({
+ width: SIZE,
+ height: SIZE,
+ fps: FPS,
+ durationFrames: 10,
+ backgroundColor: "#000000",
+ });
+}
+
+/** A 10-frame solid green clip. */
+async function greenClip(): Promise<string> {
+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "ff-clip-skeleton-"));
+ const file = path.join(dir, "green.mp4");
+ const c = stage();
+ c.add(
+ Layer.box({
+ position: "absolute",
+ x: 0,
+ y: 0,
+ width: SIZE,
+ height: SIZE,
+ background: "#00ff00",
+ }),
+ );
+ const r = await c.renderVideo({ outputPath: file, quality: "lossless" });
+ await r.cleanup?.();
+ return file;
+}
+
+/** A standing person in the left half: 17 COCO keypoints as [x, y, visibility], source px. */
+function leftPerson(): number[] {
+ const points: [number, number][] = [
+ [50, 30], // nose
+ [46, 26],
+ [54, 26], // eyes
+ [42, 28],
+ [58, 28], // ears
+ [40, 60],
+ [60, 60], // shoulders
+ [40, 100],
+ [60, 100], // elbows
+ [40, 130],
+ [60, 130], // wrists
+ [42, 120],
+ [58, 120], // hips
+ [42, 155],
+ [58, 155], // knees
+ [42, 190],
+ [58, 190], // ankles
+ ];
+ return points.flatMap(([x, y]) => [x, y, 1]);
+}
+
+const skeleton = (clip: string) =>
+ Layer.video(clip, {
+ position: "absolute",
+ x: 0,
+ y: 0,
+ width: SIZE,
+ height: SIZE,
+ fit: "cover",
+ muted: true,
+ }).withVision({
+ mode: "skeleton",
+ // Unreachable: any pose pass during render would fail.
+ baseUrl: "http://127.0.0.1:9/",
+ });
+
+function litPixels(
+ image: Awaited<ReturnType<typeof decodePngPixels>>,
+ x0: number,
+ x1: number,
+): number {
+ let lit = 0;
+ for (let y = 0; y < SIZE; y += 2)
+ for (let x = x0; x < x1; x += 2) {
+ const p = getPixel(image, x, y);
+ if (p.r + p.g + p.b > 60) lit++;
+ }
+ return lit;
+}
+
+describe("skeleton reuses comp.vision", () => {
+ it("draws the clip's matched pose from the first frame, and nothing where it has none", async () => {
+ const file = await greenClip();
+ const c = stage();
+ const layer = skeleton(file);
+ c.add(layer);
+ const clip = c.vision(layer, { cacheDir: false, sam3: SAM3, pose: true });
+ seedClipVision(
+ clip,
+ { width: SIZE, height: SIZE },
+ Array.from({ length: 40 }, (_, frame) => ({
+ kp: frame === 1 ? null : leftPerson(),
+ mask: null,
+ subject: null,
+ })),
+ );
+ const renderer = new HeadlessMediaRenderer();
+ const first = await decodePngPixels(
+ await c.renderFrame({ frame: 0, renderer }),
+ );
+ expect(litPixels(first, 0, 100)).toBeGreaterThan(20);
+ // Only the skeleton: the green footage never shows through.
+ expect(litPixels(first, 100, SIZE)).toBe(0);
+ const empty = await decodePngPixels(
+ await c.renderFrame({ frame: 1, renderer }),
+ );
+ expect(litPixels(empty, 0, SIZE)).toBe(0);
+ });
+
+ it("links skeleton layers only to clips that track pose", () => {
+ const c = stage();
+ const plain = skeleton("clip.mp4");
+ c.add(plain);
+ c.vision("clip.mp4", { cacheDir: false, pose: false });
+ (c as unknown as { linkClipLayers(): void }).linkClipLayers();
+ expect(CLIP_SUBJECT in (plain.vision as object)).toBe(false);
+
+ const d = stage();
+ const prompted = skeleton("clip.mp4");
+ d.add(prompted);
+ d.vision("clip.mp4", { cacheDir: false, sam3: SAM3 });
+ expect(() =>
+ (d as unknown as { linkClipLayers(): void }).linkClipLayers(),
+ ).toThrow(/needs `pose: true`/);
+ });
+});
diff --git a/packages/framefields/src/clip-vision.ts b/packages/framefields/src/clip-vision.ts
index 80ffc0f9..8a77ab88 100644
--- a/packages/framefields/src/clip-vision.ts
+++ b/packages/framefields/src/clip-vision.ts
@@ -6,6 +6,7 @@ import {
Signal,
} from "@framefields/core";
import {
+ type ClipPose,
type ClipSubjectSource,
COCO17_KEYPOINT_NAMES,
type InstanceMask,
@@ -27,7 +28,7 @@ import type { VisionProps } from "./effects/index.js";
export type PoseKeypointName = (typeof COCO17_KEYPOINT_NAMES)[number];
export interface ClipVisionOptions {
- /** Prompted SAM 2.1 (Tiny or Small) video segmentation. Pose is opt-in in this mode. */
+ /** Prompted SAM 2.1 (Tiny or Small) video segmentation. Pose is opt-in in this mode, as with sam3. */
sam2?: Sam2Options;
/** SAM 3 point/box or text selection with temporal tracking. Mutually exclusive with sam2. */
sam3?: Sam3Options;
@@ -43,7 +44,10 @@ export interface ClipVisionOptions {
durationFrames?: number;
/** Analysis frame rate (default: the composition's). */
fps?: number;
- /** Track 17 COCO keypoints (default true). */
+ /**
+ * Track 17 COCO keypoints (default true; opt-in with sam2/sam3, where each frame's pose
+ * is the detected person whose keypoints fall most inside the selection).
+ */
pose?: boolean;
/** Track the person's silhouette, dress and held things included (default true). */
mask?: boolean;
@@ -341,17 +345,8 @@ export class ClipVision implements ClipSubjectSource {
* subjects kept, null when it holds no subject.
*/
subject(frame: number, fps: number): InstanceMask | null | undefined {
- const c = (frame * this.fps) / fps;
- if (
- !this.exact &&
- !this.options.timeRemap &&
- Math.abs(c - Math.round(c)) > 1e-6
- )
- return undefined;
- const sourceFrame =
- this.options.timeRemap || this.exact
- ? Math.floor(c + 1e-6)
- : Math.round(c);
+ const sourceFrame = this.sourceFrame(frame, fps);
+ if (sourceFrame === undefined) return undefined;
const s = this.subjects.get(sourceFrame);
if (s === undefined) return s;
if (s === null || !this.sourceWidth) return null;
@@ -368,6 +363,54 @@ export class ClipVision implements ClipSubjectSource {
};
}
+ /**
+ * @internal The pose matched to the subject at a source frame of the clip
+ * (`frame` at `fps`), normalized to the source size, for a render-time
+ * skeleton: undefined until that frame and its smoothing window are
+ * analysed, null when no pose matched there.
+ */
+ pose(frame: number, fps: number): ClipPose | null | undefined {
+ const c = this.sourceFrame(frame, fps);
+ if (c === undefined || !this.tracksPose || !this.sourceWidth) return;
+ for (let g = c - this.smoothing; g <= c + this.smoothing; g++)
+ if (!this.results.has(Math.max(0, g))) return undefined;
+ if (!this.results.get(c)?.kp) return null;
+ const width = this.sourceWidth;
+ const height = this.sourceHeight;
+ return {
+ width,
+ height,
+ keypoints: Array.from({ length: 17 }, (_, k) => {
+ const [x, y, visibility] = this.keypoint(k, c);
+ return { x: x / width, y: y / height, visibility };
+ }),
+ };
+ }
+
+ /** Whether frames carry keypoints: on by default, opt-in for prompted (SAM) clips. */
+ get tracksPose(): boolean {
+ return this.exact
+ ? this.options.pose === true
+ : (this.options.pose ?? true);
+ }
+
+ /**
+ * The analysed source frame a render-time read at `frame` (at `fps`) maps to;
+ * undefined for an unprompted clip read between its analysed frames.
+ */
+ private sourceFrame(frame: number, fps: number): number | undefined {
+ const c = (frame * this.fps) / fps;
+ if (
+ !this.exact &&
+ !this.options.timeRemap &&
+ Math.abs(c - Math.round(c)) > 1e-6
+ )
+ return undefined;
+ return this.options.timeRemap || this.exact
+ ? Math.floor(c + 1e-6)
+ : Math.round(c);
+ }
+
/** Composition frame → clip frame (fractional). */
clipFrame(frame: number): number {
if (this.options.timeRemap)
@@ -446,7 +489,7 @@ export class ClipVision implements ClipSubjectSource {
await this.computeSam3(to);
return;
}
- const wantPose = this.options.pose ?? true;
+ const wantPose = this.tracksPose;
const wantMask = this.options.mask ?? true;
this.runner ??= VisionRunner.create({
modelsDir: this.options.modelsDir,
@@ -562,7 +605,7 @@ export class ClipVision implements ClipSubjectSource {
frame,
prompt?.points,
);
- const kp = this.options.pose
+ const kp = this.tracksPose
? await this.matchedPose(image, result.mask)
: null;
this.storeSamFrame(frame, result.width, result.height, [result], kp);
@@ -615,7 +658,7 @@ export class ClipVision implements ClipSubjectSource {
if (!this.sourceWidth) this.setSize(image.width, image.height);
const prompt = settings.prompts.find((item) => item.frame === frame);
const result = await this.sam3Tracker.process(image, frame, prompt);
- const kp = this.options.pose
+ const kp = this.tracksPose
? await this.matchedPose(image, unionMask(result.objects, image))
: null;
this.storeSamFrame(
@@ -930,13 +973,14 @@ export class ClipVision implements ClipSubjectSource {
stat.size,
stat.mtimeMs,
this.fps,
- o.pose ?? true,
+ this.tracksPose,
o.mask ?? true,
this.cell,
o.variant ?? "s",
o.confidence ?? null,
- // Prompted subjects are track-ID label maps (format 2).
- ...(o.sam2 ? [[sam?.sam2Revision(o.sam2.model), o.sam2, 2]] : []),
+ // Prompted subjects are track-ID label maps (format 2); format 3 drops
+ // entries keyed as tracking pose when SAM clips left it off by default.
+ ...(o.sam2 ? [[sam?.sam2Revision(o.sam2.model), o.sam2, 3]] : []),
...(o.sam3 && sam
? [
[
@@ -944,7 +988,7 @@ export class ClipVision implements ClipSubjectSource {
await sam.sam3Precision(undefined, o.sam3.precision),
),
o.sam3,
- 2,
+ 3,
],
]
: []),
diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.ts
index cccb7a7b..840b02d9 100644
--- a/packages/framefields/src/index.ts
+++ b/packages/framefields/src/index.ts
@@ -1335,19 +1335,20 @@ export class Composition {
* frame — physics, particles — always find every frame before it.
*/
public async prepareVision(frame: number): Promise<void> {
- this.linkClipMattes();
+ this.linkClipLayers();
await Promise.all(
[...this._clipVision.values()].map((clip) => clip.ready(0, frame)),
);
}
/**
- * Points matte-family `withVision` layers at the `comp.vision` clip they
- * show, so the cutout reuses that clip's per-frame subject instead of
- * segmenting the frame again during render. Only layers whose settings
- * would give the same subject are linked; the rest keep their own pass.
+ * Points matte-family and skeleton `withVision` layers at the `comp.vision`
+ * clip they show, so the cutout reuses that clip's per-frame subject and the
+ * skeleton draws its matched pose, instead of analysing the frame again
+ * during render. Only layers whose settings would give the same subject are
+ * linked; the rest keep their own pass.
*/
- private linkClipMattes(): void {
+ private linkClipLayers(): void {
if (this._clipVision.size === 0) return;
let layers:
| Map<
@@ -1386,13 +1387,13 @@ export class Composition {
}),
);
const layer = layers.get(n.id);
- const clip = this.clipForMatte(n.src ?? n.inputHandleId, n.vision, {
+ const clip = this.clipForLayer(n.src ?? n.inputHandleId, n.vision, {
trimStartSec: n.trimStartSec,
startFrame: layer?.startFrame ?? n.startFrame,
timeRemap: layer?.timeRemap,
});
if (clip) {
- clip.keepSubjects();
+ if (n.vision.mode !== "skeleton") clip.keepSubjects();
Object.defineProperty(n.vision, CLIP_SUBJECT, {
value: clip,
enumerable: false,
@@ -1405,7 +1406,8 @@ export class Composition {
for (const child of this.children) walk(child);
}
- private clipForMatte(
+ /** The `comp.vision` clip a matte or skeleton layer of the same source reads instead of analysing. */
+ private clipForLayer(
source: string,
v: Vision,
layer: {
@@ -1415,17 +1417,20 @@ export class Composition {
},
): ClipVision | undefined {
const mode = v.mode ?? "passthrough";
- if (mode !== "matte" && mode !== "mask" && mode !== "crop") return;
+ const skeleton = mode === "skeleton";
+ if (mode !== "matte" && mode !== "mask" && mode !== "crop" && !skeleton)
+ return;
// Anything that changes the subject, or needs results the clip lacks.
if (
- v.matteSource === "selfie" ||
- v.keyBackground === true ||
v.classes !== undefined ||
- v.enablePose === true ||
v.enableMatte === true ||
v.visionBundle !== undefined ||
- (v.maskThreshold ?? 0.5) !== 0.5 ||
- v.featherRadius !== undefined
+ (!skeleton &&
+ (v.matteSource === "selfie" ||
+ v.keyBackground === true ||
+ v.enablePose === true ||
+ (v.maskThreshold ?? 0.5) !== 0.5 ||
+ v.featherRadius !== undefined))
) {
return;
}
@@ -1439,6 +1444,14 @@ export class Composition {
(o.variant ?? "s") === (v.variant ?? "s") &&
(o.confidence ?? 0.3) === (v.confidence ?? 0.3)
) {
+ // A skeleton of a selected person must draw that person, never whoever is detected first.
+ if (skeleton && !clip.tracksPose) {
+ if (clip.exact)
+ throw new Error(
+ "A skeleton layer of a prompted comp.vision clip needs `pose: true` on that clip.",
+ );
+ continue;
+ }
return clip;
}
}
diff --git a/packages/vision/src/segmentation/clip-subject.ts b/packages/vision/src/segmentation/clip-subject.ts
index cc11edae..a6f63e8f 100644
--- a/packages/vision/src/segmentation/clip-subject.ts
+++ b/packages/vision/src/segmentation/clip-subject.ts
@@ -1,10 +1,11 @@
-import type { InstanceMask } from "../types.js";
+import type { InstanceMask, PoseKeypoint } from "../types.js";
/**
* Where a render-time Vision effect finds subject masks someone already
* computed for its clip (`comp.vision(clip)` in framefields), so a matte layer
- * reuses them instead of segmenting the frame again. Set on the Vision effect
- * instance under this key; the renderer reads it from the effect.
+ * reuses them instead of segmenting the frame again, and a skeleton layer draws
+ * the pose matched to that subject instead of the first person it detects. Set
+ * on the Vision effect instance under this key; the renderer reads it from the effect.
*/
export const CLIP_SUBJECT: unique symbol = Symbol.for(
"framefields.vision.clipSubject",
@@ -19,4 +20,18 @@ export interface ClipSubjectSource {
* `undefined` when it was not analysed, so the caller segments it itself.
*/
subject(frame: number, fps: number): InstanceMask | null | undefined;
+ /**
+ * The subject's pose for a source frame; `null` when the frame was analysed
+ * and holds no matched pose; `undefined` when it was not analysed (or the
+ * clip tracks no pose).
+ */
+ pose?(frame: number, fps: number): ClipPose | null | undefined;
+}
+
+export interface ClipPose {
+ /** 17 COCO keypoints, normalized to the source size. */
+ readonly keypoints: readonly PoseKeypoint[];
+ /** Source size in pixels, for fitting the pose to a target. */
+ readonly width: number;
+ readonly height: number;
}