Code-first video & image, rendered natively on WebGPU.
import type { AimConstraint } from "@framefields/compositions/program";
import {
type FrameSignal,
mediaSourceRange,
mediaSourceTime,
type ResolvedMediaTimeRemap,
Signal,
} from "@framefields/core";
import {
type ClipPose,
type ClipSubjectSource,
COCO17_KEYPOINT_NAMES,
type DepthMap,
defaultMatteBand,
type InstanceMask,
type MatteRefineOptions,
maskObject,
mergeSubjectMask,
VisionBundle,
VisionRunner,
} from "@framefields/vision";
import type {
Sam2Options,
Sam2Tracker,
Sam3ObjectResult,
Sam3Options,
Sam3Tracker,
} from "@framefields/vision-sam";
import { mediaDecoderCache } from "@framefields/webgpu-renderers";
import type { VisionProps } from "./effects/index.js";
import { type RetargetOptions, retargetPose } from "./retarget.js";
export type PoseKeypointName = (typeof COCO17_KEYPOINT_NAMES)[number];
export interface ClipVisionOptions {
/** Prompted SAM 2.1 (Tiny or Small) video segmentation. Pose is opt-in in this mode, as with sam3. */
sam2?: Sam2Options;
/** SAM 3 point/box or text selection with temporal tracking. Mutually exclusive with sam2. */
sam3?: Sam3Options;
/** Explicitly agree to Meta's SAM License terms, required to download and run SAM 3. */
acceptSamLicense?: boolean;
/** Composition frame the clip's first frame plays on (default 0). */
startFrame?: number;
/** Seconds trimmed off the clip's start, as on its media layer (default 0). */
trimStartSec?: number;
/** The media layer's source-time mapping. */
timeRemap?: ResolvedMediaTimeRemap;
/** Media layer duration in composition frames, when known. */
durationFrames?: number;
/** Analysis frame rate (default: the composition's). */
fps?: number;
/**
* Track 17 COCO keypoints (default true; opt-in with sam2/sam3, where each frame's pose
* is the detected person whose keypoints fall most inside the selection).
*/
pose?: boolean;
/** Track the person's silhouette, dress and held things included (default true). */
mask?: boolean;
/**
* Relative depth of every frame (Depth Anything V2 Small, near = 1), read by
* `withVision({ mode: "depth" })` layers and {@link ClipVision.depthMap}. A depth
* clip tracks no pose or silhouette unless `pose` / `mask` ask for them.
*/
depth?: boolean;
/**
* Resolve the subject's edge to soft alpha (ViTMatte Small): hair, motion
* blur and see-through parts that a hard mask cuts off. `true`, or
* `{ band, resolution }` (see `MatteRefineOptions`). Matte layers, `signals`
* masks and `subject` read the soft alpha; each prompted track keeps its own.
* Mattes persist apart from the masks, so turning this on reuses them.
*/
matting?: boolean | MatteRefineOptions;
/** Silhouette cell size in source pixels (default 8: a 1920×1080 clip gives a 240×135 grid). */
maskCell?: number;
/** Keypoints are averaged over ±`smoothing` frames, weighted by confidence (default 2; 0 = raw). */
smoothing?: number;
variant?: VisionProps["variant"];
confidence?: number;
modelsDir?: string;
baseUrl?: string;
/**
* Node: where results persist between runs. Default
* `$FRAMEFIELDS_VISION_CACHE_DIR` or `~/.cache/framefields/vision`;
* `false` keeps them in memory only. Browsers always keep them in memory.
*/
cacheDir?: string | false;
}
/** Where the clip sits in the composition, like a media layer's box and `fit`. */
export interface PersonPlacement {
x?: number;
y?: number;
width: number;
height: number;
fit?: "cover" | "contain" | "fill";
}
interface FrameResult {
score?: number;
/** 17 × [x, y, visibility], source px; null when no person was found. */
kp: Float32Array | null;
/** cellW × cellH, 0–255; null when no person was found. */
mask: Uint8Array | null;
}
/** A frame's merged subject at source resolution, run-length encoded. */
interface SubjectResult {
category: string;
area: number;
/** (value, run length) pairs over the source-sized 0–255 mask. */
values: Uint8Array;
lengths: Uint32Array;
/**
* Prompted (SAM) frames only: presence probability per track ID. The runs then hold a
* label map (track ID per pixel, 0 for none) instead of alpha.
*/
scores?: number[];
}
/** A frame's soft alpha, run-length encoded, as matting resolved it. */
interface MatteResult {
alpha: Runs;
/**
* Prompted frames only: the track label map grown into the soft edge
* (each edge pixel takes its nearest track's label), so every track keeps its own alpha.
*/
labels?: Runs;
}
interface Runs {
values: Uint8Array;
lengths: Uint32Array;
}
interface StoredRuns {
values: string;
lengths: string;
}
/** A subject as written to the subjects cache file. */
interface StoredSubject {
category: string;
area: number;
scores?: number[];
values: string;
lengths: string;
}
interface Transform {
sx: number;
sy: number;
ox: number;
oy: number;
}
/** Frames looked back over to bridge a gap where the person is lost. */
const HOLD = 24;
const CACHE_VERSION = 1;
const DEPTH_CACHE_VERSION = 1;
const MATTE_CACHE_VERSION = 1;
const MATTE_MODEL = "vitmatte-s";
const DEPTH_MODEL = "depth-anything-v2-s";
/** Depth frames held in memory when they also live on disk (each is about 1 MB). */
const DEPTH_MEMORY = 48;
/** A frame's depth as stored: 0 … 65535 for far … near. */
interface StoredDepth {
width: number;
height: number;
data: Uint16Array;
}
type VisionSam = typeof import("@framefields/vision-sam");
let visionSam: Promise<VisionSam> | undefined;
let clipVisionIds = 0;
/** SAM ships in its own chunk, loaded only when a clip asks for `sam2` or `sam3`. */
function loadVisionSam(): Promise<VisionSam> {
visionSam ??= import("@framefields/vision-sam");
return visionSam;
}
/**
* Vision results for one clip, computed per frame on demand and shared by
* everything that reads the clip: physics, layout, signals. Unlike the
* render-time `withVision`, which analyses a layer's previous rendered frame,
* this analyses the clip's own frames, so a result exists before its frame
* renders. The composition analyses what each frame needs just before
* drawing it (and a live physics world before stepping), so preview and
* export fill the cache as they play; nothing has to be analysed up front.
*
* Results depend only on the clip and settings, so they persist on disk in
* Node: a second run, or a render after a preview, reuses them. A matte-mode
* `withVision` layer of the same clip reuses them too (`keepSubjects`).
*/
export class ClipVision implements ClipSubjectSource {
readonly source: string;
readonly fps: number;
readonly compositionFps: number;
readonly startFrame: number;
readonly trimStartSec: number;
readonly smoothing: number;
/** @internal */
readonly options: ClipVisionOptions;
private readonly results = new Map<number, FrameResult>();
private sourceWidth = 0;
private sourceHeight = 0;
private cellW = 0;
private cellH = 0;
private readonly cell: number;
private queue: Promise<void> = Promise.resolve();
private runner?: VisionRunner;
private sam2Tracker?: Sam2Tracker;
private sam2Frame = -1;
private sam3Tracker?: Sam3Tracker;
private sam3Frame = -1;
private sam?: Promise<VisionSam>;
/** Clips walk their source independently, so each owns a decoder (requests to one supersede). */
private readonly decoderId = `clip-vision#${++clipVisionIds}#`;
private signalBundle?: VisionBundle;
private publishedSignals = new Set<number>();
private signalMaskFrame = -1;
/** Masks of `signalMaskFrame` by track ID (0: all tracks merged). */
private readonly signalMasks = new Map<number, InstanceMask>();
/** Prompted track count per source frame, including frames where every track is absent. */
private readonly trackCounts = new Map<number, number>();
private cacheLoaded?: Promise<void>;
private unsaved = 0;
private unsavedSubjects = 0;
private readonly blurred = new Map<number, Float32Array | null>();
private readonly smoothed = new Map<string, [number, number, number]>();
private personView?: PersonTrack;
/** Full-resolution subject masks for render-time mattes (see `keepSubjects`). */
private readonly subjects = new Map<number, SubjectResult | null>();
private wantSubjects = false;
private subjectsLoaded?: Promise<void>;
/** Soft alpha per source frame (`matting`); null when the frame holds no subject. */
private readonly mattes = new Map<number, MatteResult | null>();
private mattesLoaded?: Promise<void>;
private unsavedMattes = 0;
/** Depth per source frame; null when the frame did not decode. Bounded when on disk. */
private readonly depths = new Map<number, StoredDepth | null>();
/** Source frames whose depth is on disk. */
private readonly depthsOnDisk = new Set<number>();
private depthDir?: Promise<string | null>;
/** Set once the depth folder is known: render-time reads are synchronous. */
private depthDisk?: {
dir: string;
fs: NodeModules["fs"];
path: NodeModules["path"];
};
/** @internal Use `comp.vision(source, options)`. */
constructor(source: string, fps: number, options: ClipVisionOptions = {}) {
this.source = source;
if (options.sam2 && options.sam3)
throw new Error("Choose one prompted backend: sam2 or sam3.");
this.options = options.sam2
? {
...options,
sam2: {
...options.sam2,
prompts: options.sam2.prompts.map((prompt) => ({
...prompt,
points: prompt.points.map((point) => ({ ...point })),
})),
},
}
: options.sam3
? {
...options,
sam3: {
...options.sam3,
prompts: (
options.sam3.prompts ??
(options.sam3.text
? [{ frame: 0, text: options.sam3.text }]
: [])
).map((prompt) => ({
...prompt,
...(prompt.points
? {
points: prompt.points.map((point) => ({
...point,
})),
}
: {}),
})),
},
}
: options;
this.compositionFps = fps;
this.fps = options.fps ?? fps;
this.startFrame = Math.round(options.startFrame ?? 0);
this.trimStartSec = options.trimStartSec ?? 0;
this.smoothing = Math.max(
0,
Math.round(options.smoothing ?? (options.sam2 || options.sam3 ? 0 : 2)),
);
this.cell = Math.max(1, Math.round(options.maskCell ?? 8));
if (options.sam2 || options.sam3 || this.tracksMatting)
this.wantSubjects = true;
}
/** @internal Explicit selections must never fall back to another detected subject. */
get exact(): boolean {
return this.options.sam2 !== undefined || this.options.sam3 !== undefined;
}
/** Object, mask and optional pose signals on the composition clock, in source geometry. */
get signals(): VisionBundle {
this.keepSubjects();
this.signalBundle ??= new VisionBundle({
fps: this.compositionFps,
totalFrames: Number.MAX_SAFE_INTEGER,
maskSource: (frame) =>
this.trackIds(frame).flatMap((id) => {
const mask = this.maskAt(frame, id);
return mask && mask.area > 0 ? [mask] : [];
}),
matteSource: (frame) => {
const mask = this.maskAt(frame);
return mask
? {
mask: mask.mask,
width: mask.width,
height: mask.height,
coverage: mask.coverage,
}
: undefined;
},
});
return this.signalBundle;
}
private decoder(): ReturnType<typeof mediaDecoderCache.getVideo> {
return mediaDecoderCache.getVideo(this.source, true, this.decoderId);
}
/** Free inference sessions and the decoder. Analysed frames remain readable; later frames replay memory. */
release(): void {
mediaDecoderCache.clearNode(this.decoderId);
this.sam2Tracker?.release();
this.sam2Tracker = undefined;
this.sam2Frame = -1;
this.sam3Tracker?.release();
this.sam3Tracker = undefined;
this.sam3Frame = -1;
this.runner?.close();
this.runner = undefined;
this.signalMaskFrame = -1;
this.signalMasks.clear();
this.publishedSignals.clear();
}
/** The tracked person in source pixels; `.place()` maps it into the composition. */
get person(): PersonTrack {
this.personView ??= new PersonTrack(this);
return this.personView;
}
/**
* Analyses what reading composition frame `frame` needs (it and a few
* neighbours for smoothing). Pass a second frame to analyse a whole range.
* Calls queue, so the clip is decoded once, in order.
*/
ready(frame: number, toFrame: number = frame): Promise<void> {
const [from, to] = this.window(frame, toFrame);
const run = this.queue.then(async () => {
try {
await this.compute(from, to);
} catch (error) {
this.release();
throw error;
}
if (this.signalBundle)
this.publishSignals(Math.min(frame, toFrame), Math.max(frame, toFrame));
});
this.queue = run.catch(() => undefined);
return run;
}
/** True when composition frame `frame` can be read synchronously. */
isReady(frame: number): boolean {
const [from, to] = this.window(frame, frame);
for (let c = from; c <= to; c++) if (!this.analysed(c)) return false;
return true;
}
/**
* @internal Also keep each frame's full-resolution subject mask, merged
* like the Vision effect's matte, so a matte layer of this clip reuses it
* instead of segmenting the frame again (framefields links them).
*/
keepSubjects(): void {
this.wantSubjects = true;
}
/**
* @internal The subject at a source frame of the clip (`frame` at `fps`),
* for a render-time matte: undefined until that frame is analysed with
* subjects kept, null when it holds no subject.
*/
subject(frame: number, fps: number): InstanceMask | null | undefined {
const sourceFrame = this.sourceFrame(frame, fps);
if (sourceFrame === undefined) return undefined;
const s = this.subjects.get(sourceFrame);
if (s === undefined) return s;
if (s === null || !this.sourceWidth) return null;
const width = this.sourceWidth;
const height = this.sourceHeight;
const mask = this.alpha(sourceFrame, s);
if (!mask) return undefined;
return {
category: s.category,
mask,
width,
height,
area: s.area,
coverage: s.area / (width * height),
detectionIndex: -1,
};
}
/**
* @internal The pose matched to the subject at a source frame of the clip
* (`frame` at `fps`), normalized to the source size, for a render-time
* skeleton: undefined until that frame and its smoothing window are
* analysed, null when no pose matched there.
*/
pose(frame: number, fps: number): ClipPose | null | undefined {
const c = this.sourceFrame(frame, fps);
if (c === undefined || !this.tracksPose || !this.sourceWidth) return;
for (let g = c - this.smoothing; g <= c + this.smoothing; g++)
if (!this.results.has(Math.max(0, g))) return undefined;
if (!this.results.get(c)?.kp) return null;
const width = this.sourceWidth;
const height = this.sourceHeight;
return {
width,
height,
keypoints: Array.from({ length: 17 }, (_, k) => {
const [x, y, visibility] = this.keypoint(k, c);
return { x: x / width, y: y / height, visibility };
}),
};
}
/** Whether frames carry keypoints: on by default, opt-in for prompted (SAM) clips. */
get tracksPose(): boolean {
return this.exact
? this.options.pose === true
: (this.options.pose ?? !this.tracksDepth);
}
/** Whether frames carry the person's silhouette: on by default, off for depth clips. */
get tracksMask(): boolean {
return this.options.mask ?? !this.tracksDepth;
}
/** Whether subjects are matted to soft alpha. */
get tracksMatting(): boolean {
return this.options.matting !== undefined && this.options.matting !== false;
}
private get matteSettings(): Required<MatteRefineOptions> | undefined {
const m = this.options.matting;
if (!m) return undefined;
const o = m === true ? {} : m;
const band =
o.band ??
(this.sourceWidth
? defaultMatteBand(this.sourceWidth, this.sourceHeight)
: 0);
return { band, resolution: o.resolution ?? 1024 };
}
/**
* A stored subject's alpha at a source frame: the matte when the clip mattes
* (undefined until matted), else the mask. With `id`, that track only.
*/
private alpha(
c: number,
stored: SubjectResult,
id?: number,
): Uint8Array | undefined {
const size = this.sourceWidth * this.sourceHeight;
if (!this.tracksMatting) return subjectAlpha(stored, size, id);
const matte = this.mattes.get(c);
if (!matte) return undefined;
const alpha = decodeRuns(matte.alpha, size);
if (id === undefined || !matte.labels) return alpha;
const labels = decodeRuns(matte.labels, size);
for (let i = 0; i < size; i++) if (labels[i] !== id) alpha[i] = 0;
return alpha;
}
/** Whether frames carry relative depth. */
get tracksDepth(): boolean {
return this.options.depth === true;
}
/**
* The relative depth (near = 1) of composition frame `frame`, at the depth
* model's resolution; undefined until that frame is analysed (`ready`).
*/
depthMap(frame: number): DepthMap | undefined {
return this.depth(frame, this.compositionFps) ?? undefined;
}
/**
* @internal The relative depth of a source frame of the clip (`frame` at
* `fps`) for a render-time depth layer: undefined until analysed, null when
* the frame did not decode.
*/
depth(frame: number, fps: number): DepthMap | null | undefined {
const c = this.sourceFrame(frame, fps);
if (c === undefined || !this.tracksDepth) return undefined;
const stored = this.storedDepth(c);
if (!stored) return stored;
const data = new Float32Array(stored.data.length);
for (let i = 0; i < data.length; i++)
data[i] = (stored.data[i] as number) / 65535;
return { data, width: stored.width, height: stored.height };
}
/**
* The analysed source frame a render-time read at `frame` (at `fps`) maps to;
* undefined for an unprompted clip read between its analysed frames.
*/
private sourceFrame(frame: number, fps: number): number | undefined {
const c = (frame * this.fps) / fps;
if (
!this.exact &&
!this.options.timeRemap &&
Math.abs(c - Math.round(c)) > 1e-6
)
return undefined;
return this.options.timeRemap || this.exact
? Math.floor(c + 1e-6)
: Math.round(c);
}
/** Composition frame → clip frame (fractional). */
clipFrame(frame: number): number {
if (this.options.timeRemap)
return (
mediaSourceTime(
this.options.timeRemap,
this.localFrame(frame),
this.compositionFps,
) * this.fps
);
return (
((frame - this.startFrame) / this.compositionFps + this.trimStartSec) *
this.fps
);
}
private localFrame(frame: number): number {
const local = Math.max(0, frame - this.startFrame);
return this.options.durationFrames === undefined
? local
: Math.min(local, Math.max(0, this.options.durationFrames - 1));
}
private window(frame: number, toFrame: number): [number, number] {
const range = this.options.timeRemap
? mediaSourceRange(
this.options.timeRemap,
this.localFrame(Math.min(frame, toFrame)),
this.localFrame(Math.max(frame, toFrame)),
this.compositionFps,
)
: undefined;
const a = Math.floor(
range ? range.min * this.fps : this.clipFrame(Math.min(frame, toFrame)),
);
const b = Math.ceil(
range ? range.max * this.fps : this.clipFrame(Math.max(frame, toFrame)),
);
return [
Math.max(0, a - HOLD - this.smoothing),
Math.max(0, b + this.smoothing),
];
}
private analysed(c: number): boolean {
return (
this.analysedSubject(c) &&
(!this.tracksDepth || this.hasDepth(c)) &&
(!this.tracksMatting || this.mattes.has(c))
);
}
/** Everything but depth, which is a pass of its own. */
private analysedSubject(c: number): boolean {
return this.results.has(c) && (!this.wantSubjects || this.subjects.has(c));
}
/** Loads `@framefields/vision-sam` and validates the prompts once, on first use. */
private prompted(): Promise<VisionSam> {
this.sam ??= loadVisionSam().then((sam) => {
if (this.options.sam2) sam.validateSam2Prompts(this.options.sam2.prompts);
if (this.options.sam3) sam.validateSam3Prompts(this.options.sam3);
return sam;
});
return this.sam;
}
private async compute(from: number, to: number): Promise<void> {
if (this.exact) await this.prompted();
this.cacheLoaded ??= this.loadCache();
await this.cacheLoaded;
if (this.wantSubjects) {
this.subjectsLoaded ??= this.loadSubjects();
await this.subjectsLoaded;
}
if (this.tracksDepth) await this.computeDepth(from, to);
await this.computeSubjects(from, to);
if (this.tracksMatting) await this.computeMattes(from, to);
}
/** Pose, silhouette and subject of every frame in `from…to` not analysed yet. */
private async computeSubjects(from: number, to: number): Promise<void> {
let missing = false;
for (let c = from; c <= to; c++)
if (!this.analysedSubject(c)) missing = true;
if (!missing) return;
if (this.options.sam2) {
await this.computeSam2(to);
return;
}
if (this.options.sam3) {
await this.computeSam3(to);
return;
}
const wantPose = this.tracksPose;
const wantMask = this.tracksMask;
const runner = this.visionRunner();
const decoder = this.decoder();
for (let c = from; c <= to; c++) {
if (this.analysedSubject(c)) continue;
const image = await decodedPixels(await decoder.getFrame(c / this.fps));
if (!image) {
this.results.set(c, { kp: null, mask: null });
if (this.wantSubjects) this.subjects.set(c, null);
continue;
}
if (!this.sourceWidth) this.setSize(image.width, image.height);
// Analysed before without subjects: only segment it again.
const known = this.results.get(c);
let kp: Float32Array | null = known?.kp ?? null;
let mask: Uint8Array | null = known?.mask ?? null;
if (wantPose && !known) {
const person = (await runner.pose(image)).people[0];
if (person) {
kp = new Float32Array(51);
for (const [k, p] of person.keypoints.entries()) {
kp.set([p.x, p.y, p.visibility], k * 3);
}
}
}
const seg =
(wantMask && !known) || this.wantSubjects
? await runner.segment(image)
: undefined;
if (seg && this.wantSubjects) {
const subject = mergeSubjectMask(seg.masks);
this.subjects.set(
c,
subject
? {
category: subject.category,
area: subject.area,
...encodeRuns(subject.mask),
}
: null,
);
this.unsavedSubjects++;
}
if (seg && wantMask && !known) {
const silhouette = personSilhouette(seg);
if (silhouette) {
mask = new Uint8Array(this.cellW * this.cellH);
downsampleMean(
silhouette,
image.width,
image.height,
this.cell,
mask,
this.cellW,
this.cellH,
);
}
}
this.results.set(c, { kp, mask });
if (!known) this.unsaved++;
if (this.unsaved >= 24) await this.saveCache();
if (this.unsavedSubjects >= 24) await this.saveSubjects();
}
if (this.unsaved > 0) await this.saveCache();
if (this.unsavedSubjects > 0) await this.saveSubjects();
}
private visionRunner(): VisionRunner {
this.runner ??= VisionRunner.create({
modelsDir: this.options.modelsDir,
baseUrl: this.options.baseUrl,
...(this.options.confidence !== undefined && {
confidence: this.options.confidence,
}),
...(this.options.variant !== undefined && {
variant: this.options.variant,
}),
});
return this.runner;
}
/**
* The soft alpha of every subject in `from…to` not matted yet: one
* ViTMatte pass per frame over every track's edge. Prompted label maps grow
* into the soft edge, so each track keeps its own alpha.
*/
private async computeMattes(from: number, to: number): Promise<void> {
// The default band, and so the cache key, depends on the frame size.
if (!this.sourceWidth) return;
this.mattesLoaded ??= this.loadMattes();
await this.mattesLoaded;
let decoder: ReturnType<typeof mediaDecoderCache.getVideo> | undefined;
for (let c = from; c <= to; c++) {
if (this.mattes.has(c)) continue;
const stored = this.subjects.get(c);
if (stored === undefined) continue;
if (stored === null) {
this.mattes.set(c, null);
continue;
}
decoder ??= this.decoder();
const image = await decodedPixels(await decoder.getFrame(c / this.fps));
if (!image) {
this.mattes.set(c, null);
continue;
}
const size = image.width * image.height;
const values = decodeRuns(stored, size);
const hard = stored.scores
? values.map((label) => (label ? 255 : 0))
: values;
const matte = await this.visionRunner().refineMatte(
image,
hard,
this.matteSettings,
);
let labels: Uint8Array | undefined;
if (stored.scores) {
labels = new Uint8Array(size);
for (let i = 0; i < size; i++) {
const nearest = matte.nearest[i] as number;
if (matte.mask[i] && nearest >= 0)
labels[i] = values[nearest] as number;
}
}
this.mattes.set(c, {
alpha: encodeRuns(matte.mask),
...(labels && { labels: encodeRuns(labels) }),
});
this.signalMaskFrame = -1;
if (++this.unsavedMattes >= 24) await this.saveMattes();
}
if (this.unsavedMattes > 0) await this.saveMattes();
}
/** Depth of every source frame in `from…to` not analysed yet, saved one file per frame. */
private async computeDepth(from: number, to: number): Promise<void> {
const dir = await this.depthCacheDir();
let decoder: ReturnType<typeof mediaDecoderCache.getVideo> | undefined;
for (let c = from; c <= to; c++) {
if (this.hasDepth(c)) continue;
decoder ??= this.decoder();
const image = await decodedPixels(await decoder.getFrame(c / this.fps));
if (!image) {
this.depths.set(c, null);
continue;
}
if (!this.sourceWidth) this.setSize(image.width, image.height);
const map = await this.visionRunner().depth(image);
const stored: StoredDepth = {
width: map.width,
height: map.height,
data: Uint16Array.from(map.data, (v) => Math.round(v * 65535)),
};
this.keepDepth(c, stored);
if (dir) await this.saveDepth(dir, c, stored);
}
}
private hasDepth(c: number): boolean {
return this.depths.has(c) || this.depthsOnDisk.has(c);
}
private keepDepth(c: number, depth: StoredDepth | null): void {
this.depths.delete(c);
this.depths.set(c, depth);
if (this.depthsOnDisk.size === 0) return;
for (const k of this.depths.keys()) {
if (this.depths.size <= DEPTH_MEMORY) break;
if (this.depthsOnDisk.has(k)) this.depths.delete(k);
}
}
private storedDepth(c: number): StoredDepth | null | undefined {
const held = this.depths.get(c);
if (held !== undefined || !this.depthsOnDisk.has(c)) return held;
const disk = this.depthDisk;
if (!disk) return undefined;
const stored = readDepthFile(disk.fs, disk.path.join(disk.dir, `${c}.bin`));
if (stored) this.keepDepth(c, stored);
return stored ?? undefined;
}
/**
* Where depth frames persist: a folder of its own, keyed only by the clip
* and the model, so other options never invalidate it.
*/
private depthCacheDir(): Promise<string | null> {
this.depthDir ??= (async () => {
const node = await nodeModules();
if (!node || this.options.cacheDir === false) return null;
const { fs, path, os, crypto } = node;
try {
const stat = fs.statSync(this.source);
const root =
this.options.cacheDir ??
process.env.FRAMEFIELDS_VISION_CACHE_DIR ??
path.join(os.homedir(), ".cache", "framefields", "vision");
const key = JSON.stringify([
DEPTH_CACHE_VERSION,
path.resolve(this.source),
stat.size,
stat.mtimeMs,
this.fps,
DEPTH_MODEL,
]);
const hash = crypto
.createHash("sha256")
.update(key)
.digest("hex")
.slice(0, 24);
const dir = path.join(
root,
`${path.basename(this.source)}.${hash}.depth`,
);
fs.mkdirSync(dir, { recursive: true });
for (const name of fs.readdirSync(dir)) {
const m = /^(\d+)\.bin$/.exec(name);
if (m) this.depthsOnDisk.add(Number(m[1]));
}
this.depthDisk = { dir, fs, path };
return dir;
} catch {
return null; // remote URL or unreadable: memory only
}
})();
return this.depthDir;
}
private async saveDepth(
dir: string,
c: number,
depth: StoredDepth,
): Promise<void> {
const node = await nodeModules();
if (!node) return;
const header = new Uint32Array([depth.width, depth.height]);
const file = node.path.join(dir, `${c}.bin`);
const tmp = `${file}.${process.pid}.tmp`;
node.fs.writeFileSync(
tmp,
Buffer.concat([
Buffer.from(header.buffer),
Buffer.from(
depth.data.buffer,
depth.data.byteOffset,
depth.data.byteLength,
),
]),
);
node.fs.renameSync(tmp, file);
this.depthsOnDisk.add(c);
this.keepDepth(c, depth);
}
private async computeSam2(to: number): Promise<void> {
const settings = this.options.sam2;
if (!settings) return;
const first = settings.prompts[0]?.frame ?? 0;
for (let frame = 0; frame < Math.min(first, to + 1); frame++) {
this.results.set(frame, { kp: null, mask: null, score: 0 });
this.subjects.set(frame, null);
}
if (to < first) return;
const decoder = this.decoder();
if (!this.sam2Tracker) {
const frameCount = Math.max(
1,
Math.ceil((await decoder.durationSec()) * this.fps - 1e-6),
);
if (settings.prompts.some((prompt) => prompt.frame >= frameCount)) {
throw new Error("SAM 2 prompt frames must be inside the source video.");
}
const { Sam2Tracker } = await this.prompted();
this.sam2Tracker = await Sam2Tracker.create({
modelsDir: this.options.modelsDir,
baseUrl: this.options.baseUrl,
frameCount,
model: settings.model,
});
this.sam2Frame = first - 1;
}
for (let frame = this.sam2Frame + 1; frame <= to; frame++) {
const image = await decodedPixels(
await decoder.getFrame(frame / this.fps),
);
if (!image) throw new Error(`Cannot decode SAM 2 source frame ${frame}.`);
if (!this.sourceWidth) this.setSize(image.width, image.height);
const prompt = settings.prompts.find((item) => item.frame === frame);
const result = await this.sam2Tracker.process(
image,
frame,
prompt?.points,
);
const kp = this.tracksPose
? await this.matchedPose(image, result.mask)
: null;
this.storeSamFrame(frame, result.width, result.height, [result], kp);
if (this.unsaved >= 24) {
await this.saveCache();
await this.saveSubjects();
}
}
await this.saveCache();
await this.saveSubjects();
}
private async computeSam3(to: number): Promise<void> {
const settings = this.options.sam3;
if (!settings) return;
const first = settings.prompts[0]?.frame ?? 0;
for (let frame = 0; frame < Math.min(first, to + 1); frame++) {
this.results.set(frame, { kp: null, mask: null, score: 0 });
this.subjects.set(frame, null);
}
if (to < first) return;
const decoder = this.decoder();
if (!this.sam3Tracker) {
const frameCount = Math.max(
1,
Math.ceil((await decoder.durationSec()) * this.fps - 1e-6),
);
if (settings.prompts.some((prompt) => prompt.frame >= frameCount)) {
throw new Error("SAM 3 prompt frames must be inside the source video.");
}
const { Sam3Tracker } = await this.prompted();
this.sam3Tracker = await Sam3Tracker.create({
modelsDir: this.options.modelsDir,
baseUrl: this.options.baseUrl,
frameCount,
textThreshold: settings.textThreshold,
maxObjects: settings.maxObjects,
merge: settings.merge,
redetect: settings.redetect,
precision: settings.precision,
acceptLicense:
this.options.acceptSamLicense ?? this.options.sam3?.acceptLicense,
});
this.sam3Frame = first - 1;
}
for (let frame = this.sam3Frame + 1; frame <= to; frame++) {
const image = await decodedPixels(
await decoder.getFrame(frame / this.fps),
);
if (!image) throw new Error(`Cannot decode SAM 3 source frame ${frame}.`);
if (!this.sourceWidth) this.setSize(image.width, image.height);
const prompt = settings.prompts.find((item) => item.frame === frame);
const result = await this.sam3Tracker.process(image, frame, prompt);
const kp = this.tracksPose
? await this.matchedPose(image, unionMask(result.objects, image))
: null;
this.storeSamFrame(
frame,
result.width,
result.height,
result.objects,
kp,
);
if (this.unsaved >= 24) {
await this.saveCache();
await this.saveSubjects();
}
}
await this.saveCache();
await this.saveSubjects();
}
/** Stores a prompted frame: every track's mask as one label map, the union as the subject. */
private storeSamFrame(
frame: number,
width: number,
height: number,
objects: readonly Pick<Sam3ObjectResult, "mask" | "score">[],
kp: Float32Array | null,
): void {
const labels = new Uint8Array(width * height);
// Ascending presence, so where tracks overlap the most confident one keeps the pixel.
const order = [...objects.keys()].sort(
(a, b) => (objects[a]?.score ?? 0) - (objects[b]?.score ?? 0),
);
for (const index of order) {
const mask = objects[index]?.mask;
if (!mask) continue;
for (let i = 0; i < mask.length; i++)
if ((mask[i] ?? 0) >= 128) labels[i] = index + 1;
}
const union = Uint8Array.from(labels, (id) => (id ? 255 : 0));
const grid = new Uint8Array(this.cellW * this.cellH);
downsampleMean(
union,
width,
height,
this.cell,
grid,
this.cellW,
this.cellH,
);
let area = 0;
for (const id of labels) if (id) area++;
const scores = objects.map((o) => 1 / (1 + Math.exp(-o.score)));
this.results.set(frame, {
kp,
mask: grid,
score: Math.max(0, ...scores),
});
this.subjects.set(
frame,
area > 0
? { category: this.samCategory(), area, scores, ...encodeRuns(labels) }
: null,
);
this.trackCounts.set(frame, objects.length);
this.blurred.delete(frame);
if (this.options.sam3) this.sam3Frame = frame;
else this.sam2Frame = frame;
this.unsaved++;
this.unsavedSubjects++;
}
private samCategory(): string {
return (
this.options.sam2?.category ?? this.options.sam3?.category ?? "object"
);
}
private async matchedPose(
image: {
data: Uint8Array | Uint8ClampedArray;
width: number;
height: number;
},
mask: Uint8Array,
): Promise<Float32Array | null> {
this.runner ??= VisionRunner.create({
modelsDir: this.options.modelsDir,
baseUrl: this.options.baseUrl,
variant: this.options.variant,
});
const { people } = await this.runner.pose(image);
const ranked = people
.map((person) => ({
person,
overlap: person.keypoints.reduce((sum, point) => {
const x = Math.max(0, Math.min(image.width - 1, Math.round(point.x)));
const y = Math.max(
0,
Math.min(image.height - 1, Math.round(point.y)),
);
return (
sum + ((mask[y * image.width + x] ?? 0) / 255) * point.visibility
);
}, 0),
}))
.sort((a, b) => b.overlap - a.overlap);
const match = ranked[0];
if (!match || match.overlap < 2) return null;
return Float32Array.from(
match.person.keypoints.flatMap((point) => [
point.x,
point.y,
point.visibility,
]),
);
}
/** Track IDs at a composition frame: every prompted track, else the one subject. */
private trackIds(frame: number): number[] {
const c = Math.max(0, Math.floor(this.clipFrame(frame)));
const count =
this.subjects.get(c)?.scores?.length ?? this.trackCounts.get(c) ?? 1;
return Array.from({ length: count }, (_, i) => i + 1);
}
/** The subject (all tracks merged) or, with `id`, one track's mask, in source pixels. */
private maskAt(frame: number, id?: number): InstanceMask | undefined {
const c = Math.max(0, Math.floor(this.clipFrame(frame)));
if (this.signalMaskFrame !== c) {
this.signalMaskFrame = c;
this.signalMasks.clear();
}
const hit = this.signalMasks.get(id ?? 0);
if (hit) return hit;
if (!this.results.has(c) || !this.sourceWidth) return undefined;
const stored = this.subjects.get(c);
const size = this.sourceWidth * this.sourceHeight;
const alpha = stored ? this.alpha(c, stored, id) : new Uint8Array(size);
if (!alpha) return undefined;
const data = alpha;
let area = stored?.area ?? 0;
if (id !== undefined && stored?.scores) {
area = 0;
for (const alpha of data) if (alpha) area++;
}
const mask: InstanceMask = {
category: stored?.category ?? this.samCategory(),
mask: data,
width: this.sourceWidth,
height: this.sourceHeight,
area,
coverage: area / size,
detectionIndex: (id ?? 1) - 1,
trackId: id ?? 1,
};
this.signalMasks.set(id ?? 0, mask);
return mask;
}
private publishSignals(from: number, to: number): void {
const bundle = this.signalBundle;
if (!bundle || !this.sourceWidth) return;
bundle.setFrameSize(this.sourceWidth, this.sourceHeight);
for (let frame = from; frame <= to; frame++) {
if (this.publishedSignals.has(frame)) continue;
this.publishedSignals.add(frame);
const c = Math.max(0, Math.floor(this.clipFrame(frame)));
const mask = this.maskAt(frame);
if (!mask) continue;
const before = bundle.getObjectResult(frame - 1).objects;
const scores = this.subjects.get(c)?.scores;
const objects = this.trackIds(frame).flatMap((id) => {
const track = this.maskAt(frame, id);
if (!track) return [];
return [
maskObject(
track,
frame,
this.compositionFps,
scores?.[id - 1] ?? this.results.get(c)?.score ?? 1,
before.find((object) => object.trackId === id),
),
];
});
bundle.setObjectResult(frame, objects);
const object =
objects[0] ?? maskObject(mask, frame, this.compositionFps, 0);
const kp = this.results.get(c)?.kp;
bundle.setPoseResult(frame, {
people: kp
? [
{
score: object.score,
boundingBox: object.boundingBox,
trackId: 1,
keypoints: Array.from({ length: 17 }, (_, i) => ({
x: (kp[i * 3] ?? 0) / mask.width,
y: (kp[i * 3 + 1] ?? 0) / mask.height,
visibility: kp[i * 3 + 2] ?? 0,
})),
},
]
: [],
});
}
}
private setSize(width: number, height: number): void {
this.sourceWidth = width;
this.sourceHeight = height;
this.cellW = Math.ceil(width / this.cell);
this.cellH = Math.ceil(height / this.cell);
}
/** @internal */
size(): { width: number; height: number } {
return { width: this.sourceWidth, height: this.sourceHeight };
}
/** @internal Raw result for a clip frame, or throws when it was never analysed. */
result(c: number): FrameResult | undefined {
return this.results.get(Math.max(0, c));
}
/** @internal Confidence-weighted keypoint over the smoothing window, in source px. */
keypoint(k: number, c: number): [number, number, number] {
c = Math.max(0, c);
const key = `${k}:${c}`;
const hit = this.smoothed.get(key);
if (hit) return hit;
const avg = (center: number): [number, number, number] | null => {
let sx = 0;
let sy = 0;
let sv = 0;
let sw = 0;
for (let g = center - this.smoothing; g <= center + this.smoothing; g++) {
const r = this.mustHave(g < 0 ? 0 : g);
if (!r.kp) continue;
const v = r.kp[k * 3 + 2];
const w = (this.smoothing + 1 - Math.abs(g - center)) * v * v;
sx += r.kp[k * 3] * w;
sy += r.kp[k * 3 + 1] * w;
sv += v * w;
sw += w;
}
return sw > 1e-6 ? [sx / sw, sy / sw, sv / sw] : null;
};
let out: [number, number, number] | null = null;
// Lost? Hold the last place the person was seen (within HOLD frames).
for (let d = 0; d <= HOLD && !out; d++) if (c - d >= 0) out = avg(c - d);
out ??= [this.sourceWidth / 2, this.sourceHeight / 2, 0];
this.smoothed.set(key, out);
return out;
}
/** @internal Softened silhouette grid for a clip frame (null: no person). */
blurredMask(c: number): Float32Array | null {
c = Math.max(0, c);
if (this.blurred.has(c)) return this.blurred.get(c) ?? null;
this.mustHave(c);
const mask = this.exact
? (this.results.get(c)?.mask ?? null)
: bridgedMask(this.results, c, this.smoothing);
const grid = mask ? boxBlur(mask, this.cellW, this.cellH, 2) : null;
this.blurred.set(c, grid);
return grid;
}
/** @internal */
grid(): { cell: number; width: number; height: number } {
return { cell: this.cell, width: this.cellW, height: this.cellH };
}
private mustHave(c: number): FrameResult {
const r = this.results.get(c);
if (r) return r;
throw new Error(
`ClipVision "${this.source}": frame ${c} is not analysed yet. ` +
"Read it while rendering (the composition prepares each frame), in a live physics world, " +
`or after \`await clip.ready(frame)\`.`,
);
}
// ── Persistence (Node) ────────────────────────────────────────────────
private cacheFile?: string | null;
private async cachePath(): Promise<string | null> {
if (this.cacheFile !== undefined) return this.cacheFile;
this.cacheFile = null;
if (
this.options.cacheDir === false ||
typeof process === "undefined" ||
!process.versions?.node
) {
return null;
}
try {
const [fs, path, os, crypto] = await Promise.all([
import("node:fs"),
import("node:path"),
import("node:os"),
import("node:crypto"),
]);
const stat = fs.statSync(this.source);
const dir =
this.options.cacheDir ??
process.env.FRAMEFIELDS_VISION_CACHE_DIR ??
path.join(os.homedir(), ".cache", "framefields", "vision");
const o = this.options;
const sam = this.exact ? await this.prompted() : undefined;
const key = JSON.stringify([
CACHE_VERSION,
path.resolve(this.source),
stat.size,
stat.mtimeMs,
this.fps,
this.tracksPose,
this.tracksMask,
this.cell,
o.variant ?? "s",
o.confidence ?? null,
// Prompted subjects are track-ID label maps (format 2); format 3 drops
// entries keyed as tracking pose when SAM clips left it off by default.
...(o.sam2 ? [[sam?.sam2Revision(o.sam2.model), o.sam2, 3]] : []),
...(o.sam3 && sam
? [
[
sam.sam3Revision(
await sam.sam3Precision(undefined, o.sam3.precision),
),
o.sam3,
3,
],
]
: []),
]);
const hash = crypto
.createHash("sha256")
.update(key)
.digest("hex")
.slice(0, 24);
fs.mkdirSync(dir, { recursive: true });
this.cacheFile = path.join(
dir,
`${path.basename(this.source)}.${hash}.json`,
);
} catch {
this.cacheFile = null; // remote URL or unreadable: memory only
}
return this.cacheFile;
}
private async loadCache(): Promise<void> {
const file = await this.cachePath();
if (!file) return;
try {
const fs = await import("node:fs");
if (!fs.existsSync(file)) return;
const json = JSON.parse(fs.readFileSync(file, "utf8")) as {
width: number;
height: number;
cell: number;
frames: Record<
string,
{ kp: number[] | null; mask: string | null; score?: number }
>;
};
if (json.cell !== this.cell) return;
this.setSize(json.width, json.height);
for (const [c, r] of Object.entries(json.frames)) {
this.results.set(Number(c), {
kp: r.kp ? Float32Array.from(r.kp) : null,
score: r.score,
mask: r.mask ? new Uint8Array(Buffer.from(r.mask, "base64")) : null,
});
}
} catch {
// A damaged cache is recomputed.
}
}
private async saveCache(): Promise<void> {
this.unsaved = 0;
const file = await this.cachePath();
if (!file || !this.sourceWidth) return;
const frames: Record<
string,
{ kp: number[] | null; mask: string | null; score?: number }
> = {};
for (const [c, r] of this.results) {
frames[c] = {
kp: r.kp ? Array.from(r.kp, (v) => Math.round(v * 100) / 100) : null,
mask: r.mask ? Buffer.from(r.mask).toString("base64") : null,
score: r.score,
};
}
const fs = await import("node:fs");
const tmp = `${file}.${process.pid}.tmp`;
fs.writeFileSync(
tmp,
JSON.stringify({
width: this.sourceWidth,
height: this.sourceHeight,
cell: this.cell,
frames,
}),
);
fs.renameSync(tmp, file);
}
/** Subjects live beside the results: large, and only matte layers read them. */
private async subjectsPath(): Promise<string | null> {
const file = await this.cachePath();
return file ? file.replace(/\.json$/, ".subjects.json") : null;
}
private async loadSubjects(): Promise<void> {
const file = await this.subjectsPath();
if (!file) return;
try {
const fs = await import("node:fs");
if (!fs.existsSync(file)) return;
const json = JSON.parse(fs.readFileSync(file, "utf8")) as {
width: number;
height: number;
frames: Record<string, StoredSubject | null>;
};
if (this.sourceWidth && json.width !== this.sourceWidth) return;
if (!this.sourceWidth) this.setSize(json.width, json.height);
for (const [c, s] of Object.entries(json.frames)) {
const lengths = Buffer.from(s?.lengths ?? "", "base64");
this.subjects.set(
Number(c),
s && {
category: s.category,
area: s.area,
...(s.scores && { scores: s.scores }),
values: new Uint8Array(Buffer.from(s.values, "base64")),
lengths: new Uint32Array(
lengths.buffer.slice(
lengths.byteOffset,
lengths.byteOffset + lengths.byteLength,
),
),
},
);
}
} catch {
// A damaged cache is recomputed.
}
}
/**
* Mattes live beside the subjects they refine, keyed by the matting model
* and settings, so toggling matting never recomputes the masks.
*/
private async mattesPath(): Promise<string | null> {
const file = await this.cachePath();
const node = await nodeModules();
const settings = this.matteSettings;
if (!file || !node || !settings) return null;
const hash = node.crypto
.createHash("sha256")
.update(
JSON.stringify([
MATTE_CACHE_VERSION,
MATTE_MODEL,
settings.band,
settings.resolution,
]),
)
.digest("hex")
.slice(0, 12);
return file.replace(/\.json$/, `.matte-${hash}.json`);
}
private async loadMattes(): Promise<void> {
const file = await this.mattesPath();
if (!file) return;
try {
const fs = await import("node:fs");
if (!fs.existsSync(file)) return;
const json = JSON.parse(fs.readFileSync(file, "utf8")) as {
width: number;
height: number;
frames: Record<
string,
{ alpha: StoredRuns; labels?: StoredRuns } | null
>;
};
if (json.width !== this.sourceWidth || json.height !== this.sourceHeight)
return;
for (const [c, m] of Object.entries(json.frames)) {
this.mattes.set(
Number(c),
m && {
alpha: fromStoredRuns(m.alpha),
...(m.labels && { labels: fromStoredRuns(m.labels) }),
},
);
}
} catch {
// A damaged cache is recomputed.
}
}
private async saveMattes(): Promise<void> {
this.unsavedMattes = 0;
const file = await this.mattesPath();
if (!file || !this.sourceWidth) return;
const frames: Record<
string,
{ alpha: StoredRuns; labels?: StoredRuns } | null
> = {};
for (const [c, m] of this.mattes) {
frames[c] = m && {
alpha: toStoredRuns(m.alpha),
...(m.labels && { labels: toStoredRuns(m.labels) }),
};
}
const fs = await import("node:fs");
const tmp = `${file}.${process.pid}.tmp`;
fs.writeFileSync(
tmp,
JSON.stringify({
width: this.sourceWidth,
height: this.sourceHeight,
frames,
}),
);
fs.renameSync(tmp, file);
}
private async saveSubjects(): Promise<void> {
this.unsavedSubjects = 0;
const file = await this.subjectsPath();
if (!file || !this.sourceWidth) return;
const frames: Record<string, StoredSubject | null> = {};
for (const [c, s] of this.subjects) {
frames[c] = s && {
category: s.category,
area: s.area,
...(s.scores && { scores: s.scores }),
values: Buffer.from(s.values).toString("base64"),
lengths: Buffer.from(
s.lengths.buffer,
s.lengths.byteOffset,
s.lengths.byteLength,
).toString("base64"),
};
}
const fs = await import("node:fs");
const tmp = `${file}.${process.pid}.tmp`;
fs.writeFileSync(
tmp,
JSON.stringify({
width: this.sourceWidth,
height: this.sourceHeight,
frames,
}),
);
fs.renameSync(tmp, file);
}
}
/**
* The person in a {@link ClipVision}: pose keypoints and silhouette at any
* composition frame, in source pixels or, after `place()`, composition
* pixels. Reads are synchronous; the frame must be analysed (it is while
* rendering, inside a live physics world, or after `await clip.ready(frame)`).
*/
export class PersonTrack {
/** @internal */
constructor(
readonly clip: ClipVision,
private readonly t?: Transform,
private readonly placement?: PersonPlacement,
) {}
/** The same person in composition pixels for a clip laid out at `placement`. */
place(placement: PersonPlacement): PersonTrack {
return new PersonTrack(this.clip, undefined, placement);
}
private transform(): Transform {
if (this.t) return this.t;
const p = this.placement;
const { width: sw, height: sh } = this.clip.size();
if (!p || !sw) return { sx: 1, sy: 1, ox: 0, oy: 0 };
let sx = p.width / sw;
let sy = p.height / sh;
if (p.fit !== "fill") {
const s = p.fit === "contain" ? Math.min(sx, sy) : Math.max(sx, sy);
sx = s;
sy = s;
}
return {
sx,
sy,
ox: (p.x ?? 0) + (p.width - sw * sx) / 2,
oy: (p.y ?? 0) + (p.height - sh * sy) / 2,
};
}
/** A keypoint at a composition frame (fractional frames interpolate). */
point(
name: PoseKeypointName,
frame: number,
): { x: number; y: number; visibility: number } {
const k = keypointIndex(name);
const c = this.clip.clipFrame(frame);
const c0 = Math.floor(c);
const u = c - c0;
const a = this.clip.keypoint(k, c0);
const b = u > 0 ? this.clip.keypoint(k, c0 + 1) : a;
const t = this.transform();
return {
x: (a[0] + (b[0] - a[0]) * u) * t.sx + t.ox,
y: (a[1] + (b[1] - a[1]) * u) * t.sy + t.oy,
visibility: a[2] + (b[2] - a[2]) * u,
};
}
/**
* Aim constraints that make a 3D character copy this person's pose, for
* a glTF model's `aim`: `Layer.glb(src, { aim: person.retarget({ rig: "mixamo" }) })`.
* Place the person first (`place()`), so directions read in composition px.
*/
retarget(options?: RetargetOptions): AimConstraint[] {
return retargetPose(this, options);
}
/** Signals for a keypoint, for `.signal("x", …)`; read while rendering. */
signals(name: PoseKeypointName): {
x: FrameSignal<number>;
y: FrameSignal<number>;
visibility: FrameSignal<number>;
} {
const read = (axis: "x" | "y" | "visibility") =>
Signal.builder({
type: "custom",
fn: (ctx) => this.point(name, ctx.frame)[axis],
}) as FrameSignal<number>;
return { x: read("x"), y: read("y"), visibility: read("visibility") };
}
/**
* @internal The softened silhouette grid at a composition frame and how
* placed px map onto it (`cell = (px − origin) / scale − 0.5`), for GPU
* consumers such as particles. Null when no person was found.
*/
silhouetteGrid(frame: number): {
data: Float32Array;
width: number;
height: number;
originX: number;
originY: number;
scaleX: number;
scaleY: number;
} | null {
const grid = this.clip.blurredMask(Math.round(this.clip.clipFrame(frame)));
if (!grid) return null;
const { cell, width, height } = this.clip.grid();
const t = this.transform();
return {
data: grid,
width,
height,
originX: t.ox,
originY: t.oy,
scaleX: cell * t.sx,
scaleY: cell * t.sy,
};
}
/**
* The silhouette near a point at a composition frame: `alpha` 0–1 (softened
* over a few cells; inside > 0.5) and its gradient `gx`, `gy` per px,
* pointing into the body. Push things out with `-gradient`.
*/
sample(
frame: number,
x: number,
y: number,
): { alpha: number; gx: number; gy: number } {
const grid = this.clip.blurredMask(Math.round(this.clip.clipFrame(frame)));
if (!grid) return { alpha: 0, gx: 0, gy: 0 };
const { cell, width: w, height: h } = this.clip.grid();
const t = this.transform();
const cx = (x - t.ox) / t.sx / cell - 0.5;
const cy = (y - t.oy) / t.sy / cell - 0.5;
const read = (gx: number, gy: number) =>
gx < 0 || gy < 0 || gx >= w || gy >= h ? 0 : grid[gy * w + gx];
const bil = (px: number, py: number) => {
const x0 = Math.floor(px);
const y0 = Math.floor(py);
const ux = px - x0;
const uy = py - y0;
const a = read(x0, y0) + (read(x0 + 1, y0) - read(x0, y0)) * ux;
const b =
read(x0, y0 + 1) + (read(x0 + 1, y0 + 1) - read(x0, y0 + 1)) * ux;
return a + (b - a) * uy;
};
return {
alpha: bil(cx, cy),
gx: (bil(cx + 1, cy) - bil(cx - 1, cy)) / (2 * cell * t.sx),
gy: (bil(cx, cy + 1) - bil(cx, cy - 1)) / (2 * cell * t.sy),
};
}
}
function keypointIndex(name: PoseKeypointName): number {
const i = COCO17_KEYPOINT_NAMES.indexOf(name);
if (i < 0) {
throw new Error(
`Unknown keypoint "${name}". Use one of ${COCO17_KEYPOINT_NAMES.join(", ")}.`,
);
}
return i;
}
/** RGBA pixels of a decoded frame (headless buffer, or a canvas in the browser). */
async function decodedPixels(
frame: {
width: number;
height: number;
buffer?: Uint8Array;
canvas?: unknown;
} | null,
): Promise<{
data: Uint8Array | Uint8ClampedArray;
width: number;
height: number;
} | null> {
if (!frame || frame.width <= 0) return null;
if (frame.buffer)
return { data: frame.buffer, width: frame.width, height: frame.height };
const canvas = frame.canvas as
| {
getContext(type: "2d"): {
getImageData(
x: number,
y: number,
w: number,
h: number,
): { data: Uint8ClampedArray };
} | null;
}
| undefined;
const ctx = canvas?.getContext("2d");
if (!ctx) return null;
return {
data: ctx.getImageData(0, 0, frame.width, frame.height).data,
width: frame.width,
height: frame.height,
};
}
type Segmentation = Awaited<ReturnType<VisionRunner["segment"]>>;
/**
* The largest person's mask, plus instances that overlap it and are no more
* than twice its size (a flowing dress often comes back as "kite").
*/
function personSilhouette(seg: Segmentation): Uint8Array | undefined {
return mergeSubjectMask(seg.masks, {
classes: ["person"],
overlap: 0.2,
maxGrowth: 3.5,
})?.mask;
}
function maskSum(m: Uint8Array | null | undefined): number {
if (!m) return 0;
let s = 0;
for (let i = 0; i < m.length; i++) s += m[i];
return s;
}
/**
* Bridges 1–2 frame segmentation part dropouts (e.g. a fast-spinning skirt
* missed on a single frame) across the smoothing window.
*/
function bridgedMask(
results: Map<number, FrameResult>,
c: number,
smoothing: number,
): Uint8Array | null {
const base = results.get(c)?.mask ?? null;
if (smoothing <= 0) return base;
const baseSum = maskSum(base);
let prevFrame = -1;
let prevMask: Uint8Array | null = null;
let nextFrame = -1;
let nextMask: Uint8Array | null = null;
const threshold = Math.max(1000, baseSum * 1.35);
for (let d = 1; d <= smoothing + 1; d++) {
if (prevFrame < 0 && c - d >= 0) {
const m = results.get(c - d)?.mask ?? null;
if (maskSum(m) > threshold) {
prevFrame = c - d;
prevMask = m;
}
}
if (nextFrame < 0) {
const m = results.get(c + d)?.mask ?? null;
if (maskSum(m) > threshold) {
nextFrame = c + d;
nextMask = m;
}
}
}
if (!prevMask && !nextMask) return base;
const ref = prevMask ?? nextMask;
if (!ref) return base;
const out = base ? new Uint8Array(base) : new Uint8Array(ref.length);
if (prevMask && nextMask && nextFrame > prevFrame) {
const u = (c - prevFrame) / (nextFrame - prevFrame);
for (let i = 0; i < out.length; i++) {
const v = Math.round(prevMask[i] + (nextMask[i] - prevMask[i]) * u);
if (v > out[i]) out[i] = v;
}
} else {
for (let i = 0; i < out.length; i++) {
if (ref[i] > out[i]) out[i] = ref[i];
}
}
return out;
}
function downsampleMean(
src: Uint8Array,
w: number,
h: number,
cell: number,
dst: Uint8Array,
mw: number,
mh: number,
): void {
for (let my = 0; my < mh; my++) {
for (let mx = 0; mx < mw; mx++) {
let sum = 0;
let n = 0;
for (let y = my * cell; y < Math.min(h, (my + 1) * cell); y++) {
for (let x = mx * cell; x < Math.min(w, (mx + 1) * cell); x++) {
sum += src[y * w + x];
n++;
}
}
dst[my * mw + mx] = n ? Math.round(sum / n) : 0;
}
}
}
function toStoredRuns(runs: Runs): StoredRuns {
return {
values: Buffer.from(runs.values).toString("base64"),
lengths: Buffer.from(
runs.lengths.buffer,
runs.lengths.byteOffset,
runs.lengths.byteLength,
).toString("base64"),
};
}
function fromStoredRuns(stored: StoredRuns): Runs {
const lengths = Buffer.from(stored.lengths, "base64");
return {
values: new Uint8Array(Buffer.from(stored.values, "base64")),
lengths: new Uint32Array(
lengths.buffer.slice(
lengths.byteOffset,
lengths.byteOffset + lengths.byteLength,
),
),
};
}
/** A mask as (value, run length) pairs: long flat runs with soft edges. */
function encodeRuns(mask: Uint8Array): {
values: Uint8Array;
lengths: Uint32Array;
} {
let runs = 0;
for (let i = 1; i <= mask.length; i++)
if (i === mask.length || mask[i] !== mask[i - 1]) runs++;
const values = new Uint8Array(runs);
const lengths = new Uint32Array(runs);
let r = 0;
let start = 0;
for (let i = 1; i <= mask.length; i++) {
if (i === mask.length || mask[i] !== mask[start]) {
values[r] = mask[start];
lengths[r++] = i - start;
start = i;
}
}
return { values, lengths };
}
function decodeRuns(
runs: { values: Uint8Array; lengths: Uint32Array },
size: number,
): Uint8Array {
const out = new Uint8Array(size);
let at = 0;
for (let r = 0; r < runs.values.length; r++) {
const end = Math.min(size, at + runs.lengths[r]);
if (runs.values[r] !== 0) out.fill(runs.values[r], at, end);
at = end;
}
return out;
}
/** Separable box blur to 0–1, so gradients reach a little past the edge. */
function boxBlur(
src: Uint8Array,
w: number,
h: number,
r: number,
): Float32Array {
const tmp = new Float32Array(w * h);
const dst = new Float32Array(w * h);
for (let y = 0; y < h; y++) {
for (let x = 0; x < w; x++) {
let s = 0;
let n = 0;
for (let k = -r; k <= r; k++) {
const xx = x + k;
if (xx >= 0 && xx < w) {
s += src[y * w + xx];
n++;
}
}
tmp[y * w + x] = s / n / 255;
}
}
for (let y = 0; y < h; y++) {
for (let x = 0; x < w; x++) {
let s = 0;
let n = 0;
for (let k = -r; k <= r; k++) {
const yy = y + k;
if (yy >= 0 && yy < h) {
s += tmp[yy * w + x];
n++;
}
}
dst[y * w + x] = s / n;
}
}
return dst;
}
type SeedFrame = {
kp: number[] | null;
mask: Uint8Array | null;
/** Source-sized subject mask for render-time mattes; null = no subject. */
subject?: Uint8Array | null;
/** Prompted frames: a source-sized track label map (track ID per pixel, 0 for none). */
tracks?: Uint8Array;
};
/** @internal Test hook: a clip whose results are given instead of computed. */
export function clipVisionFromResults(
fps: number,
size: { width: number; height: number; cell: number },
frames: SeedFrame[],
options: ClipVisionOptions = {},
): ClipVision {
const clip = new ClipVision("memory", fps, {
...options,
maskCell: size.cell,
cacheDir: false,
});
return seedClipVision(clip, size, frames);
}
/** @internal Test hook: gives a clip's per-frame results instead of computing them. */
export function seedClipVision(
clip: ClipVision,
size: { width: number; height: number },
frames: SeedFrame[],
): ClipVision {
const c = clip as unknown as {
setSize(w: number, h: number): void;
results: Map<number, FrameResult>;
subjects: Map<number, SubjectResult | null>;
};
c.setSize(size.width, size.height);
frames.forEach((f, i) => {
c.results.set(i, {
kp: f.kp ? Float32Array.from(f.kp) : null,
mask: f.mask,
});
if (f.tracks) {
let area = 0;
let count = 0;
for (const id of f.tracks) {
if (id) area++;
if (id > count) count = id;
}
c.subjects.set(i, {
category: "object",
area,
scores: Array.from({ length: count }, () => 1),
...encodeRuns(f.tracks),
});
return;
}
if (f.subject === undefined) return;
let area = 0;
for (const v of f.subject ?? []) area += v;
c.subjects.set(
i,
f.subject && {
category: "person",
area: Math.round(area / 255),
...encodeRuns(f.subject),
},
);
});
return clip;
}
/** A stored subject as 0/255 alpha: label maps merge every track, or keep track `id` only. */
function subjectAlpha(
stored: SubjectResult,
size: number,
id?: number,
): Uint8Array {
const values = decodeRuns(stored, size);
if (!stored.scores) return values;
return values.map((label) =>
(id === undefined ? label > 0 : label === id) ? 255 : 0,
);
}
/** Every track's mask merged, for matching one pose to the whole selection. */
function unionMask(
objects: readonly Pick<Sam3ObjectResult, "mask">[],
image: { width: number; height: number },
): Uint8Array {
const union = new Uint8Array(image.width * image.height);
for (const { mask } of objects)
for (let i = 0; i < union.length; i++)
union[i] = Math.max(union[i] ?? 0, mask[i] ?? 0);
return union;
}
interface NodeModules {
fs: typeof import("node:fs");
path: typeof import("node:path");
os: typeof import("node:os");
crypto: typeof import("node:crypto");
}
let nodeModulesLoaded: Promise<NodeModules | null> | undefined;
/** Node's file modules, or null in a browser. */
function nodeModules(): Promise<NodeModules | null> {
nodeModulesLoaded ??=
typeof process === "undefined" || !process.versions?.node
? Promise.resolve(null)
: Promise.all([
import("node:fs"),
import("node:path"),
import("node:os"),
import("node:crypto"),
]).then(([fs, path, os, crypto]) => ({ fs, path, os, crypto }));
return nodeModulesLoaded;
}
/** A depth frame written by `saveDepth`; null when the file is missing or damaged. */
function readDepthFile(
fs: NodeModules["fs"],
file: string,
): StoredDepth | null {
try {
const buf = fs.readFileSync(file);
if (buf.byteLength < 8) return null;
const width = buf.readUInt32LE(0);
const height = buf.readUInt32LE(4);
if (buf.byteLength !== 8 + width * height * 2) return null;
const data = new Uint16Array(width * height);
Buffer.from(data.buffer).set(buf.subarray(8));
return { width, height, data };
} catch {
return null;
}
}