Code-first video & image, rendered natively on WebGPU.
feat: depth and relief, footage as a 3D surface
okanasl34bb85645dcd
diff --git a/.changeset/vision-depth-relief.md b/.changeset/vision-depth-relief.mdnew file mode 100644index 00000000..0bdacd21--- /dev/null+++ b/.changeset/vision-depth-relief.md@@ -0,0 +1,10 @@+---+"framefields": minor+---++Depth and relief: footage as a 3D surface.++- `comp.vision(clip, { depth: true })` analyses each frame's relative depth with Depth Anything V2 Small (Apache-2.0, pinned by revision and SHA-256), ahead of render and cached one file per frame. Read it with `clip.depthMap(frame)`; `runner.depth(image)` runs it on one image.+- `withVision({ mode: "depth" })` draws a clip's depth as gray (white = near), always for the frame on screen; it makes its depth clip when there is none.+- `layer.withRelief(heightLayer, { amount })` lifts a 3D layer into a surface by another layer's brightness, along the default camera's rays: from that lens it is the flat layer, pixel for pixel; a moving camera sees it in 3D, lit by the scene and hiding what passes behind it. With a depth layer, footage joins the 3D scene.+- Layers inside a `matte` or `relief` now link to their `comp.vision` clips too.diff --git a/nodes/node-compositor/src/renderers/webgpu-renderer.ts b/nodes/node-compositor/src/renderers/webgpu-renderer.tsindex 88200bde..c69ca7a6 100644--- a/nodes/node-compositor/src/renderers/webgpu-renderer.ts+++ b/nodes/node-compositor/src/renderers/webgpu-renderer.ts@@ -1149,6 +1149,8 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {const is3DScene = Boolean(cameraVM || has3DLayer || lightVMs.length > 0);let camera: Camera3DPose | null = null;+ /** Where a relief keeps its look from: the default camera at the scene's field of view. */+ let reliefEye: [number, number, number] = [0, 0, 0];let dofEnabled = false;const sceneLights: Light3DOptions[] = [];let sceneEnvironment: Environment3DOptions | null = null;@@ -1160,6 +1162,7 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {const camStub = camOp?.id ? targetsById[camOp.id] : undefined;const fov = camStub?.cameraFov ?? camOp?.lens?.fov ?? 50;camera = Camera3D.createDefaultCamera(nativeWidth, nativeHeight, fov);+ reliefEye = [...camera.eye] as [number, number, number];camera.near = camOp?.lens?.near ?? 1.0;camera.far = camOp?.lens?.far ?? 50000.0;camera.zoom = camStub?.cameraZoom ?? camOp?.lens?.zoom ?? 1.0;@@ -1355,11 +1358,37 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {meshBuffers?: GPUMeshBuffers;};const pending3DItems: Pending3DItem[] = [];- /** Textures the queued 3D items still read: their layers and mattes. */+ /** Textures the queued 3D items still read: their layers, mattes and reliefs. */const pendingTextures = (): GPUTexture[] =>pending3DItems- .flatMap((q) => [q.texture, q.opts.matte])+ .flatMap((q) => [+ q.texture,+ q.opts.matte,+ q.kind === "quad" ? q.opts.relief?.texture : undefined,+ ]).filter((t): t is GPUTexture => Boolean(t));+ /** Reliefs of 3D layers by owner id, drawn just before their owner. */+ const reliefs = new Map<string, { texture: GPUTexture; amount: number }>();+ /** The relief a 3D layer drawn at (x, y, w, h) of its pass carries, if any. */+ const reliefOpts = (+ id: string | undefined,+ x: number,+ y: number,+ w: number,+ h: number,+ ): Pick<DrawQuad3DOpts, "relief"> => {+ const r = id === undefined ? undefined : reliefs.get(id);+ if (!r) return {};+ const { width, height } = r.texture;+ return {+ relief: {+ texture: r.texture,+ uvRect: [x / width, y / height, w / width, h / height],+ amount: r.amount,+ eye: reliefEye,+ },+ };+ };const flush3DPass = (curTex: GPUTexture,@@ -1812,16 +1841,18 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {let activeTex = initialTex;let activeView = initialView;let activeExcludes = excludes;- // Matte layers are not drawn on their own: their owner draws them.+ // Matte and relief layers are not drawn on their own: their owner draws them.const mattesByOwner = new Map<string, VirtualMediaData>();+ const reliefsByOwner = new Map<string, VirtualMediaData>();for (const vm of vms) {- const owner = (vm.operation as any)?.matteOf;- if (owner) mattesByOwner.set(owner, vm);+ const op = vm.operation as any;+ if (op?.matteOf) mattesByOwner.set(op.matteOf, vm);+ if (op?.reliefOf) reliefsByOwner.set(op.reliefOf, vm);}for (const child of sorted) {const lop = child.operation as any;if (lop.kind === "camera" || lop.kind === "light") continue;- if (lop.matteOf) continue;+ if (lop.matteOf || lop.reliefOf) continue;const targetId = lop.id || lop.inputHandleId;const target = targetId ? targetsById[targetId] : undefined;const isChild3D =@@ -1860,53 +1891,64 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {continue;}nodeIndex += 1;- const matteVM = lop.matteId ? mattesByOwner.get(lop.id) : undefined;- if (matteVM) {- // Draws a layer alone on transparent. 3D content it queues is- // rendered into it right away, in a pass of its own.- const isolate = async (vm: VirtualMediaData, index: number) => {- const queued = pending3DItems.splice(0);- const busy = [- targetTexture,- activeTex,- ...activeExcludes,- ...queued.flatMap((q) => [q.texture, q.opts.matte]),- ].filter((t): t is GPUTexture => Boolean(t));- const tex = ctx.renderer.getTemporaryTexture(- targetW,- targetH,- busy,- );- const view = tex.createView();- ctx.renderer- .beginFrame(- encoder,- view,- { r: 0, g: 0, b: 0, a: 0 },- targetW,- targetH,- "clear",- )- .end();- const out = await renderLayerVM(- vm,- index,- tex,+ // Draws a layer alone on transparent. 3D content it queues is+ // rendered into it right away, in a pass of its own.+ const isolate = async (vm: VirtualMediaData, index: number) => {+ const queued = pending3DItems.splice(0);+ const busy = [+ targetTexture,+ activeTex,+ ...activeExcludes,+ ...queued.flatMap((q) => [+ q.texture,+ q.opts.matte,+ q.kind === "quad" ? q.opts.relief?.texture : undefined,+ ]),+ ].filter((t): t is GPUTexture => Boolean(t));+ const tex = ctx.renderer.getTemporaryTexture(targetW, targetH, busy);+ const view = tex.createView();+ ctx.renderer+ .beginFrame(+ encoder,view,- busy.filter((t) => t !== targetTexture),- accumulatedOpacity,- parentMatrix,- parentRect,+ { r: 0, g: 0, b: 0, a: 0 },targetW,targetH,- parent3DMatrix,- false,- );- const drawn = out?.finalTex ?? tex;- const flat = flush3DPass(drawn, drawn.createView(), busy).finalTex;- pending3DItems.push(...queued);- return flat;- };+ "clear",+ )+ .end();+ const out = await renderLayerVM(+ vm,+ index,+ tex,+ view,+ busy.filter((t) => t !== targetTexture),+ accumulatedOpacity,+ parentMatrix,+ parentRect,+ targetW,+ targetH,+ parent3DMatrix,+ false,+ );+ const drawn = out?.finalTex ?? tex;+ const flat = flush3DPass(drawn, drawn.createView(), busy).finalTex;+ pending3DItems.push(...queued);+ return flat;+ };+ // A 3D layer's relief is drawn flat first; its quads then read it.+ const reliefVM = lop.reliefId ? reliefsByOwner.get(lop.id) : undefined;+ if (reliefVM && isChild3D) {+ const texture = await isolate(reliefVM, nodeIndex);+ activeExcludes = [...activeExcludes, texture];+ nodeIndex += 1;+ reliefs.set(lop.id, {+ texture,+ amount: lop.reliefAmount ?? 0,+ });+ }+ const matteVM = lop.matteId ? mattesByOwner.get(lop.id) : undefined;+ if (matteVM) {const mode: string = (matteVM.operation as any).matteMode ?? "alpha";const blend =mode === "luma"@@ -3839,6 +3881,7 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {metallic: target?.metallic ?? lop.metallic,...surfaceProps(target, lop),...(modelMatrix && { modelMatrix }),+ ...reliefOpts(lop.id, localX, localY, layerW, layerH),};pending3DItems.push({@@ -5519,6 +5562,7 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {metallic: target?.metallic ?? lop.metallic,...surfaceProps(target, lop),...(modelMatrix && { modelMatrix }),+ ...reliefOpts(lop.id, localX, localY, layerW, layerH),};// The texture quad carries the layer's backdrop and border; adiff --git a/nodes/node-vision/src/renderers/webgpu-renderer.ts b/nodes/node-vision/src/renderers/webgpu-renderer.tsindex 43c5d8b1..b3bcadf0 100644--- a/nodes/node-vision/src/renderers/webgpu-renderer.ts+++ b/nodes/node-vision/src/renderers/webgpu-renderer.ts@@ -562,6 +562,22 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {const childMedia = virtualMedia?.children?.[0];if (!childMedia) return;+ if (op.mode === "depth") {+ drawClipDepth(+ ctx,+ encoder,+ targetView,+ targetWidth,+ targetHeight,+ (op as { effect?: object }).effect,+ props.sourceTimeSec === undefined+ ? (props.frame ?? 0)+ : props.sourceTimeSec * (props.fps ?? 24),+ props.fps ?? 24,+ );+ return;+ }+// 1. Persistent per-node child texture (see childTextures*). Reading the// texture we just drew would race the end-of-frame submit; reading last// frame's already-submitted pixels at the start of this call is correct.@@ -1119,6 +1135,70 @@ function clipPoseFor(: undefined;}+/** Per-node upload target for depth frames (the depth model's resolution). */+const depthTextures = new WeakMap<object, GPUTexture>();++/**+ * Depth mode: the linked `comp.vision` clip's depth for this source frame as+ * gray (white = near), stretched over the layer. Depth is analysed ahead of+ * the render, so it belongs to this frame's pixels, never the previous one's.+ */+function drawClipDepth(+ ctx: Parameters<WebGPUNodeRenderer>[0]["ctx"],+ encoder: GPUCommandEncoder,+ targetView: GPUTextureView,+ width: number,+ height: number,+ effect: object | undefined,+ frame: number,+ fps: number,+): void {+ const source = linkedClip(effect);+ if (!source?.depth)+ throw new Error(+ "A depth layer reads its clip's depth: add the video to the composition (framefields links it to a `comp.vision(src, { depth: true })` clip).",+ );+ const depth = source.depth(frame, fps);+ const pass = ctx.renderer.beginFrame(+ encoder,+ targetView,+ { r: 0, g: 0, b: 0, a: 0 },+ width,+ height,+ "clear",+ );+ if (depth) {+ if (Math.abs(depth.width / depth.height / (width / height) - 1) > 0.02)+ throw new Error(+ `A depth layer keeps its clip's aspect ratio (${depth.width}×${depth.height} source vs a ${width}×${height} layer); crop after the depth layer instead.`,+ );+ let tex = effect ? depthTextures.get(effect) : undefined;+ if (!tex || tex.width !== depth.width || tex.height !== depth.height) {+ tex?.destroy();+ tex = ctx.device.createTexture({+ size: [depth.width, depth.height],+ format: "rgba8unorm",+ usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST,+ label: "vision_depth",+ });+ if (effect) depthTextures.set(effect, tex);+ }+ const gray = new Uint8Array(depth.width * depth.height * 4);+ for (let i = 0; i < depth.data.length; i++) {+ const v = Math.round((depth.data[i] as number) * 255);+ gray.set([v, v, v, 255], i * 4);+ }+ ctx.device.queue.writeTexture(+ { texture: tex },+ gray,+ { bytesPerRow: depth.width * 4 },+ [depth.width, depth.height],+ );+ ctx.renderer.drawTexture(pass, tex, { x: 0, y: 0, width, height });+ }+ pass.end();+}+function linkedClip(effect: object | undefined): ClipSubjectSource | undefined {return (effect as Record<symbol, unknown> | undefined)?.[CLIP_SUBJECT] as| ClipSubjectSourcediff --git a/nodes/node-vision/src/shared/config.ts b/nodes/node-vision/src/shared/config.tsindex 481ff8f5..7b8e6600 100644--- a/nodes/node-vision/src/shared/config.ts+++ b/nodes/node-vision/src/shared/config.ts@@ -18,6 +18,8 @@ import { z } from "zod";* - skeleton — COCO-17 pose skeleton of the primary person* - boxes — the child with tracked-object boxes* - tracking — boxes plus track centers+ * - depth — relative depth of the clip's frame as gray, white = near+ * (read from a linked `comp.vision` clip, analysed ahead of render)*/export const VISION_MODES = ["passthrough",@@ -27,6 +29,7 @@ export const VISION_MODES = ["skeleton","boxes","tracking",+ "depth",] as const;export type VisionMode = (typeof VISION_MODES)[number];diff --git a/packages/compositions/src/program/mattes.ts b/packages/compositions/src/program/mattes.tsindex af641e34..825ff25a 100644--- a/packages/compositions/src/program/mattes.ts+++ b/packages/compositions/src/program/mattes.ts@@ -1,7 +1,7 @@/**- * Track mattes: a node's `matte` is hoisted out into its own layer right- * after it, so layout places it like any sibling and the renderer can draw- * owner and matte separately and combine them.+ * Track mattes and reliefs: a node's `matte` or `relief` is hoisted out into+ * its own layer right after it, so layout places it like any sibling and the+ * renderer can draw owner and companion separately and combine them.*/import type { LayoutNode, MatteMode } from "./schema.js";@@ -9,25 +9,16 @@ type AnyNode = LayoutNode & Record<string, unknown>;const GEOMETRY = ["x", "y", "width", "height"] as const;-function hoistOne(owner: AnyNode): AnyNode[] {- const matte = owner.matte as { node: AnyNode; mode?: MatteMode } | undefined;- const { matte: _drop, ...rest } = owner;- const self = { ...rest } as AnyNode;- if (Array.isArray(self.children)) {- self.children = hoistMattes(self.children as AnyNode[]) as never;- }- if (!matte?.node) return [self];-- const src = matte.node;- const id = (src.id as string | undefined) ?? `${String(owner.id)}__matte`;- const node: AnyNode = {- position: "absolute",- ...src,- id,- matteOf: String(owner.id),- matteMode: matte.mode ?? "alpha",- };- // With no geometry of its own the matte covers its owner's box.+/** A companion layer drawn for its owner, never by itself. */+function companion(+ owner: AnyNode,+ src: AnyNode,+ suffix: string,+ tag: Record<string, unknown>,+): AnyNode {+ const id = (src.id as string | undefined) ?? `${String(owner.id)}__${suffix}`;+ const node: AnyNode = { position: "absolute", ...src, id, ...tag };+ // With no geometry of its own it covers its owner's box.if (GEOMETRY.every((k) => src[k] === undefined)) {for (const k of GEOMETRY) {if (typeof owner[k] === "number") node[k] = owner[k];@@ -44,11 +35,41 @@ function hoistOne(owner: AnyNode): AnyNode[] {if (Array.isArray(node.children)) {node.children = hoistMattes(node.children as AnyNode[]) as never;}- self.matteId = id;- return [self, node];+ return node;+}++function hoistOne(owner: AnyNode): AnyNode[] {+ const matte = owner.matte as { node: AnyNode; mode?: MatteMode } | undefined;+ const relief = owner.relief as { node: AnyNode; amount?: number } | undefined;+ const { matte: _matte, relief: _relief, ...rest } = owner;+ const self = { ...rest } as AnyNode;+ if (Array.isArray(self.children)) {+ self.children = hoistMattes(self.children as AnyNode[]) as never;+ }+ const out = [self];+ if (matte?.node) {+ const node = companion(owner, matte.node, "matte", {+ matteOf: String(owner.id),+ matteMode: matte.mode ?? "alpha",+ });+ self.matteId = node.id as string;+ out.push(node);+ }+ if (relief?.node) {+ // The relief is read as a flat picture over its owner's box.+ const node = companion(owner, relief.node, "relief", {+ reliefOf: String(owner.id),+ is3D: false,+ z: 0,+ });+ self.reliefId = node.id as string;+ self.reliefAmount = relief.amount ?? 0;+ out.push(node);+ }+ return out;}-/** Returns the tree with every `matte` turned into a sibling matte layer. */+/** Returns the tree with every `matte` and `relief` turned into a sibling companion layer. */export function hoistMattes<T extends LayoutNode>(nodes: readonly T[]): T[] {return nodes.flatMap((n) => hoistOne(n as AnyNode)) as unknown as T[];}diff --git a/packages/compositions/src/program/schema.ts b/packages/compositions/src/program/schema.tsindex 5f8ddbbf..c6646410 100644--- a/packages/compositions/src/program/schema.ts+++ b/packages/compositions/src/program/schema.ts@@ -778,6 +778,23 @@ const BaseNodeFields = {/** Set by the compiler: this layer is the matte of node `matteOf` (not drawn by itself). */matteOf: z.string().optional(),matteMode: MatteModeSchema.optional(),+ /**+ * Relief (3D layers): another layer whose brightness pushes this one's+ * surface toward the camera, white by `amount` px, along the default+ * camera's rays, so that lens still sees the flat picture. Laid out and+ * timed like a matte; drawn flat over this node's box.+ */+ relief: z+ .object({+ node: z.custom<unknown>((v) => typeof v === "object" && v !== null),+ amount: z.number().optional(),+ })+ .optional(),+ /** Set by the compiler: the id of this node's hoisted relief layer. */+ reliefId: z.string().optional(),+ /** Set by the compiler: this layer is the relief of node `reliefOf` (not drawn by itself). */+ reliefOf: z.string().optional(),+ reliefAmount: z.number().optional(),/** Per-node motion: tracks keyed by node id. */animation: LayerAnimationSchema.default({ tracks: [] }),/** Vision configuration or effect attached directly to the node. */@@ -839,6 +856,10 @@ export interface CompositionNodeBase {matteId?: string;matteOf?: string;matteMode?: MatteMode;+ relief?: { node: unknown; amount?: number };+ reliefId?: string;+ reliefOf?: string;+ reliefAmount?: number;envIntensity?: number;startFrame?: number;durationFrames?: number;diff --git a/packages/framefields/NOTICE.md b/packages/framefields/NOTICE.mdindex eac4655e..f0279ed7 100644--- a/packages/framefields/NOTICE.md+++ b/packages/framefields/NOTICE.md@@ -12,3 +12,7 @@ framefields is licensed under Apache-2.0. It contains no model weights: every vi| SAM 3 (ONNX exports of Meta's SAM 3 checkpoints) | Meta SAM License | required at runtime |Before SAM 3 weights are downloaded or run, you must accept Meta's SAM License (https://github.com/facebookresearch/sam3/blob/main/LICENSE): pass `acceptSamLicense: true` in `comp.vision` options (or `acceptLicense: true` to `Sam3Tracker`), or set `FRAMEFIELDS_ACCEPT_SAM_LICENSE=1`. Your use of SAM 3 is then governed by that license.++## Depth++`comp.vision(clip, { depth: true })` and `withVision({ mode: "depth" })` estimate depth with Depth Anything V2 **Small** (onnx-community export of `depth-anything/Depth-Anything-V2-Small`), whose weights are Apache-2.0. The larger Depth Anything V2 checkpoints are licensed CC-BY-NC-4.0 and are not used.diff --git a/packages/framefields/src/clip-vision.ts b/packages/framefields/src/clip-vision.tsindex 8eb57f08..5290ea36 100644--- a/packages/framefields/src/clip-vision.ts+++ b/packages/framefields/src/clip-vision.ts@@ -9,6 +9,7 @@ import {type ClipPose,type ClipSubjectSource,COCO17_KEYPOINT_NAMES,+ type DepthMap,type InstanceMask,maskObject,mergeSubjectMask,@@ -51,6 +52,12 @@ export interface ClipVisionOptions {pose?: boolean;/** Track the person's silhouette, dress and held things included (default true). */mask?: boolean;+ /**+ * Relative depth of every frame (Depth Anything V2 Small, near = 1), read by+ * `withVision({ mode: "depth" })` layers and {@link ClipVision.depthMap}. A depth+ * clip tracks no pose or silhouette unless `pose` / `mask` ask for them.+ */+ depth?: boolean;/** Silhouette cell size in source pixels (default 8: a 1920×1080 clip gives a 240×135 grid). */maskCell?: number;/** Keypoints are averaged over ±`smoothing` frames, weighted by confidence (default 2; 0 = raw). */@@ -118,6 +125,17 @@ interface Transform {const HOLD = 24;const CACHE_VERSION = 1;+const DEPTH_CACHE_VERSION = 1;+const DEPTH_MODEL = "depth-anything-v2-s";+/** Depth frames held in memory when they also live on disk (each is about 1 MB). */+const DEPTH_MEMORY = 48;++/** A frame's depth as stored: 0 … 65535 for far … near. */+interface StoredDepth {+ width: number;+ height: number;+ data: Uint16Array;+}type VisionSam = typeof import("@framefields/vision-sam");@@ -184,6 +202,17 @@ export class ClipVision implements ClipSubjectSource {private readonly subjects = new Map<number, SubjectResult | null>();private wantSubjects = false;private subjectsLoaded?: Promise<void>;+ /** Depth per source frame; null when the frame did not decode. Bounded when on disk. */+ private readonly depths = new Map<number, StoredDepth | null>();+ /** Source frames whose depth is on disk. */+ private readonly depthsOnDisk = new Set<number>();+ private depthDir?: Promise<string | null>;+ /** Set once the depth folder is known: render-time reads are synchronous. */+ private depthDisk?: {+ dir: string;+ fs: NodeModules["fs"];+ path: NodeModules["path"];+ };/** @internal Use `comp.vision(source, options)`. */constructor(source: string, fps: number, options: ClipVisionOptions = {}) {@@ -382,7 +411,41 @@ export class ClipVision implements ClipSubjectSource {get tracksPose(): boolean {return this.exact? this.options.pose === true- : (this.options.pose ?? true);+ : (this.options.pose ?? !this.tracksDepth);+ }++ /** Whether frames carry the person's silhouette: on by default, off for depth clips. */+ get tracksMask(): boolean {+ return this.options.mask ?? !this.tracksDepth;+ }++ /** Whether frames carry relative depth. */+ get tracksDepth(): boolean {+ return this.options.depth === true;+ }++ /**+ * The relative depth (near = 1) of composition frame `frame`, at the depth+ * model's resolution; undefined until that frame is analysed (`ready`).+ */+ depthMap(frame: number): DepthMap | undefined {+ return this.depth(frame, this.compositionFps) ?? undefined;+ }++ /**+ * @internal The relative depth of a source frame of the clip (`frame` at+ * `fps`) for a render-time depth layer: undefined until analysed, null when+ * the frame did not decode.+ */+ depth(frame: number, fps: number): DepthMap | null | undefined {+ const c = this.sourceFrame(frame, fps);+ if (c === undefined || !this.tracksDepth) return undefined;+ const stored = this.storedDepth(c);+ if (!stored) return stored;+ const data = new Float32Array(stored.data.length);+ for (let i = 0; i < data.length; i++)+ data[i] = (stored.data[i] as number) / 65535;+ return { data, width: stored.width, height: stored.height };}/**@@ -447,6 +510,11 @@ export class ClipVision implements ClipSubjectSource {}private analysed(c: number): boolean {+ return this.analysedSubject(c) && (!this.tracksDepth || this.hasDepth(c));+ }++ /** Everything but depth, which is a pass of its own. */+ private analysedSubject(c: number): boolean {return this.results.has(c) && (!this.wantSubjects || this.subjects.has(c));}@@ -468,8 +536,10 @@ export class ClipVision implements ClipSubjectSource {this.subjectsLoaded ??= this.loadSubjects();await this.subjectsLoaded;}+ if (this.tracksDepth) await this.computeDepth(from, to);let missing = false;- for (let c = from; c <= to; c++) if (!this.analysed(c)) missing = true;+ for (let c = from; c <= to; c++)+ if (!this.analysedSubject(c)) missing = true;if (!missing) return;if (this.options.sam2) {@@ -481,20 +551,11 @@ export class ClipVision implements ClipSubjectSource {return;}const wantPose = this.tracksPose;- const wantMask = this.options.mask ?? true;- this.runner ??= VisionRunner.create({- modelsDir: this.options.modelsDir,- baseUrl: this.options.baseUrl,- ...(this.options.confidence !== undefined && {- confidence: this.options.confidence,- }),- ...(this.options.variant !== undefined && {- variant: this.options.variant,- }),- });+ const wantMask = this.tracksMask;+ const runner = this.visionRunner();const decoder = this.decoder();for (let c = from; c <= to; c++) {- if (this.analysed(c)) continue;+ if (this.analysedSubject(c)) continue;const image = await decodedPixels(await decoder.getFrame(c / this.fps));if (!image) {this.results.set(c, { kp: null, mask: null });@@ -507,7 +568,7 @@ export class ClipVision implements ClipSubjectSource {let kp: Float32Array | null = known?.kp ?? null;let mask: Uint8Array | null = known?.mask ?? null;if (wantPose && !known) {- const person = (await this.runner.pose(image)).people[0];+ const person = (await runner.pose(image)).people[0];if (person) {kp = new Float32Array(51);for (const [k, p] of person.keypoints.entries()) {@@ -517,7 +578,7 @@ export class ClipVision implements ClipSubjectSource {}const seg =(wantMask && !known) || this.wantSubjects- ? await this.runner.segment(image)+ ? await runner.segment(image): undefined;if (seg && this.wantSubjects) {const subject = mergeSubjectMask(seg.masks);@@ -557,6 +618,140 @@ export class ClipVision implements ClipSubjectSource {if (this.unsavedSubjects > 0) await this.saveSubjects();}+ private visionRunner(): VisionRunner {+ this.runner ??= VisionRunner.create({+ modelsDir: this.options.modelsDir,+ baseUrl: this.options.baseUrl,+ ...(this.options.confidence !== undefined && {+ confidence: this.options.confidence,+ }),+ ...(this.options.variant !== undefined && {+ variant: this.options.variant,+ }),+ });+ return this.runner;+ }++ /** Depth of every source frame in `from…to` not analysed yet, saved one file per frame. */+ private async computeDepth(from: number, to: number): Promise<void> {+ const dir = await this.depthCacheDir();+ let decoder: ReturnType<typeof mediaDecoderCache.getVideo> | undefined;+ for (let c = from; c <= to; c++) {+ if (this.hasDepth(c)) continue;+ decoder ??= this.decoder();+ const image = await decodedPixels(await decoder.getFrame(c / this.fps));+ if (!image) {+ this.depths.set(c, null);+ continue;+ }+ if (!this.sourceWidth) this.setSize(image.width, image.height);+ const map = await this.visionRunner().depth(image);+ const stored: StoredDepth = {+ width: map.width,+ height: map.height,+ data: Uint16Array.from(map.data, (v) => Math.round(v * 65535)),+ };+ this.keepDepth(c, stored);+ if (dir) await this.saveDepth(dir, c, stored);+ }+ }++ private hasDepth(c: number): boolean {+ return this.depths.has(c) || this.depthsOnDisk.has(c);+ }++ private keepDepth(c: number, depth: StoredDepth | null): void {+ this.depths.delete(c);+ this.depths.set(c, depth);+ if (this.depthsOnDisk.size === 0) return;+ for (const k of this.depths.keys()) {+ if (this.depths.size <= DEPTH_MEMORY) break;+ if (this.depthsOnDisk.has(k)) this.depths.delete(k);+ }+ }++ private storedDepth(c: number): StoredDepth | null | undefined {+ const held = this.depths.get(c);+ if (held !== undefined || !this.depthsOnDisk.has(c)) return held;+ const disk = this.depthDisk;+ if (!disk) return undefined;+ const stored = readDepthFile(disk.fs, disk.path.join(disk.dir, `${c}.bin`));+ if (stored) this.keepDepth(c, stored);+ return stored ?? undefined;+ }++ /**+ * Where depth frames persist: a folder of its own, keyed only by the clip+ * and the model, so other options never invalidate it.+ */+ private depthCacheDir(): Promise<string | null> {+ this.depthDir ??= (async () => {+ const node = await nodeModules();+ if (!node || this.options.cacheDir === false) return null;+ const { fs, path, os, crypto } = node;+ try {+ const stat = fs.statSync(this.source);+ const root =+ this.options.cacheDir ??+ process.env.FRAMEFIELDS_VISION_CACHE_DIR ??+ path.join(os.homedir(), ".cache", "framefields", "vision");+ const key = JSON.stringify([+ DEPTH_CACHE_VERSION,+ path.resolve(this.source),+ stat.size,+ stat.mtimeMs,+ this.fps,+ DEPTH_MODEL,+ ]);+ const hash = crypto+ .createHash("sha256")+ .update(key)+ .digest("hex")+ .slice(0, 24);+ const dir = path.join(+ root,+ `${path.basename(this.source)}.${hash}.depth`,+ );+ fs.mkdirSync(dir, { recursive: true });+ for (const name of fs.readdirSync(dir)) {+ const m = /^(\d+)\.bin$/.exec(name);+ if (m) this.depthsOnDisk.add(Number(m[1]));+ }+ this.depthDisk = { dir, fs, path };+ return dir;+ } catch {+ return null; // remote URL or unreadable: memory only+ }+ })();+ return this.depthDir;+ }++ private async saveDepth(+ dir: string,+ c: number,+ depth: StoredDepth,+ ): Promise<void> {+ const node = await nodeModules();+ if (!node) return;+ const header = new Uint32Array([depth.width, depth.height]);+ const file = node.path.join(dir, `${c}.bin`);+ const tmp = `${file}.${process.pid}.tmp`;+ node.fs.writeFileSync(+ tmp,+ Buffer.concat([+ Buffer.from(header.buffer),+ Buffer.from(+ depth.data.buffer,+ depth.data.byteOffset,+ depth.data.byteLength,+ ),+ ]),+ );+ node.fs.renameSync(tmp, file);+ this.depthsOnDisk.add(c);+ this.keepDepth(c, depth);+ }+private async computeSam2(to: number): Promise<void> {const settings = this.options.sam2;if (!settings) return;@@ -966,7 +1161,7 @@ export class ClipVision implements ClipSubjectSource {stat.mtimeMs,this.fps,this.tracksPose,- o.mask ?? true,+ this.tracksMask,this.cell,o.variant ?? "s",o.confidence ?? null,@@ -1568,3 +1763,45 @@ function unionMask(union[i] = Math.max(union[i] ?? 0, mask[i] ?? 0);return union;}++interface NodeModules {+ fs: typeof import("node:fs");+ path: typeof import("node:path");+ os: typeof import("node:os");+ crypto: typeof import("node:crypto");+}++let nodeModulesLoaded: Promise<NodeModules | null> | undefined;++/** Node's file modules, or null in a browser. */+function nodeModules(): Promise<NodeModules | null> {+ nodeModulesLoaded ??=+ typeof process === "undefined" || !process.versions?.node+ ? Promise.resolve(null)+ : Promise.all([+ import("node:fs"),+ import("node:path"),+ import("node:os"),+ import("node:crypto"),+ ]).then(([fs, path, os, crypto]) => ({ fs, path, os, crypto }));+ return nodeModulesLoaded;+}++/** A depth frame written by `saveDepth`; null when the file is missing or damaged. */+function readDepthFile(+ fs: NodeModules["fs"],+ file: string,+): StoredDepth | null {+ try {+ const buf = fs.readFileSync(file);+ if (buf.byteLength < 8) return null;+ const width = buf.readUInt32LE(0);+ const height = buf.readUInt32LE(4);+ if (buf.byteLength !== 8 + width * height * 2) return null;+ const data = new Uint16Array(width * height);+ Buffer.from(data.buffer).set(buf.subarray(8));+ return { width, height, data };+ } catch {+ return null;+ }+}diff --git a/packages/framefields/src/effects/classes.ts b/packages/framefields/src/effects/classes.tsindex ce977278..79f22060 100644--- a/packages/framefields/src/effects/classes.ts+++ b/packages/framefields/src/effects/classes.ts@@ -775,7 +775,8 @@ export interface VisionProps {| "crop"| "skeleton"| "boxes"- | "tracking";+ | "tracking"+ | "depth";/*** Subject alpha for mask/matte/crop: instance masks (default; any COCO class) or the* selfie matte (fastest, but only reliable when a person fills much of the frame).diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.tsindex 1d70687a..647f38ee 100644--- a/packages/framefields/src/index.ts+++ b/packages/framefields/src/index.ts@@ -1243,6 +1243,7 @@ export class Composition {private _prepareHooks: Array<(ctx: FrameContext) => Promise<void> | void> =[];private readonly _clipVision = new Map<string, ClipVision>();+ private _linking = false;private _mediaSignals?: MediaSignalTimeline;private _visionConfig?: VisionProps;private _visionBundle?: VisionBundle;@@ -1342,14 +1343,13 @@ export class Composition {}/**- * Points matte-family and skeleton `withVision` layers at the `comp.vision`- * clip they show, so the cutout reuses that clip's per-frame subject and the- * skeleton draws its matched pose, instead of analysing the frame again- * during render. Only layers whose settings would give the same subject are+ * Points matte-family, skeleton and depth `withVision` layers at the+ * `comp.vision` clip they show, so the cutout reuses that clip's per-frame+ * subject, the skeleton draws its matched pose and the depth layer its+ * frame's depth, instead of analysing the frame again during render. Only layers whose settings would give the same subject are* linked; the rest keep their own pass.*/private linkClipLayers(): void {- if (this._clipVision.size === 0) return;let layers:| Map<string | undefined,@@ -1368,6 +1368,8 @@ export class Composition {trimStartSec?: number;vision?: unknown;children?: unknown;+ matte?: { node?: unknown };+ relief?: { node?: unknown };};if (n.kind === "media" &&@@ -1393,7 +1395,8 @@ export class Composition {timeRemap: layer?.timeRemap,});if (clip) {- if (n.vision.mode !== "skeleton") clip.keepSubjects();+ if (n.vision.mode !== "skeleton" && n.vision.mode !== "depth")+ clip.keepSubjects();Object.defineProperty(n.vision, CLIP_SUBJECT, {value: clip,enumerable: false,@@ -1402,6 +1405,9 @@ export class Composition {}}if (Array.isArray(n.children)) for (const c of n.children) walk(c);+ // Companion layers (a matte, a relief) can show a clip too.+ walk(n.matte?.node);+ walk(n.relief?.node);};for (const child of this.children) walk(child);}@@ -1417,6 +1423,7 @@ export class Composition {},): ClipVision | undefined {const mode = v.mode ?? "passthrough";+ if (mode === "depth") return this.depthClip(source, layer);const skeleton = mode === "skeleton";if (mode !== "matte" && mode !== "mask" && mode !== "crop" && !skeleton)return;@@ -1458,6 +1465,39 @@ export class Composition {return undefined;}+ /**+ * The clip a depth layer reads: one of the same source and timing that+ * tracks depth, or a new depth clip, since depth is never analysed at render+ * time (it would trail the picture by a frame).+ */+ private depthClip(+ source: string,+ layer: {+ startFrame?: number;+ trimStartSec?: number;+ timeRemap?: ResolvedMediaTimeRemap;+ },+ ): ClipVision {+ for (const clip of this._clipVision.values()) {+ if (+ clip.tracksDepth &&+ clip.source === source &&+ clip.startFrame === (layer.startFrame ?? 0) &&+ clip.trimStartSec === (layer.trimStartSec ?? 0) &&+ remapKey(clip.options.timeRemap) === remapKey(layer.timeRemap)+ )+ return clip;+ }+ return this.vision(source, {+ depth: true,+ ...(layer.startFrame !== undefined && { startFrame: layer.startFrame }),+ ...(layer.trimStartSec !== undefined && {+ trimStartSec: layer.trimStartSec,+ }),+ ...(layer.timeRemap !== undefined && { timeRemap: layer.timeRemap }),+ });+ }+/*** One-shot, ffmpeg-free vision analysis of a media source.*@@ -1577,6 +1617,8 @@ export class Composition {}const signals = this._mediaSignals;signals?.advance(ctx.frame);+ // Depth layers bring their own clip, so links come before the check.+ this.linkClipLayers();if (this._clipVision.size === 0 && this._prepareHooks.length === 0) {signals?.reach(ctx.frame);return undefined;@@ -2182,6 +2224,16 @@ export class Composition {public toVirtualMedia(options: Partial<CompositorToProgramOptions> = {},): VirtualMediaData {+ // Every compile links vision layers to their clips (a depth layer makes+ // its own), so the very first frame drawn reads them.+ if (!this._linking) {+ this._linking = true;+ try {+ this.linkClipLayers();+ } finally {+ this._linking = false;+ }+ }const durationMs = options.durationMs ?? this.durationMs;const signals = options.signals ?? this.signals;return compositorToProgram(this.toSpec(), {@@ -2360,6 +2412,7 @@ export type AnimatableNode<vision?: Vision;relighting?: NormalRelightingOptions;matte?: { node: LayoutNode; mode?: MatteMode };+ relief?: { node: LayoutNode; amount?: number };deflicker?: DeflickerOptions;effects?: CompatibleEffect<TCap>[];blurRegions?: unknown[];@@ -2424,6 +2477,21 @@ export type AnimatableNode<withMatte(matte: LayoutNode, mode?: MatteMode): AnimatableNode<T, TCap>;/** Alias of {@link withMatte}. */withMask(mask: LayoutNode, mode?: MatteMode): AnimatableNode<T, TCap>;+ /**+ * Relief (3D layers): `relief` (any layer, typically a video's+ * `withVision({ mode: "depth" })`) lifts this layer's plane into a surface,+ * white `amount` px toward the camera. The surface is pushed along the+ * default camera's rays, so from that lens it looks exactly like the flat+ * layer; a moving camera then sees it in 3D, lit by the scene's lights and+ * hiding what passes behind it. Where the relief jumps (an object's+ * outline) the surface stretches across with the edge pixels, so keep+ * camera moves modest; footage never saw those sides. Laid out and timed+ * like a matte; drawn flat over this node's box.+ */+ withRelief(+ relief: LayoutNode,+ options?: { amount?: number },+ ): AnimatableNode<T, TCap>;withPbrGlass(options?: Record<string, unknown>): AnimatableNode<T, TCap>;deformWithAudio(config?: MeshAudioDeformConfig | Record<string, unknown>,@@ -2570,6 +2638,16 @@ function withAnimation<});}+ Object.defineProperty(target, "withRelief", {+ value: (relief: LayoutNode, options: { amount?: number } = {}) => {+ target.relief = { node: relief, ...options };+ return target;+ },+ enumerable: false,+ writable: true,+ configurable: true,+ });+Object.defineProperty(target, "withPbrGlass", {value: (options?: Record<string, unknown>) => {target.glass = options ?? {};diff --git a/packages/framefields/src/relief-render.test.ts b/packages/framefields/src/relief-render.test.tsnew file mode 100644index 00000000..f1770f03--- /dev/null+++ b/packages/framefields/src/relief-render.test.ts@@ -0,0 +1,152 @@+import { Canvas, loadImage } from "skia-canvas";+import { describe, expect, it } from "vitest";+import { Composition, hoistMattes, Layer, Layer3D } from "./index.js";++const W = 400;+const H = 200;+/** The default camera's distance at its 50° field of view: z = 0 maps 1:1 to pixels. */+const D = H / 2 / Math.tan((50 * Math.PI) / 360);++async function decode(png: Buffer) {+ const img = await loadImage(png);+ const canvas = new Canvas(img.width, img.height);+ const ctx = canvas.getContext("2d");+ ctx.drawImage(img, 0, 0);+ const { data } = ctx.getImageData(0, 0, img.width, img.height);+ return (x: number, y: number) => {+ const i = (Math.round(y) * img.width + Math.round(x)) * 4;+ return { r: data[i]!, g: data[i + 1]!, b: data[i + 2]! };+ };+}++const box = { position: "absolute", x: 0, y: 0, width: W, height: H } as const;++const square = (background: string) =>+ Layer.box({+ position: "absolute",+ x: 140,+ y: 40,+ width: 120,+ height: 120,+ background,+ });++/** A red plate with a green square, lifted 80 px there, and a blue card 40 px in front of it. */+function scene(relief: boolean, eyeX = W / 2) {+ const c = new Composition({+ width: W,+ height: H,+ fps: 30,+ durationMs: 100,+ backgroundColor: "#000000",+ });+ const plate = Layer.box({+ ...box,+ id: "plate",+ background: "#ff0000",+ is3D: true,+ material: "unlit",+ children: [square("#00ff00")],+ });+ c.add([+ Layer.camera({+ id: "lens",+ mode: "lookAt",+ x: eyeX,+ y: H / 2,+ z: -D,+ targetX: W / 2,+ targetY: H / 2,+ targetZ: 0,+ }),+ relief+ ? plate.withRelief(+ Layer.box({+ ...box,+ id: "height",+ background: "#000000",+ children: [square("#ffffff")],+ }),+ { amount: 80 },+ )+ : plate,+ Layer3D.plane({+ id: "card",+ x: W / 2 - 15,+ y: H / 2 - 15,+ z: -40,+ width: 30,+ height: 30,+ background: "#0000ff",+ material: "unlit",+ }),+ ]);+ return c;+}++describe("relief", () => {+ it("hoists a relief into a flat sibling sized like its owner", () => {+ const [owner, relief] = hoistMattes([+ {+ id: "plate",+ kind: "box",+ x: 0,+ y: 0,+ width: W,+ height: H,+ is3D: true,+ relief: { node: { id: "d", kind: "box", is3D: true }, amount: 50 },+ } as never,+ ]) as Array<Record<string, unknown>>;+ expect(owner).toMatchObject({ reliefId: "d", reliefAmount: 50 });+ expect(owner.relief).toBeUndefined();+ expect(relief).toMatchObject({+ id: "d",+ reliefOf: "plate",+ is3D: false,+ x: 0,+ y: 0,+ width: W,+ height: H,+ });+ });++ describe.skipIf(process.env.FRAMEFIELDS_SKIP_GPU === "1")("render", () => {+ it("looks like the flat layer from the default camera", async () => {+ const flat = await decode(await scene(false).renderFrame({ frame: 0 }));+ const lifted = await decode(await scene(true).renderFrame({ frame: 0 }));+ for (const [x, y] of [+ [20, 20],+ [150, 50],+ [250, 150],+ [380, 180],+ ] as const) {+ expect(lifted(x, y)).toEqual(flat(x, y));+ }+ });++ it("rises in front of what sits between it and the camera", async () => {+ const flat = await decode(await scene(false).renderFrame({ frame: 0 }));+ const lifted = await decode(await scene(true).renderFrame({ frame: 0 }));+ // Flat, the card 40 px in front shows; lifted 80 px, the plate's square covers it.+ expect(flat(W / 2, H / 2)).toMatchObject({ g: 0, b: 255 });+ expect(lifted(W / 2, H / 2)).toMatchObject({ g: 255, b: 0 });+ });++ it("shows its height to a camera off the original lens", async () => {+ const side = await decode(+ await scene(true, W / 2 + 300).renderFrame({ frame: 0 }),+ );+ const sideFlat = await decode(+ await scene(false, W / 2 + 300).renderFrame({ frame: 0 }),+ );+ // Seen from the side, the raised square stands off the plate, so its+ // edge lands elsewhere than the flat square's.+ const leftEdge = (at: (x: number, y: number) => { g: number }) => {+ for (let x = 0; x < W; x++) if (at(x, 60).g > 128) return x;+ return W;+ };+ expect(Math.abs(leftEdge(sideFlat) - leftEdge(side))).toBeGreaterThan(8);+ });+ });+});diff --git a/packages/vision/src/decode/depth.test.ts b/packages/vision/src/decode/depth.test.tsnew file mode 100644index 00000000..75f0cac5--- /dev/null+++ b/packages/vision/src/decode/depth.test.ts@@ -0,0 +1,33 @@+import { describe, expect, it } from "vitest";+import { decodeDepth, depthInputSize } from "./depth.js";++describe("depthInputSize", () => {+ it("keeps the frame's aspect in multiples of 14 with the short side fixed", () => {+ expect(depthInputSize(1920, 1080, 518)).toEqual([924, 518]);+ expect(depthInputSize(1080, 1920, 518)).toEqual([518, 924]);+ expect(depthInputSize(512, 512, 518)).toEqual([518, 518]);+ });+});++describe("decodeDepth", () => {+ it("maps the frame's range to 0 (far) … 1 (near), ignoring a stray extreme", () => {+ const w = 100;+ const h = 100;+ const raw = Float32Array.from(+ { length: w * h },+ (_, i) => 2 + (i % w) / (w - 1),+ );+ raw[0] = 1000; // one hot pixel+ const { data, width, height } = decodeDepth(raw, w, h);+ expect([width, height]).toEqual([w, h]);+ expect(data[0]).toBe(1);+ expect(data[1]).toBeCloseTo(0, 1);+ expect(data[w - 1]).toBeCloseTo(1, 1);+ expect(data[w + 50]).toBeCloseTo(0.5, 1);+ });++ it("gives a flat frame zero depth instead of dividing by zero", () => {+ const { data } = decodeDepth(new Float32Array(16).fill(3), 4, 4);+ expect([...data]).toEqual(new Array(16).fill(0));+ });+});diff --git a/packages/vision/src/decode/depth.ts b/packages/vision/src/decode/depth.tsnew file mode 100644index 00000000..c8157fbc--- /dev/null+++ b/packages/vision/src/decode/depth.ts@@ -0,0 +1,41 @@+import type { DepthMap } from "../types.js";++/** Model input size for a frame: the short side at `short`, the other kept to the frame's aspect, both multiples of the 14 px ViT patch. */+export function depthInputSize(+ width: number,+ height: number,+ short: number,+): [number, number] {+ const snap = (v: number) => Math.max(14, Math.round(v / 14) * 14);+ return width >= height+ ? [snap((short * width) / height), snap(short)]+ : [snap(short), snap((short * height) / width)];+}++/** Share of samples clipped at each end, so a stray hot pixel does not set the range. */+const TAIL = 0.01;++/**+ * Depth Anything decode: `predicted_depth [1, h, w]` relative inverse depth →+ * 0 (farthest) … 1 (nearest), at the model's resolution. The range is the+ * frame's 1st…99th percentile, so it is relative to the frame, not metric.+ */+export function decodeDepth(+ predicted: ArrayLike<number>,+ width: number,+ height: number,+): DepthMap {+ const n = width * height;+ const step = Math.max(1, Math.floor(n / 4096));+ const samples: number[] = [];+ for (let i = 0; i < n; i += step) samples.push(predicted[i] as number);+ samples.sort((a, b) => a - b);+ const lo = samples[Math.floor((samples.length - 1) * TAIL)] as number;+ const hi = samples[Math.ceil((samples.length - 1) * (1 - TAIL))] as number;+ const scale = hi > lo ? 1 / (hi - lo) : 0;+ const data = new Float32Array(n);+ for (let i = 0; i < n; i++) {+ data[i] = Math.min(1, Math.max(0, ((predicted[i] as number) - lo) * scale));+ }+ return { data, width, height };+}diff --git a/packages/vision/src/decode/preprocess.ts b/packages/vision/src/decode/preprocess.tsindex b8636037..0e8a9ea0 100644--- a/packages/vision/src/decode/preprocess.ts+++ b/packages/vision/src/decode/preprocess.ts@@ -8,6 +8,7 @@* | RTMDet-Ins | keep ratio, content top-left | 114 | BGR | (v − mean) / std (ImageNet) |* | RTMO | keep ratio, content centered | 114 | BGR | raw 0..255 |* | Selfie | stretch to input | — | RGB | v / 255 |+ * | Depth Any. | stretch, aspect kept by size | — | RGB | (v − mean) / std (ImageNet) |** Getting any of these wrong does not crash — it silently degrades accuracy — so the specs* live here and every runner path goes through `imageToTensor`. Input sizes differ per model@@ -52,7 +53,7 @@ type Normalization = Omit<PreprocessSpec, "width" | "height">;/** Per-family normalization; the input size always comes from the model's registry entry. */export const PREPROCESS_BY_FAMILY: Readonly<- Record<"rtmdet-ins" | "rtmo" | "selfie", Normalization>+ Record<"rtmdet-ins" | "rtmo" | "selfie" | "depth-anything", Normalization>> = {"rtmdet-ins": {fit: "topLeft",@@ -75,6 +76,13 @@ export const PREPROCESS_BY_FAMILY: Readonly<std: [255, 255, 255],padValue: 0,},+ "depth-anything": {+ fit: "stretch",+ channels: "rgb",+ mean: [123.675, 116.28, 103.53],+ std: [58.395, 57.12, 57.375],+ padValue: 0,+ },};export type PreprocessFamily = keyof typeof PREPROCESS_BY_FAMILY;diff --git a/packages/vision/src/index.ts b/packages/vision/src/index.tsindex 7ba5d173..62a670b1 100644--- a/packages/vision/src/index.ts+++ b/packages/vision/src/index.ts@@ -1,4 +1,5 @@export * from "./analysis/analyze-sequence.js";+export * from "./decode/depth.js";export * from "./decode/preprocess.js";export * from "./decode/rtmdet-ins.js";export * from "./decode/rtmo.js";diff --git a/packages/vision/src/model/registry.ts b/packages/vision/src/model/registry.tsindex a24ec3b1..6cb083ad 100644--- a/packages/vision/src/model/registry.ts+++ b/packages/vision/src/model/registry.ts@@ -33,6 +33,7 @@ export type VisionModelKey =| "rtmo-m"| "selfie-square"| "selfie-landscape"+ | "depth-anything-v2-s"| (string & {});/** Which decoder a model's outputs feed (see runner/vision-runner.ts). */@@ -40,6 +41,7 @@ export type VisionModelFamily =| "rtmdet-ins"| "rtmo"| "selfie"+ | "depth-anything"| (string & {});export interface VisionModelDescriptor {@@ -71,6 +73,8 @@ const RTMDET_INS_SOURCE =const RTMO_SOURCE = "RTMO (OpenMMLab mmpose, Apache-2.0), ONNX export";const SELFIE_SOURCE ="MediaPipe Selfie Segmenter (Google, Apache-2.0), ONNX export";+const DEPTH_SOURCE =+ "Depth Anything V2 Small (TikTok / HKU, Apache-2.0), ONNX export";const models: Record<VisionModelKey, VisionModelDescriptor> = {// ── Detection + instance segmentation (COCO 80) ─────────────────────@@ -188,6 +192,23 @@ const models: Record<VisionModelKey, VisionModelDescriptor> = {input: [256, 144],source: SELFIE_SOURCE,},+ // ── Monocular depth (relative, near = 1) ────────────────────────────+ // The graph takes any size in multiples of its 14 px patch; `input` is the+ // short side the runner scales to, keeping the frame's aspect ratio.+ "depth-anything-v2-s": {+ key: "depth-anything-v2-s",+ family: "depth-anything",+ filename: "depth-anything-v2-s.onnx",+ url: hf(+ "onnx-community/depth-anything-v2-small",+ "4472b7362082ad9968fee890ca0f1e5aca36b93d",+ "onnx/model.onnx",+ ),+ bytes: 99_060_839,+ sha256: "afb6a5c28f3b6bf1618c6e43f02073ef9dfdc70e937502d51603e57b0a1df10c",+ input: [518, 518],+ source: DEPTH_SOURCE,+ },};export const VISION_MODELS: Readonly<diff --git a/packages/vision/src/runner/vision-runner.ts b/packages/vision/src/runner/vision-runner.tsindex 4124c3fd..078d41e2 100644--- a/packages/vision/src/runner/vision-runner.ts+++ b/packages/vision/src/runner/vision-runner.ts@@ -1,4 +1,5 @@import type { DetectedObject } from "@framefields/core";+import { decodeDepth, depthInputSize } from "../decode/depth.js";import {type InputTransform,imageToTensor,@@ -39,6 +40,7 @@ import {type VisionTensorOutputs,} from "../runtime/session-provider.js";import {+ type DepthMap,isGpuImage,type PersonMatte,type PoseResult,@@ -276,6 +278,24 @@ export class VisionRunner {});}+ /** Relative depth of the whole frame (Depth Anything V2 Small), near = 1, at the model's resolution. */+ public async depth(image: VisionFrame): Promise<DepthMap> {+ const key: VisionModelKey = "depth-anything-v2-s";+ const input = depthInputSize(+ image.width,+ image.height,+ VISION_MODELS[key].input[1],+ );+ const { outputs } = await this.infer(key, image, input);+ const predicted = output(outputs, "predicted_depth", key);+ const dims = predicted.dims;+ return decodeDepth(+ predicted.data as Float32Array,+ dims[dims.length - 1] ?? input[0],+ dims[dims.length - 2] ?? input[1],+ );+ }+public close(): void {if (!this._ownsCore) return;const core = this._core;@@ -341,12 +361,14 @@ export class VisionRunner {private infer(key: VisionModelKey,image: VisionFrame,+ /** Input size for graphs that take any (default: the registry's). */+ input?: readonly [number, number],): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {const queue = this._core.runQueue;const previous = queue.get(key) ?? Promise.resolve();const run = previous.then(- () => this.inferNow(key, image),- () => this.inferNow(key, image),+ () => this.inferNow(key, image, input),+ () => this.inferNow(key, image, input),);const tail = run.catch(() => undefined);queue.set(key, tail);@@ -359,6 +381,7 @@ export class VisionRunner {private async inferNow(key: VisionModelKey,frame: VisionFrame,+ input?: readonly [number, number],): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {const session = await this.session(key);const desc = VISION_MODELS[key];@@ -366,7 +389,7 @@ export class VisionRunner {throw new Error(`VisionRunner cannot run '${desc.family}' models; use the family's own tracker (e.g. Sam2Tracker or Sam3Tracker from @framefields/vision-sam).`,);- const spec = preprocessSpec(desc.family, desc.input);+ const spec = preprocessSpec(desc.family, input ?? desc.input);const dims = [1, 3, spec.height, spec.width];let image: VisionImageInput;if (isGpuImage(frame)) {diff --git a/packages/vision/src/segmentation/clip-subject.ts b/packages/vision/src/segmentation/clip-subject.tsindex a6f63e8f..06630a1e 100644--- a/packages/vision/src/segmentation/clip-subject.ts+++ b/packages/vision/src/segmentation/clip-subject.ts@@ -1,10 +1,11 @@-import type { InstanceMask, PoseKeypoint } from "../types.js";+import type { DepthMap, InstanceMask, PoseKeypoint } from "../types.js";/*** Where a render-time Vision effect finds subject masks someone already* computed for its clip (`comp.vision(clip)` in framefields), so a matte layer- * reuses them instead of segmenting the frame again, and a skeleton layer draws- * the pose matched to that subject instead of the first person it detects. Set+ * reuses them instead of segmenting the frame again, a skeleton layer draws+ * the pose matched to that subject instead of the first person it detects, and+ * a depth layer shows the depth analysed for each frame (not one behind). Set* on the Vision effect instance under this key; the renderer reads it from the effect.*/export const CLIP_SUBJECT: unique symbol = Symbol.for(@@ -26,6 +27,11 @@ export interface ClipSubjectSource {* clip tracks no pose).*/pose?(frame: number, fps: number): ClipPose | null | undefined;+ /**+ * The frame's relative depth (near = 1); `null` when the frame could not be+ * decoded; `undefined` when it was not analysed (or the clip tracks no depth).+ */+ depth?(frame: number, fps: number): DepthMap | null | undefined;}export interface ClipPose {diff --git a/packages/vision/src/types.ts b/packages/vision/src/types.tsindex fa6e4376..6afa7b13 100644--- a/packages/vision/src/types.ts+++ b/packages/vision/src/types.ts@@ -51,6 +51,16 @@ export interface SegmentationResult {readonly masks: readonly InstanceMask[];}+/**+ * Relative depth of a whole frame (Depth Anything), 0 = farthest … 1 = nearest,+ * row-major at `width × height` (the model's resolution, the frame's aspect).+ */+export interface DepthMap {+ readonly data: Float32Array;+ readonly width: number;+ readonly height: number;+}+/** Person-vs-background alpha (Selfie Segmenter), aligned to the ORIGINAL source frame. */export interface PersonMatte {readonly mask: Uint8Array; // frame-sized, continuous alpha 0..255diff --git a/packages/webgpu-renderers/src/renderer3d/quad3d-pipeline.ts b/packages/webgpu-renderers/src/renderer3d/quad3d-pipeline.tsindex 35a6a599..cea99b80 100644--- a/packages/webgpu-renderers/src/renderer3d/quad3d-pipeline.ts+++ b/packages/webgpu-renderers/src/renderer3d/quad3d-pipeline.ts@@ -49,6 +49,28 @@ export function createLightsLayout(});}+/**+ * A layer's relief: the brightness of `texture` over `uvRect` pushes the+ * layer's plane toward `eye` by up to `amount` px, along the rays to `eye`, so+ * from there the surface still looks like the flat layer. Where the+ * brightness jumps (an object's outline) the surface stretches across.+ */+export interface Relief3D {+ texture: GPUTexture;+ /** Where the layer's box sits in `texture`: [u, v, width, height], 0..1. */+ uvRect: [number, number, number, number];+ /** Height of white, in world px. */+ amount: number;+ /** The viewpoint the surface keeps its look from: the default camera's eye. */+ eye: [number, number, number];+}++/** Grid cells per side of a relief surface. */+const RELIEF_GRID = 512;++/** Floats in the quad's model uniform: transform, colors, params, material, surface, then relief. */+const MODEL_FLOATS = 52 + SURFACE_PARAMS_FLOATS + 12;+export interface ModelUniformData extends SurfaceParams {modelMatrix: Float32Array | number[];normalMatrix?: Float32Array | number[];@@ -65,6 +87,8 @@ export interface ModelUniformData extends SurfaceParams {material?: "lit" | "unlit";/** Screen-space track matte: the layer shows where this texture's alpha does. */matte?: GPUTexture;+ /** Turns the flat quad into a surface lifted by another picture's brightness. */+ relief?: Relief3D;shininess?: number;roughness?: number;specularIntensity?: number;@@ -115,8 +139,14 @@ export class Quad3DPipeline {private modelPool: BufferPool;private vertexBuffer: GPUBuffer;private indexBuffer: GPUBuffer;- // 16 (model) + 16 (normal) + 4 (tint) + 4 (borderColor) + 4 (params) + 4 (dimensions) + 4 (materialParams) + 8 (pbr, emissive) = 60 floats (240 bytes)- private modelData = new Float32Array(52 + SURFACE_PARAMS_FLOATS);+ /** The relief grid, built on first use. */+ private reliefGrid?: {+ vertices: GPUBuffer;+ indices: GPUBuffer;+ count: number;+ };+ // 16 (model) + 16 (normal) + 4 (tint) + 4 (borderColor) + 4 (params) + 4 (dimensions) + 4 (materialParams) + 8 (pbr, emissive) + 12 (relief) = 72 floats (288 bytes)+ private modelData = new Float32Array(MODEL_FLOATS);constructor(device: GPUDevice,@@ -127,7 +157,7 @@ export class Quad3DPipeline {) {this.device = device;this.modelPool = new BufferPool(- (52 + SURFACE_PARAMS_FLOATS) * 4,+ MODEL_FLOATS * 4,GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,);@@ -156,6 +186,11 @@ export class Quad3DPipeline {buffer: { type: "uniform" },},matteLayoutEntry(1),+ {+ binding: 2,+ visibility: GPUShaderStage.VERTEX,+ texture: { sampleType: "float" },+ },],});@@ -289,6 +324,14 @@ export class Quad3DPipeline {this.modelData.set(packMaterialParams(data), 48);packSurfaceParams(data, this.modelData, 52);+ const relief = data.relief;+ const r = 52 + SURFACE_PARAMS_FLOATS;+ this.modelData.set(+ relief ? [relief.amount, 1, 0, RELIEF_GRID] : [0, 0, 0, 0],+ r,+ );+ this.modelData.set(relief?.uvRect ?? [0, 0, 1, 1], r + 4);+ this.modelData.set([...(relief?.eye ?? [0, 0, 0]), 0], r + 8);const modelBuffer = this.modelPool.getBuffer(this.device, this.modelData);return this.device.createBindGroup({@@ -299,10 +342,75 @@ export class Quad3DPipeline {binding: 1,resource: data.matte?.createView() ?? fullCoverageView(this.device),},+ {+ binding: 2,+ resource:+ data.relief?.texture.createView() ?? fullCoverageView(this.device),+ },],});}+ /** Binds the geometry for `data` and draws it: the quad, or the relief grid. */+ private drawGeometry(+ pass: GPURenderPassEncoder,+ data: ModelUniformData,+ ): void {+ if (!data.relief) {+ pass.setVertexBuffer(0, this.vertexBuffer);+ pass.setIndexBuffer(this.indexBuffer, "uint16");+ pass.drawIndexed(6);+ return;+ }+ this.reliefGrid ??= this.createReliefGrid();+ pass.setVertexBuffer(0, this.reliefGrid.vertices);+ pass.setIndexBuffer(this.reliefGrid.indices, "uint32");+ pass.drawIndexed(this.reliefGrid.count);+ }++ /** A RELIEF_GRID² grid over the unit quad, laid out like the quad's vertices. */+ private createReliefGrid(): {+ vertices: GPUBuffer;+ indices: GPUBuffer;+ count: number;+ } {+ const side = RELIEF_GRID + 1;+ const vertices = new Float32Array(side * side * 8);+ for (let j = 0; j < side; j++) {+ for (let i = 0; i < side; i++) {+ const u = i / RELIEF_GRID;+ const v = j / RELIEF_GRID;+ vertices.set([u - 0.5, v - 0.5, 0, u, v, 0, 0, -1], (j * side + i) * 8);+ }+ }+ const indices = new Uint32Array(RELIEF_GRID * RELIEF_GRID * 6);+ let k = 0;+ for (let j = 0; j < RELIEF_GRID; j++) {+ for (let i = 0; i < RELIEF_GRID; i++) {+ const a = j * side + i;+ indices.set([a, a + 1, a + side + 1, a, a + side + 1, a + side], k);+ k += 6;+ }+ }+ const upload = (data: Float32Array | Uint32Array, usage: number) => {+ const buffer = this.device.createBuffer({+ size: data.byteLength,+ usage: usage | GPUBufferUsage.COPY_DST,+ mappedAtCreation: true,+ });+ new (+ data.constructor as Float32ArrayConstructor | Uint32ArrayConstructor+ )(buffer.getMappedRange()).set(data);+ buffer.unmap();+ return buffer;+ };+ return {+ vertices: upload(vertices, GPUBufferUsage.VERTEX),+ indices: upload(indices, GPUBufferUsage.INDEX),+ count: indices.length,+ };+ }+private textureBindGroup(texture: GPUTexture,sampler: GPUSampler,@@ -340,9 +448,7 @@ export class Quad3DPipeline {pass.setBindGroup(1, modelBindGroup);pass.setBindGroup(2, textureBindGroup);pass.setBindGroup(3, lightsBindGroup);- pass.setVertexBuffer(0, this.vertexBuffer);- pass.setIndexBuffer(this.indexBuffer, "uint16");- pass.drawIndexed(6);+ this.drawGeometry(pass, data);}/** Draws the quad into a shadow map (`lightCameraBindGroup` holds the light's view-projection). */@@ -357,14 +463,14 @@ export class Quad3DPipeline {pass.setBindGroup(0, lightCameraBindGroup);pass.setBindGroup(1, this.packModel(data));pass.setBindGroup(2, this.textureBindGroup(texture, sampler));- pass.setVertexBuffer(0, this.vertexBuffer);- pass.setIndexBuffer(this.indexBuffer, "uint16");- pass.drawIndexed(6);+ this.drawGeometry(pass, data);}destroy(): void {this.modelPool.destroy();this.vertexBuffer.destroy();this.indexBuffer.destroy();+ this.reliefGrid?.vertices.destroy();+ this.reliefGrid?.indices.destroy();}}diff --git a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts b/packages/webgpu-renderers/src/renderer3d/renderer3d.tsindex 01d45ca6..120a2548 100644--- a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts+++ b/packages/webgpu-renderers/src/renderer3d/renderer3d.ts@@ -54,7 +54,7 @@ import {type MeshMaterialMaps,} from "./mesh3d-pipeline.js";import { MorphPipeline } from "./morph-pipeline.js";-import { Quad3DPipeline } from "./quad3d-pipeline.js";+import { Quad3DPipeline, type Relief3D } from "./quad3d-pipeline.js";import { SHADOW_DEPTH_FORMAT } from "./shadow-depth.js";import {type Aabb,@@ -119,6 +119,8 @@ export interface DrawQuad3DOpts extends SurfaceParams {* this texture's alpha does and keeps its place in the depth buffer.*/matte?: GPUTexture;+ /** Lifts the layer's plane into a surface by another picture's brightness. */+ relief?: Relief3D;}/**@@ -913,6 +915,7 @@ export class Renderer3D {textureExtent: c.opts.textureExtent,opacity: c.opts.opacity,colorTint: c.opts.colorTint,+ relief: c.opts.relief,},);}@@ -1385,6 +1388,7 @@ export class Renderer3D {emissive: opts.emissive,emissiveIntensity: opts.emissiveIntensity,matte: opts.matte,+ relief: opts.relief,},useMrt,this.passWritesHdr,diff --git a/packages/webgpu-renderers/src/shaders/quad3d.ts b/packages/webgpu-renderers/src/shaders/quad3d.tsindex 035e508c..746c8353 100644--- a/packages/webgpu-renderers/src/shaders/quad3d.ts+++ b/packages/webgpu-renderers/src/shaders/quad3d.ts@@ -9,6 +9,9 @@ import { matte3dWgsl } from "./matte3d.js";export const quad3dWgsl = `${lighting3dWgsl}+// Grid cells either side a relief's normals span.+const RELIEF_NORMAL_CELLS : f32 = 4.0;+struct ModelUniforms {modelMatrix : mat4x4<f32>,normalMatrix : mat4x4<f32>,@@ -19,11 +22,15 @@ struct ModelUniforms {materialParams : vec4<f32>, // x = shininess, y = specularIntensity, z = ambientIntensity, w = materialMode (0=unlit, 1=lit)pbr : vec4<f32>, // x = roughness (<0 = default), y = metallicemissive : vec4<f32>, // rgb = sRGB emissive color, w = strength+ relief : vec4<f32>, // x = amount (px), y = on, w = grid cells per side+ reliefRect : vec4<f32>, // xy = where the layer's box starts in reliefTex (0..1), zw = its size+ reliefEye : vec4<f32>, // xyz = the eye the surface keeps its look from (the default camera)};@group(0) @binding(0) var<uniform> cam : CameraUniforms;@group(1) @binding(0) var<uniform> model : ModelUniforms;-${matte3dWgsl(1, 1)}@group(2) @binding(0) var texSampler : sampler;+${matte3dWgsl(1, 1)}@group(1) @binding(2) var reliefTex : texture_2d<f32>;+@group(2) @binding(0) var texSampler : sampler;@group(2) @binding(1) var mainTex : texture_2d<f32>;${lightsBindingsWgsl}@@ -42,13 +49,48 @@ struct VertexOutput {@location(4) localPos : vec2<f32>,};+// Relief: the layer's plane pushed toward the eye by the brightness of a+// second picture, so a flat quad becomes a surface (see DrawQuad3DOpts.relief).+fn reliefAt(uv: vec2<f32>) -> f32 {+ let dims = vec2<f32>(textureDimensions(reliefTex));+ let p = (model.reliefRect.xy + clamp(uv, vec2<f32>(0.0), vec2<f32>(1.0)) * model.reliefRect.zw) * dims - 0.5;+ let i = floor(p);+ let f = p - i;+ // Texels of the layer's own box only: past its edge the relief was never drawn.+ let lo = max(floor(model.reliefRect.xy * dims), vec2<f32>(0.0));+ let hi = min(ceil((model.reliefRect.xy + model.reliefRect.zw) * dims) - 1.0, dims - 1.0);+ let a = textureLoad(reliefTex, vec2<i32>(clamp(i, lo, hi)), 0).rgb;+ let b = textureLoad(reliefTex, vec2<i32>(clamp(i + vec2<f32>(1.0, 0.0), lo, hi)), 0).rgb;+ let c = textureLoad(reliefTex, vec2<i32>(clamp(i + vec2<f32>(0.0, 1.0), lo, hi)), 0).rgb;+ let d = textureLoad(reliefTex, vec2<i32>(clamp(i + vec2<f32>(1.0, 1.0), lo, hi)), 0).rgb;+ let rgb = mix(mix(a, b, f.x), mix(c, d, f.x), f.y);+ return dot(rgb, vec3<f32>(0.2126, 0.7152, 0.0722));+}++// The plane's front: the side the default camera sees.+fn reliefFront() -> vec3<f32> {+ return normalize((model.normalMatrix * vec4<f32>(0.0, 0.0, -1.0, 0.0)).xyz);+}++// A point of the plane lifted by its relief along the ray to the eye, so from+// the eye it lands on the same pixel as the flat picture.+fn reliefPoint(local: vec2<f32>, uv: vec2<f32>) -> vec3<f32> {+ let flat = (model.modelMatrix * vec4<f32>(local, 0.0, 1.0)).xyz;+ let n = reliefFront();+ let h = model.relief.x * reliefAt(uv);+ let toEye = model.reliefEye.xyz - flat;+ let along = dot(toEye, n);+ return select(flat + n * h, flat + toEye * (h / along), along > 1.0);+}+@vertexfn vs_main(in: VertexInput) -> VertexOutput {var out: VertexOutput;let hasDimensions = model.dimensions.x > 0.0 && model.dimensions.y > 0.0;+ let hasRelief = model.relief.y > 0.5;var pad = 0.0;- if (hasDimensions) {+ if (hasDimensions && !hasRelief) {let viewPos = cam.viewMatrix * model.modelMatrix * vec4<f32>(0.0, 0.0, 0.0, 1.0);let viewDepth = max(abs(viewPos.z), 0.1);let projScale = max(cam.projMatrix[1][1], 0.001);@@ -58,12 +100,28 @@ fn vs_main(in: VertexInput) -> VertexOutput {let scalePad = select(vec2<f32>(1.0), 1.0 + vec2<f32>(2.0 * pad) / model.dimensions.xy, hasDimensions);let paddedPos = vec3<f32>(in.position.xy * scalePad, in.position.z);- let worldPos4 = model.modelMatrix * vec4<f32>(paddedPos, 1.0);- out.worldPos = worldPos4.xyz;+ var worldPos4 = model.modelMatrix * vec4<f32>(paddedPos, 1.0);// Normal transformed by 3x3 normal matrixlet norm4 = model.normalMatrix * vec4<f32>(in.normal, 0.0);out.worldNormal = normalize(norm4.xyz);+ if (hasRelief) {+ let p = reliefPoint(in.position.xy, in.uv);+ // Central differences a few cells wide: an 8-bit height map steps every+ // level, and normals from adjacent cells would light those steps as rings.+ let e = RELIEF_NORMAL_CELLS / max(model.relief.w, 1.0);+ let dx = vec2<f32>(e, 0.0);+ let dy = vec2<f32>(0.0, e);+ let tx = reliefPoint(in.position.xy + dx, in.uv + dx) - reliefPoint(in.position.xy - dx, in.uv - dx);+ let ty = reliefPoint(in.position.xy + dy, in.uv + dy) - reliefPoint(in.position.xy - dy, in.uv - dy);+ var nrm = normalize(cross(tx, ty));+ if (dot(nrm, reliefFront()) < 0.0) {+ nrm = -nrm;+ }+ worldPos4 = vec4<f32>(p, 1.0);+ out.worldNormal = nrm;+ }+ out.worldPos = worldPos4.xyz;out.clipPos = cam.viewProjMatrix * worldPos4;diff --git a/plugins/framefields/skills/framefields-compose/SKILL.md b/plugins/framefields/skills/framefields-compose/SKILL.mdindex 819e3187..b97b37d1 100644--- a/plugins/framefields/skills/framefields-compose/SKILL.md+++ b/plugins/framefields/skills/framefields-compose/SKILL.md@@ -377,6 +377,18 @@ Layer.video("assets/surf.mp4", { position: "absolute", x: 0, y: 0, width: W, hei.withMatte(Layer.text("OCEAN", { position: "absolute", x: 0, y: 340, width: W, fontSize: 380, align: "center", fill: "#fff" }));```+### Relief: footage as a 3D surface++`node.withRelief(heightLayer, { amount })` turns a 3D layer into a surface: the brightness of `heightLayer` (laid out and timed like a matte, drawn flat over the node's box) pushes it up to `amount` px toward the camera. It moves along the **default camera's** rays, so from that lens the surface is pixel for pixel the flat layer; move the camera and the footage has real parallax, takes the scene's lights, and hides 3D objects that pass behind its raised parts. With a video's own depth it puts the footage in the 3D scene:++```typescript+const plate = { position: "absolute", x: 0, y: 0, width: W, height: H, fit: "cover", muted: true } as const;+Layer.video("assets/table.mp4", { ...plate, is3D: true })+ .withRelief(Layer.video("assets/table.mp4", plate).withVision({ mode: "depth" }), { amount: 300 });+```++Where depth jumps (an object's outline) the surface stretches across with the edge pixels: footage never saw those sides. Keep camera moves within about 15° of the original lens, or hide the outlines.+---## Scenes & Transitionsdiff --git a/plugins/framefields/skills/framefields-vision/SKILL.md b/plugins/framefields/skills/framefields-vision/SKILL.mdindex 66f0ad52..bd0b4589 100644--- a/plugins/framefields/skills/framefields-vision/SKILL.md+++ b/plugins/framefields/skills/framefields-vision/SKILL.md@@ -29,8 +29,9 @@ Vision runs ONNX models locally. Nothing is sent anywhere. Models download on fi| Detect + segment (one shared pass), COCO-80 | RTMDet-Ins | 24 MB | 43 MB | 116 MB | 640² || Pose, 17 COCO keypoints | RTMO | 27 MB (416²) | 40 MB | 89 MB | 640² || Person matte | MediaPipe Selfie Segmenter | square 0.5 MB (256²), landscape 0.5 MB (256×144) for wide frames | | | |+| Relative depth (`depth: true`) | Depth Anything V2 Small | 99 MB, one size | | | short side 518, frame aspect |-About 380 MB for all of them. `variant: "t" | "s" | "m"` picks the size.+About 480 MB for all of them. `variant: "t" | "s" | "m"` picks the size.### SAM 2.1 Tiny and Small (`sam2`, Apache-2.0)Meta SAM 2.1 Hiera checkpoints in Diffusion Studio's fp16 video exports, with the same five graphs and memory bank (the prompted frame, the 6 latest frames, 16 object pointers). Point and box prompts only. Pick one with `sam2: { model }`:@@ -93,7 +94,7 @@ const vision = comp.withVision({ enableSegmentation: true, enablePose: true });Layer.video("assets/dancer.mp4").withVision({ mode: "matte", enableSegmentation: true });```-Node modes: `passthrough`, `mask`, `matte`, `crop`, `skeleton`, `boxes`, `tracking`. `skeleton` draws only the pose on transparency: the first person detected, or, for a layer of a `comp.vision` clip, the pose that clip matched to its selection. For `mask` / `matte` / `crop`, `matteSource: "instance"` (default; any COCO class, overlapping parts such as a dress merged into the subject) or `"selfie"` (people only, fastest). `confidence` (default 0.3) and `classes: ["person", …]` filter detections.+Node modes: `passthrough`, `mask`, `matte`, `crop`, `skeleton`, `boxes`, `tracking`, `depth`. `depth` draws the frame's relative depth as gray (white = near); it always reads a `comp.vision` clip with `depth: true`, made for it when there is none, so it is never a frame behind. `skeleton` draws only the pose on transparency: the first person detected, or, for a layer of a `comp.vision` clip, the pose that clip matched to its selection. For `mask` / `matte` / `crop`, `matteSource: "instance"` (default; any COCO class, overlapping parts such as a dress merged into the subject) or `"selfie"` (people only, fastest). `confidence` (default 0.3) and `classes: ["person", …]` filter detections.Measured per 1280–2048 px frame on CPU (`onnxruntime-node`); the GPU is faster:@@ -130,6 +131,7 @@ await dancer.ready(0); // analyse a frame (or ready(0, 120)) to read- The composition analyses each frame's clips just before drawing it, in every render path and the preview. Nothing runs up front.- Results persist in Node, keyed by the file and settings (`cacheDir: false` opts out). Later renders and renders after a preview are instant.- Options: `startFrame` / `trimStartSec` (where the clip plays; pass `trimStartSec` yourself when you give a path instead of the layer), `fps` (analysis rate), `pose`, `mask`, `maskCell` (silhouette grid, default 8 px), `smoothing` (frames, default 2), `variant`. A lost person's last pose is held for up to a second.+- `depth: true` adds each frame's relative depth (Depth Anything V2 Small; 0 far … 1 near, per-frame range, not metric) at about 924×518 for 16:9. A depth clip tracks no pose or silhouette unless `pose` / `mask` ask. Read it with `clip.depthMap(frame)` (`{ data, width, height }`), or draw it with `withVision({ mode: "depth" })` and lift a 3D layer with it via `withRelief` (framefields-compose). Depth frames persist one file each, keyed by the file only, so other options never invalidate them. About 0.4 s per frame on an M1 Pro GPU.- A `withVision({ mode: "matte" })` cutout (or `mask` / `crop`) of the same clip reuses the clip's subject for the frame being drawn: segmented once, aligned with the plate. Layers with `classes`, `keyBackground`, `matteSource: "selfie"` or different model settings keep their own pass. With several `comp.vision` clips of one source, the matte links to the **first one created**.## Prompted tracking with SAM