Code-first video & image, rendered natively on WebGPU.
feat(vision-sam): per-backend SAM 3 encoder, one shared encoder, clean seed frames
okanasl72f6aaaf3ce3
diff --git a/.changeset/sam3-dynamic-models.md b/.changeset/sam3-dynamic-models.mdnew file mode 100644index 00000000..fa43378b--- /dev/null+++ b/.changeset/sam3-dynamic-models.md@@ -0,0 +1,7 @@+---+"@framefields/vision-sam": minor+"@framefields/vision": minor+"framefields": patch+---++SAM 3 downloads the image encoder its backend runs fastest: fp16 on WebGPU and CUDA, q4 on the CPU (`SAM3_ENCODER`, override with `sam3.precision`). Providers report their backend before any download (`SessionProvider.backend()`, `sessionCompute`); the auto provider decides with a one-node probe graph and, on Linux with the NVIDIA driver, tries CUDA first. All SAM 3 trackers in a process share one encoder session and reuse its features per frame. Text-seeded objects now output and remember their seed mask on the prompt frame, as SAM does, instead of a speckled re-decode.diff --git a/packages/framefields/src/clip-vision.ts b/packages/framefields/src/clip-vision.tsindex 4dad9e80..84d2d548 100644--- a/packages/framefields/src/clip-vision.ts+++ b/packages/framefields/src/clip-vision.ts@@ -601,6 +601,7 @@ export class ClipVision implements ClipSubjectSource {frameCount,textThreshold: settings.textThreshold,maxObjects: settings.maxObjects,+ precision: settings.precision,acceptLicense:this.options.acceptSamLicense ?? this.options.sam3?.acceptLicense,});@@ -936,7 +937,17 @@ export class ClipVision implements ClipSubjectSource {o.confidence ?? null,// Prompted subjects are track-ID label maps (format 2)....(o.sam2 ? [[sam?.SAM2_TINY_REVISION, o.sam2, 2]] : []),- ...(o.sam3 ? [[sam?.SAM3_REVISION, o.sam3, 2]] : []),+ ...(o.sam3 && sam+ ? [+ [+ sam.sam3Revision(+ await sam.sam3Precision(undefined, o.sam3.precision),+ ),+ o.sam3,+ 2,+ ],+ ]+ : []),]);const hash = crypto.createHash("sha256")diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.tsindex d9309cd4..4e78df54 100644--- a/packages/framefields/src/index.ts+++ b/packages/framefields/src/index.ts@@ -143,6 +143,7 @@ export type {Sam2Point,Sam2Prompt,Sam3Options,+ Sam3Precision,Sam3Prompt,} from "@framefields/vision-sam";diff --git a/packages/vision-sam/src/models/registry.ts b/packages/vision-sam/src/models/registry.tsindex fb36ebf6..bfd42cb4 100644--- a/packages/vision-sam/src/models/registry.ts+++ b/packages/vision-sam/src/models/registry.ts@@ -9,6 +9,7 @@import {registerVisionModel,VISION_MODELS,+ type VisionCompute,type VisionModelDescriptor,} from "@framefields/vision";@@ -27,9 +28,25 @@ export type Sam2ModelKey =| "sam2.1-tiny-memory_attention"| "sam2.1-tiny-pointer_tpos";+/** Weight formats published for the SAM 3 tracker's image encoder. */+export type Sam3Precision = "fp16" | "q4f16" | "q4";++/**+ * The encoder precision each backend runs fastest. Measured on an M1 Pro (one 1008² frame):+ * WebGPU fp16 3.65 s, q4f16 4.43 s, q4 6.63 s, int8 140 s (its ops fall back to the CPU);+ * fp16 is also the closest to full precision. On WebGPU, 4-bit weights are dequantized+ * before every matmul, so they only save memory. CUDA runs fp16 on tensor cores. CPUs have+ * no fast fp16 path, so they get fp32-activation q4.+ */+export const SAM3_ENCODER: Readonly<Record<VisionCompute, Sam3Precision>> = {+ webgpu: "fp16",+ cuda: "fp16",+ cpu: "q4",+};+export type Sam3ModelKey =- | "sam3-vision_encoder"- | "sam3-vision_encoder-data"+ | `sam3-vision_encoder-${Sam3Precision}`+ | `sam3-vision_encoder-${Sam3Precision}-data`| "sam3-mask_decoder"| "sam3-memory_encoder"| "sam3-memory_attention"@@ -92,6 +109,36 @@ const tracker = sam3("onnx-community/sam3-tracker-ONNX","429305c8a5b3de597243d919a07e4e6bdcd00ef7",);+/** The tracker encoder at `precision`: its graph and the weights file the graph names. */+function encoder(+ precision: Sam3Precision,+ [graphBytes, graphSha]: readonly [number, string],+ [dataBytes, dataSha]: readonly [number, string],+): VisionModelDescriptor[] {+ const { graph, data } = sam3EncoderFiles(precision);+ return [+ tracker(`sam3-vision_encoder-${precision}`, `sam3-${graph}`, [+ `onnx/${graph}`,+ graphBytes,+ graphSha,+ ]),+ tracker(`sam3-vision_encoder-${precision}-data`, `sam3-${data}`, [+ `onnx/${data}`,+ dataBytes,+ dataSha,+ ]),+ ];+}++/** File names of the encoder at `precision`; the graph refers to its weights by `data`. */+export function sam3EncoderFiles(precision: Sam3Precision): {+ graph: string;+ data: string;+} {+ const graph = `vision_encoder_${precision}.onnx`;+ return { graph, data: `${graph}_data` };+}+const video = sam3("jax-image-tools/sam3-video-onnx","e942689a73d942f83c69b5965881cb3a0f6153ba",@@ -138,16 +185,39 @@ export const SAM_MODELS: readonly VisionModelDescriptor[] = [34228,"3ab5bfce40ed01208301b2d8f596b9c9f7f493dcd7e09ef01ec22e5be3694bd7",]),- tracker("sam3-vision_encoder", "sam3-vision_encoder_q4f16.onnx", [- "onnx/vision_encoder_q4f16.onnx",- 1307816,- "4b96732adab3d7d70ac0964ce540a61fad097fbc10c9b27612f47d676877cefc",- ]),- tracker("sam3-vision_encoder-data", "sam3-vision_encoder_q4f16.onnx_data", [- "onnx/vision_encoder_q4f16.onnx_data",- 295622656,- "d1d60808376dbd48e34cf89feaee708c13abed229f2398ab512149db34fdab07",- ]),+ ...encoder(+ "fp16",+ [+ 1263005,+ "f8c52be6de99124bb17f25792054406abc482ce3502166f948d5c5849cd14d02",+ ],+ [+ 934729728,+ "4b021a4d3068c9e9153f2f2fcc9e6d280156228671c00ab6c2ddfa737da7f151",+ ],+ ),+ ...encoder(+ "q4f16",+ [+ 1307816,+ "4b96732adab3d7d70ac0964ce540a61fad097fbc10c9b27612f47d676877cefc",+ ],+ [+ 295622656,+ "d1d60808376dbd48e34cf89feaee708c13abed229f2398ab512149db34fdab07",+ ],+ ),+ ...encoder(+ "q4",+ [+ 1320396,+ "b80cb1cb6ab80efe646ad49ffa11d398af76e8912eb966971932ef2d8e7fe11b",+ ],+ [+ 368954368,+ "be632cacb4a82ef8be28285e49821b9dd7f96e3231cb90df631890f367110555",+ ],+ ),video("sam3-mask_decoder", "sam3-mask_decoder.onnx", ["mask_decoder.onnx",17809322,diff --git a/packages/vision-sam/src/sam3/encoder.test.ts b/packages/vision-sam/src/sam3/encoder.test.tsnew file mode 100644index 00000000..e33f4e13--- /dev/null+++ b/packages/vision-sam/src/sam3/encoder.test.ts@@ -0,0 +1,107 @@+/**+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ * All rights reserved.+ *+ * This source code is licensed under the SAM License found in the+ * LICENSE file in the root directory of this source tree.+ */++import type {+ SessionProvider,+ VisionModelStore,+ VisionTensorInput,+} from "@framefields/vision";+import { describe, expect, it } from "vitest";+import {+ ensureSamModelsRegistered,+ SAM_MODELS,+ SAM3_ENCODER,+} from "../models/registry.js";+import { holdSam3Encoder } from "./encoder.js";++function fakes() {+ const stats = { sessions: 0, runs: 0, released: 0, ensured: [] as string[] };+ const output = (n: number) => ({ data: new Float32Array(n), dims: [n] });+ const provider: SessionProvider = {+ kind: "webgpu",+ createSession: async () => {+ stats.sessions++;+ return {+ run: async () => ({}),+ runMany: async (_inputs: readonly VisionTensorInput[]) => {+ stats.runs++;+ return {+ "image_embeddings.0": output(4),+ "image_embeddings.1": output(4),+ "image_embeddings.2": output(4),+ };+ },+ release: () => {+ stats.released++;+ },+ };+ },+ };+ const store = {+ ensure: async (key: string) => {+ stats.ensured.push(key);+ return new Uint8Array(1);+ },+ } as unknown as VisionModelStore;+ return { provider, store, stats };+}++const frame = (value: number) => ({+ data: new Uint8Array(4 * 4 * 4).fill(value),+ width: 4,+ height: 4,+});++describe("shared SAM 3 encoder", () => {+ it("runs once per frame for every tracker on a provider, and frees with the last", async () => {+ const { provider, store, stats } = fakes();+ const [a, b] = await Promise.all([+ holdSam3Encoder(store, provider, "fp16"),+ holdSam3Encoder(store, provider, "fp16"),+ ]);+ expect(stats.sessions).toBe(1);+ expect(stats.ensured).toEqual([+ "sam3-vision_encoder-fp16",+ "sam3-vision_encoder-fp16-data",+ ]);+ // Separate decodes of one frame: equal pixels, different buffers.+ await Promise.all([a.encode(frame(7)), b.encode(frame(7))]);+ expect(stats.runs).toBe(1);+ await b.encode(frame(8));+ expect(stats.runs).toBe(2);+ a.release();+ a.release();+ expect(stats.released).toBe(0);+ b.release();+ expect(stats.released).toBe(1);+ await holdSam3Encoder(store, provider, "fp16");+ expect(stats.sessions).toBe(2);+ });++ it("keeps precisions apart", async () => {+ const { provider, store, stats } = fakes();+ await holdSam3Encoder(store, provider, "fp16");+ await holdSam3Encoder(store, provider, "q4");+ expect(stats.sessions).toBe(2);+ });+});++describe("SAM 3 encoder variants", () => {+ it("pins a graph and its weights for every precision a backend picks", () => {+ ensureSamModelsRegistered();+ const keys = new Set(SAM_MODELS.map((model) => model.key));+ for (const precision of Object.values(SAM3_ENCODER)) {+ expect(keys).toContain(`sam3-vision_encoder-${precision}`);+ expect(keys).toContain(`sam3-vision_encoder-${precision}-data`);+ }+ // q4f16 keeps the cache file names it was first downloaded under.+ expect(+ SAM_MODELS.find((m) => m.key === "sam3-vision_encoder-q4f16")?.filename,+ ).toBe("sam3-vision_encoder_q4f16.onnx");+ });+});diff --git a/packages/vision-sam/src/sam3/encoder.ts b/packages/vision-sam/src/sam3/encoder.tsnew file mode 100644index 00000000..c0c1d938--- /dev/null+++ b/packages/vision-sam/src/sam3/encoder.ts@@ -0,0 +1,155 @@+/**+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ * All rights reserved.+ *+ * This source code is licensed under the SAM License found in the+ * LICENSE file in the root directory of this source tree.+ */++import {+ imageToTensor,+ type SessionProvider,+ type VisionImageInput,+ type VisionModelStore,+ type VisionSession,+} from "@framefields/vision";+import { type Sam3Precision, sam3EncoderFiles } from "../models/registry.js";+import { sam3Floats, sam3Run } from "./text-selector.js";++/** One frame through the SAM 3 image encoder. */+export interface Sam3Features {+ /** The 1008² preprocessed input (the text selector reuses it). */+ readonly pixels: Float32Array;+ readonly feats0: Float32Array;+ readonly feats1: Float32Array;+ /** Lowest-resolution features, conditioned on memory per object. */+ readonly raw: Float32Array;+}++/** A tracker's hold on the shared encoder. */+export interface Sam3EncoderHandle {+ encode(image: VisionImageInput): Promise<Sam3Features>;+ release(): void;+}++/** Frames kept: clips of one source ask for the same frame back to back. */+const RECENT = 2;++/**+ * The encoder is most of SAM 3's per-frame cost (3.6 s at fp16 on an M1 Pro), and every+ * clip of a source sees identical pixels. So trackers on one provider and precision share+ * one session and the features of the last frames, keyed by pixel content.+ */+class SharedEncoder {+ private users = 0;+ private readonly recent: { key: string; features: Promise<Sam3Features> }[] =+ [];++ constructor(+ private readonly session: VisionSession,+ private readonly forget: () => void,+ ) {}++ hold(): Sam3EncoderHandle {+ this.users++;+ let held = true;+ return {+ encode: (image) => this.encode(image),+ release: () => {+ if (!held) return;+ held = false;+ if (--this.users > 0) return;+ this.forget();+ this.recent.length = 0;+ this.session.release();+ },+ };+ }++ private encode(image: VisionImageInput): Promise<Sam3Features> {+ const key = frameKey(image);+ const hit = this.recent.find((entry) => entry.key === key);+ if (hit) return hit.features;+ const entry = { key, features: this.run(image) };+ this.recent.push(entry);+ if (this.recent.length > RECENT) this.recent.shift();+ entry.features.catch(() => {+ const index = this.recent.indexOf(entry);+ if (index >= 0) this.recent.splice(index, 1);+ });+ return entry.features;+ }++ private async run(image: VisionImageInput): Promise<Sam3Features> {+ const { tensor: pixels } = imageToTensor(image, {+ width: 1008,+ height: 1008,+ fit: "stretch",+ channels: "rgb",+ padValue: 0,+ mean: [127.5, 127.5, 127.5],+ std: [127.5, 127.5, 127.5],+ });+ const out = await sam3Run(this.session, [+ { name: "pixel_values", data: pixels, dims: [1, 3, 1008, 1008] },+ ]);+ return {+ pixels,+ feats0: sam3Floats(out, "image_embeddings.0"),+ feats1: sam3Floats(out, "image_embeddings.1"),+ raw: sam3Floats(out, "image_embeddings.2"),+ };+ }+}++const shared = new WeakMap<+ SessionProvider,+ Map<Sam3Precision, Promise<SharedEncoder>>+>();++/** Holds the process's SAM 3 encoder for `provider` at `precision`, creating it on first use. */+export async function holdSam3Encoder(+ store: VisionModelStore,+ provider: SessionProvider,+ precision: Sam3Precision,+): Promise<Sam3EncoderHandle> {+ let byPrecision = shared.get(provider);+ if (!byPrecision) {+ byPrecision = new Map();+ shared.set(provider, byPrecision);+ }+ const encoders = byPrecision;+ let encoder = encoders.get(precision);+ if (!encoder) {+ encoder = (async () => {+ const session = await provider.createSession(+ await store.ensure(`sam3-vision_encoder-${precision}`),+ {+ externalData: [+ {+ path: sam3EncoderFiles(precision).data,+ data: await store.ensure(`sam3-vision_encoder-${precision}-data`),+ },+ ],+ },+ );+ return new SharedEncoder(session, () => encoders.delete(precision));+ })();+ encoders.set(precision, encoder);+ encoder.catch(() => encoders.delete(precision));+ }+ return (await encoder).hold();+}++/** Content key of a frame: size plus two 32-bit FNV-1a hashes over its bytes. */+function frameKey(image: VisionImageInput): string {+ const bytes = image.data;+ let a = 0x811c9dc5,+ b = 0x01000193 ^ bytes.length;+ for (let i = 0; i < bytes.length; i++) {+ const byte = bytes[i] ?? 0;+ a = Math.imul(a ^ byte, 0x01000193);+ b = Math.imul(b ^ byte, 0x5bd1e995) ^ (b >>> 15);+ }+ return `${image.width}x${image.height}:${(a >>> 0).toString(36)}${(b >>> 0).toString(36)}`;+}diff --git a/packages/vision-sam/src/sam3/text-selector.ts b/packages/vision-sam/src/sam3/text-selector.tsindex 9a162b6f..332cf8c3 100644--- a/packages/vision-sam/src/sam3/text-selector.ts+++ b/packages/vision-sam/src/sam3/text-selector.ts@@ -79,7 +79,7 @@ export class Sam3TextSelector {}/**- * Masks (288², binary) of the instances matching `text` with sigmoid score of at least+ * Mask logits (288²) of the instances matching `text` with sigmoid score of at least* `threshold`, best first and at most `max`. Queries that repeat an accepted instance* (mask IoU above 0.5) are skipped.*/@@ -143,10 +143,7 @@ export class Sam3TextSelector {const selected: Float32Array[] = [];for (const query of ranked) {if (selected.length >= max || (scores[query] ?? -Infinity) < logit) break;- const candidate = Float32Array.from(- masks.subarray(query * CELLS, (query + 1) * CELLS),- (v) => (v > 0 ? 1 : 0),- );+ const candidate = masks.slice(query * CELLS, (query + 1) * CELLS);if (selected.every((kept) => maskIou(kept, candidate) <= 0.5))selected.push(candidate);}@@ -165,10 +162,10 @@ function maskIou(a: Float32Array, b: Float32Array): number {let both = 0,either = 0;for (let i = 0; i < a.length; i++) {- const x = a[i] ?? 0,- y = b[i] ?? 0;- both += x * y;- either += Math.max(x, y);+ const x = (a[i] ?? 0) > 0,+ y = (b[i] ?? 0) > 0;+ if (x && y) both++;+ if (x || y) either++;}return either ? both / either : 0;}diff --git a/packages/vision-sam/src/sam3/tracker.ts b/packages/vision-sam/src/sam3/tracker.tsindex 8deb10ce..de5454f6 100644--- a/packages/vision-sam/src/sam3/tracker.ts+++ b/packages/vision-sam/src/sam3/tracker.ts@@ -8,8 +8,8 @@import {createDefaultSessionProvider,- imageToTensor,type SessionProvider,+ sessionCompute,type VisionImageInput,VisionModelStore,type VisionModelStoreOptions,@@ -18,9 +18,14 @@ import {type VisionTensorOutputs,} from "@framefields/vision";import { assertSamLicenseAccepted } from "../license.js";-import { ensureSamModelsRegistered } from "../models/registry.js";+import {+ ensureSamModelsRegistered,+ SAM3_ENCODER,+ type Sam3Precision,+} from "../models/registry.js";import { sam2Tokens } from "../sam2/memory.js";import type { Sam2Point } from "../sam2/types.js";+import { holdSam3Encoder, type Sam3EncoderHandle } from "./encoder.js";import { sam3DenseDecoder, sam3SupportGraphs } from "./graphs.js";import { Sam3MemoryBank, sam3Position } from "./memory.js";import { Sam3TextSelector, sam3Floats, sam3Run } from "./text-selector.js";@@ -31,9 +36,18 @@ import {validateSam3Prompts,} from "./types.js";-/** All graph and tokenizer revisions participate in ClipVision's persistent cache identity. */-export const SAM3_REVISION =- "sam3-q4f16-429305c8-video-e942689a-support-68ce2746-text-fe8f8e12-v1";+/** Every graph and tokenizer revision, and the encoder precision: ClipVision's cache identity. */+export function sam3Revision(precision: Sam3Precision): string {+ return `sam3-${precision}-429305c8-video-e942689a-support-68ce2746-text-fe8f8e12-v2`;+}++/** The encoder precision a tracker on `provider` uses: the override, else the backend's best. */+export async function sam3Precision(+ provider: SessionProvider = createDefaultSessionProvider(),+ override?: Sam3Precision,+): Promise<Sam3Precision> {+ return override ?? SAM3_ENCODER[await sessionCompute(provider)];+}export interface Sam3TrackerOptions extends VisionModelStoreOptions {readonly provider?: SessionProvider;@@ -42,12 +56,13 @@ export interface Sam3TrackerOptions extends VisionModelStoreOptions {readonly textThreshold?: number;/** Most instances a text prompt selects. Default 1. */readonly maxObjects?: number;+ /** Image-encoder weights; defaults to the fastest for the provider's backend (`SAM3_ENCODER`). */+ readonly precision?: Sam3Precision;/** Explicitly agree to Meta's SAM License terms to download or run SAM models. */readonly acceptLicense?: boolean;}type Graph =- | "vision_encoder"| "mask_decoder"| "dense_decoder"| "dense_prompt"@@ -59,8 +74,16 @@ type Graph =interface Seed {readonly points?: readonly Sam2Point[];readonly dense?: Float32Array;+ /**+ * A mask seed's own 1008² logits (±10). Like SAM's `_use_mask_as_output`, the seed+ * frame outputs and remembers the given mask; the decoder only supplies its pointer.+ */+ readonly mask?: Float32Array;}+/** Logit SAM uses for a given mask's pixels (and its object score): ±10. */+const MASK_LOGIT = 10;+function tensor(name: string,data: VisionTensorInput["data"],@@ -75,6 +98,7 @@ function tensor(*/export class Sam3Tracker {private readonly sessions = new Map<Graph, VisionSession>();+ private encoder?: Sam3EncoderHandle;private readonly position = sam3Position();private readonly banks: Sam3MemoryBank[] = [];private temporal?: Float32Array;@@ -119,19 +143,10 @@ export class Sam3Tracker {const provider = options.provider ?? createDefaultSessionProvider();const tracker = new Sam3Tracker(store, provider, options);try {- tracker.sessions.set(- "vision_encoder",- await provider.createSession(- await store.ensure("sam3-vision_encoder"),- {- externalData: [- {- path: "vision_encoder_q4f16.onnx_data",- data: await store.ensure("sam3-vision_encoder-data"),- },- ],- },- ),+ tracker.encoder = await holdSam3Encoder(+ store,+ provider,+ await sam3Precision(provider, options.precision),);const support = sam3SupportGraphs(await store.ensure("sam3-support"),@@ -194,30 +209,11 @@ export class Sam3Tracker {if (this.lastFrame < 0 && !prompt)throw new Error("SAM 3 needs a prompt on its first frame.");- const { tensor: pixels } = imageToTensor(image, {- width: 1008,- height: 1008,- fit: "stretch",- channels: "rgb",- padValue: 0,- mean: [127.5, 127.5, 127.5],- std: [127.5, 127.5, 127.5],- });- const vision = await this.run("vision_encoder", [- tensor("pixel_values", pixels, [1, 3, 1008, 1008]),- ]);- const raw = sam3Floats(vision, "image_embeddings.2");+ if (!this.encoder) throw new Error("SAM 3 tracker has been released.");+ const { pixels, feats0, feats1, raw } = await this.encoder.encode(image);const shared = [- tensor(- "feats0",- sam3Floats(vision, "image_embeddings.0"),- [1, 32, 288, 288],- ),- tensor(- "feats1",- sam3Floats(vision, "image_embeddings.1"),- [1, 64, 144, 144],- ),+ tensor("feats0", feats0, [1, 32, 288, 288]),+ tensor("feats1", feats1, [1, 64, 144, 144]),];const seeds = await this.seeds(pixels, prompt);const objects: Sam3ObjectResult[] = [];@@ -227,18 +223,16 @@ export class Sam3Tracker {? this.initialFeatures(raw): await this.condition(raw, frame, bank);const decoded = await this.decode(shared, features, seed);- await this.remember(bank, raw, decoded, frame, seed !== undefined);- const score = sam3Floats(decoded, "object_score_logits")[0] ?? -Infinity;+ const logits = seed?.mask ?? sam3Floats(decoded, "high_res_masks");+ const score = seed?.mask+ ? MASK_LOGIT+ : (sam3Floats(decoded, "object_score_logits")[0] ?? -Infinity);+ await this.remember(bank, raw, decoded, frame, seed, logits, score);objects.push({id: index + 1,- mask: sam3Mask(- sam3Floats(decoded, "high_res_masks"),- image.width,- image.height,- score,- ),+ mask: sam3Mask(logits, image.width, image.height, score),score,- iou: Math.max(...sam3Floats(decoded, "iou_scores")),+ iou: seed?.mask ? 1 : Math.max(...sam3Floats(decoded, "iou_scores")),});}this.lastFrame = frame;@@ -273,12 +267,16 @@ export class Sam3Tracker {this.options.maxObjects ?? 1,);this.banks.length = 0;- for (const [index, mask] of masks.entries()) {+ for (const [index, logits] of masks.entries()) {this.banks.push(this.newBank());+ const binary = Float32Array.from(logits, (v) => (v > 0 ? 1 : 0));const dense = await this.run("dense_prompt", [- tensor("mask", mask, [1, 1, 288, 288]),+ tensor("mask", binary, [1, 1, 288, 288]),]);- seeds.set(index, { dense: sam3Floats(dense, "dense_prompt") });+ seeds.set(index, {+ dense: sam3Floats(dense, "dense_prompt"),+ mask: seedLogits(logits),+ });}}return seeds;@@ -374,21 +372,15 @@ export class Sam3Tracker {raw: Float32Array,decoded: VisionTensorOutputs,frame: number,- prompted: boolean,+ seed: Seed | undefined,+ logits: Float32Array,+ score: number,): Promise<void> {const memory = await this.run("memory_encoder", [tensor("vision_features", raw, [1, 256, 72, 72]),- tensor(- "pred_masks_high_res",- sam3Floats(decoded, "high_res_masks"),- [1, 1, 1008, 1008],- ),- tensor(- "object_score_logits",- sam3Floats(decoded, "object_score_logits"),- [1, 1],- ),- tensor("is_mask_from_pts", new Uint8Array([prompted ? 1 : 0]), [1]),+ tensor("pred_masks_high_res", logits, [1, 1, 1008, 1008]),+ tensor("object_score_logits", Float32Array.of(score), [1, 1]),+ tensor("is_mask_from_pts", new Uint8Array([seed ? 1 : 0]), [1]),]);bank.remember({@@ -397,13 +389,15 @@ export class Sam3Tracker {positions: sam3Floats(memory, "memory_pos_enc").slice(),pointer: sam3Floats(decoded, "object_pointer").slice(),},- prompted,+ seed !== undefined,);}release(): void {for (const session of this.sessions.values()) session.release();this.sessions.clear();+ this.encoder?.release();+ this.encoder = undefined;this.selector?.release();this.banks.length = 0;this.temporal = undefined;@@ -443,3 +437,29 @@ export function sam3Mask(}return mask;}++/** A 288² concept mask's logits as SAM's 1008² seed logits: bilinear, then ±10. */+function seedLogits(logits: Float32Array): Float32Array {+ const out = new Float32Array(1008 * 1008);+ for (let y = 0; y < 1008; y++) {+ const sy = Math.max(0, Math.min(287, ((y + 0.5) * 288) / 1008 - 0.5)),+ y0 = Math.floor(sy),+ y1 = Math.min(287, y0 + 1),+ fy = sy - y0;+ for (let x = 0; x < 1008; x++) {+ const sx = Math.max(0, Math.min(287, ((x + 0.5) * 288) / 1008 - 0.5)),+ x0 = Math.floor(sx),+ x1 = Math.min(287, x0 + 1),+ fx = sx - x0;+ const top =+ (logits[y0 * 288 + x0] ?? 0) * (1 - fx) ++ (logits[y0 * 288 + x1] ?? 0) * fx,+ bottom =+ (logits[y1 * 288 + x0] ?? 0) * (1 - fx) ++ (logits[y1 * 288 + x1] ?? 0) * fx;+ out[y * 1008 + x] =+ top * (1 - fy) + bottom * fy > 0 ? MASK_LOGIT : -MASK_LOGIT;+ }+ }+ return out;+}diff --git a/packages/vision-sam/src/sam3/types.ts b/packages/vision-sam/src/sam3/types.tsindex 2f214d1b..e24d5098 100644--- a/packages/vision-sam/src/sam3/types.ts+++ b/packages/vision-sam/src/sam3/types.ts@@ -7,6 +7,7 @@*/import type { SamLicenseOptions } from "../license.js";+import type { Sam3Precision } from "../models/registry.js";import type { Sam2Point } from "../sam2/types.js";import { validateSam2Prompts } from "../sam2/types.js";@@ -35,6 +36,8 @@ export interface Sam3Options extends SamLicenseOptions {readonly textThreshold?: number;/** Most instances a text prompt selects, as track IDs 1…N by score. Default 1. */readonly maxObjects?: number;+ /** Image-encoder weights; defaults to the fastest for the backend in use (`SAM3_ENCODER`). */+ readonly precision?: Sam3Precision;}export function validateSam3Prompts(options: Sam3Options): void {diff --git a/packages/vision/src/model/probe-model.test.ts b/packages/vision/src/model/probe-model.test.tsnew file mode 100644index 00000000..621d43ed--- /dev/null+++ b/packages/vision/src/model/probe-model.test.ts@@ -0,0 +1,14 @@+import * as ort from "onnxruntime-node";+import { describe, expect, it } from "vitest";+import { probeModel } from "./probe-model.js";++describe("probeModel", () => {+ it("is a valid ONNX model that onnxruntime runs", async () => {+ const session = await ort.InferenceSession.create(probeModel());+ const { y } = await session.run({+ x: new ort.Tensor("float32", new Float32Array([3]), [1]),+ });+ expect(Array.from(y?.data as Float32Array)).toEqual([3]);+ await session.release();+ });+});diff --git a/packages/vision/src/model/probe-model.ts b/packages/vision/src/model/probe-model.tsnew file mode 100644index 00000000..45f4028f--- /dev/null+++ b/packages/vision/src/model/probe-model.ts@@ -0,0 +1,69 @@+import { concat, encoder, lengthDelimited, varint } from "./onnx-proto.js";++// ModelProto / GraphProto / NodeProto / ValueInfoProto / TypeProto field numbers.+const MODEL_IR_VERSION = 1;+const MODEL_OPSET_IMPORT = 8;+const MODEL_GRAPH = 7;+const OPSET_VERSION = 2;+const GRAPH_NODE = 1;+const GRAPH_NAME = 2;+const GRAPH_INPUT = 11;+const GRAPH_OUTPUT = 12;+const NODE_INPUT = 1;+const NODE_OUTPUT = 2;+const NODE_OP_TYPE = 4;+const VALUE_NAME = 1;+const VALUE_TYPE = 2;+const TYPE_TENSOR = 1;+const TENSOR_ELEM_TYPE = 1;+const TENSOR_SHAPE = 2;+const SHAPE_DIM = 1;+const DIM_VALUE = 1;+const FLOAT = 1;++const text = (field: number, value: string) =>+ lengthDelimited(field, encoder.encode(value));+const number = (field: number, value: number) =>+ new Uint8Array([...varint(field * 8), ...varint(value)]);++function floatVector(name: string): Uint8Array {+ const dim = lengthDelimited(SHAPE_DIM, number(DIM_VALUE, 1));+ const tensor = concat([+ number(TENSOR_ELEM_TYPE, FLOAT),+ lengthDelimited(TENSOR_SHAPE, dim),+ ]);+ return concat([+ text(VALUE_NAME, name),+ lengthDelimited(VALUE_TYPE, lengthDelimited(TYPE_TENSOR, tensor)),+ ]);+}++let probe: Uint8Array | undefined;++/**+ * A one-node ONNX model (`y = Identity(x)`, float[1]). Creating a session from it tells+ * whether an execution provider works here before any real model is downloaded.+ */+export function probeModel(): Uint8Array {+ probe ??= concat([+ number(MODEL_IR_VERSION, 8),+ lengthDelimited(MODEL_OPSET_IMPORT, number(OPSET_VERSION, 13)),+ lengthDelimited(+ MODEL_GRAPH,+ concat([+ lengthDelimited(+ GRAPH_NODE,+ concat([+ text(NODE_INPUT, "x"),+ text(NODE_OUTPUT, "y"),+ text(NODE_OP_TYPE, "Identity"),+ ]),+ ),+ text(GRAPH_NAME, "probe"),+ lengthDelimited(GRAPH_INPUT, floatVector("x")),+ lengthDelimited(GRAPH_OUTPUT, floatVector("y")),+ ]),+ ),+ ]);+ return probe;+}diff --git a/packages/vision/src/runtime/session-provider.test.ts b/packages/vision/src/runtime/session-provider.test.tsindex 88363b0d..d8edfc5b 100644--- a/packages/vision/src/runtime/session-provider.test.ts+++ b/packages/vision/src/runtime/session-provider.test.ts@@ -47,6 +47,15 @@ describe("session provider selection", () => {}});+ it("knows its backend before the first real session, then keeps it", async () => {+ vi.stubEnv("FRAMEFIELDS_VISION_PROVIDER", "cpu");+ const provider = new AutoSessionProvider();+ expect(await provider.backend()).toBe("cpu");+ expect(provider.kind).toBe("node");+ vi.stubEnv("FRAMEFIELDS_VISION_PROVIDER", "webgpu");+ expect(await provider.backend()).toBe("cpu");+ });+it("shares one default provider per process", () => {expect(createDefaultSessionProvider()).toBe(createDefaultSessionProvider());});diff --git a/packages/vision/src/runtime/session-provider.ts b/packages/vision/src/runtime/session-provider.tsindex 92a5c623..d035074d 100644--- a/packages/vision/src/runtime/session-provider.ts+++ b/packages/vision/src/runtime/session-provider.ts@@ -1,3 +1,4 @@+import { probeModel } from "../model/probe-model.js";import { NodeWebGPUProvider } from "./node-webgpu-provider.js";/**@@ -38,16 +39,32 @@ export interface VisionSessionOptions {}[];}+/** The hardware sessions run on; it decides which model precision runs fastest. */+export type VisionCompute = "webgpu" | "cuda" | "cpu";+export interface SessionProvider {readonly kind: "node" | "webgpu";/** The GPU device sessions run on, when they accept `gpuBuffer` inputs from it. */readonly device?: GPUDevice;+ /**+ * Where sessions will run, known before any model is downloaded (see+ * {@link sessionCompute}). Optional for custom providers.+ */+ backend?(): Promise<VisionCompute>;createSession(modelBytes: Uint8Array,options?: VisionSessionOptions,): Promise<VisionSession>;}+/** Where a provider's sessions run: its own answer, else WebGPU for WebGPU providers, else CPU. */+export function sessionCompute(+ provider: SessionProvider,+): Promise<VisionCompute> {+ if (provider.backend) return provider.backend();+ return Promise.resolve(provider.kind === "webgpu" ? "webgpu" : "cpu");+}+let defaultProvider: (() => SessionProvider) | undefined;let visionDevice: GPUDevice | undefined;@@ -74,7 +91,7 @@ export function setDefaultSessionProvider(/** Which backend the process's default provider chose, reported once. */export interface VisionBackendInfo {- readonly backend: "webgpu" | "cpu" | "cuda";+ readonly backend: VisionCompute;/** WebGPU on the renderer's device: frames reach the models without leaving the GPU. */readonly sharedDevice: boolean;/** Why WebGPU wasn't used, when it fell back. */@@ -145,18 +162,39 @@ export class AutoSessionProvider implements SessionProvider {): Promise<VisionSession> {if (this._resolved)return this._resolved.createSession(modelBytes, options);- if (!this._resolving) {- this._resolving = this.resolve(modelBytes, options);- this._resolving.catch(() => {+ if (this._resolving)+ return (await this._resolving).provider.createSession(+ modelBytes,+ options,+ );+ const { provider, first } = await this.start(modelBytes, options);+ return first ?? provider.createSession(modelBytes, options);+ }++ /** Where sessions run. Before the first session it decides with a one-node probe graph. */+ public async backend(): Promise<VisionCompute> {+ if (!this._resolved && !this._resolving)+ (await this.start(probeModel())).first?.release();+ const provider = this._resolved ?? (await this._resolving)?.provider;+ if (!provider) throw new Error("No vision backend could be resolved.");+ return sessionCompute(provider);+ }++ private start(+ modelBytes: Uint8Array,+ options?: VisionSessionOptions,+ ): Promise<{ provider: SessionProvider; first?: VisionSession }> {+ const resolving = this.resolve(modelBytes, options);+ this._resolving = resolving;+ resolving.then(+ ({ provider }) => {+ this._resolved = provider;+ },+ () => {this._resolving = undefined;- });- const { provider, first } = await this._resolving;- this._resolved = provider;- if (first) return first;- return provider.createSession(modelBytes, options);- }- const { provider } = await this._resolving;- return provider.createSession(modelBytes, options);+ },+ );+ return resolving;}private async resolve(@@ -174,11 +212,20 @@ export class AutoSessionProvider implements SessionProvider {}if (backend === "cuda") {reportBackend({ backend: "cuda", sharedDevice: false });- return {- provider: new NodeSessionProvider({- executionProviders: ["cuda", "cpu"],- }),- };+ return { provider: cudaProvider() };+ }+ if (backend === "auto" && (await hasNvidiaDriver())) {+ // NVIDIA on Linux: CUDA when onnxruntime-node can load its CUDA EP, else WebGPU/CPU.+ try {+ const first = await new NodeSessionProvider({+ executionProviders: ["cuda"],+ }).createSession(modelBytes, options);+ reportBackend({ backend: "cuda", sharedDevice: false });+ first.release();+ return { provider: cudaProvider() };+ } catch {+ // No CUDA 12 / cuDNN 9 libraries: fall through.+ }}const webgpu = new NodeWebGPUProvider({wasmFallback: false,@@ -201,6 +248,23 @@ export class AutoSessionProvider implements SessionProvider {}}+function cudaProvider(): NodeSessionProvider {+ return new NodeSessionProvider({ executionProviders: ["cuda", "cpu"] });+}++/** True on Linux with the NVIDIA kernel driver loaded (checked without touching CUDA). */+async function hasNvidiaDriver(): Promise<boolean> {+ if (typeof process === "undefined" || process.platform !== "linux")+ return false;+ try {+ const { access } = await import("node:fs/promises");+ await access("/proc/driver/nvidia/version");+ return true;+ } catch {+ return false;+ }+}+let processProvider: AutoSessionProvider | undefined;/**@@ -285,6 +349,12 @@ export class NodeSessionProvider implements SessionProvider {this._executionProviders = options.executionProviders ?? ["cpu"];}+ public backend(): Promise<VisionCompute> {+ return Promise.resolve(+ this._executionProviders.includes("cuda") ? "cuda" : "cpu",+ );+ }+private ortPromise?: Promise<typeof import("onnxruntime-node")>;private async ort(): Promise<typeof import("onnxruntime-node")> {diff --git a/plugins/framefields/skills/framefields-effects/SKILL.md b/plugins/framefields/skills/framefields-effects/SKILL.mdindex 16ebd96f..8723055c 100644--- a/plugins/framefields/skills/framefields-effects/SKILL.md+++ b/plugins/framefields/skills/framefields-effects/SKILL.md@@ -202,7 +202,7 @@ Inference runs on the GPU through `onnxruntime-web`'s WebGPU backend (an optiona| Variable | Effect || --- | --- |-| `FRAMEFIELDS_VISION_PROVIDER` | `auto` (default), `webgpu`, `cpu`, or `cuda` (onnxruntime-node's CUDA EP on Linux x64; needs the CUDA 12 and cuDNN 9 libraries). A forced backend that fails throws instead of falling back. |+| `FRAMEFIELDS_VISION_PROVIDER` | `auto` (default: CUDA first on Linux with the NVIDIA driver loaded, then WebGPU, then CPU), `webgpu`, `cpu`, or `cuda` (onnxruntime-node's CUDA EP on Linux x64; needs the CUDA 12 and cuDNN 9 libraries). A forced backend that fails throws instead of falling back. || `FRAMEFIELDS_VISION_THREADS` | CPU threads per model. Default on Linux: the CPUs the process may use, cgroup quota included; elsewhere ONNX Runtime's default. |### How it behaves at render time@@ -226,7 +226,7 @@ Inference runs on the GPU through `onnxruntime-web`'s WebGPU backend (an optionaUse SAM for a specific object among similar objects, or to select an object via semantic text prompt (e.g. "performer in black suit"). SAM tracking and text selection run via `@framefields/vision-sam`, an optional peer under Meta's SAM License: install it next to `framefields` (`pnpm add @framefields/vision-sam`); `framefields` imports it only when a clip uses `sam2` or `sam3`, and prompts are validated on that first use. Import `Sam2Tracker`/`Sam3Tracker` from `@framefields/vision-sam`. To use SAM at runtime, license acceptance is required via `acceptSamLicense: true`, `sam2: { acceptLicense: true, ... }`, `sam3: { acceptLicense: true, ... }`, or the environment variable `FRAMEFIELDS_ACCEPT_SAM_LICENSE=1`.-`comp.vision(layer, { sam2 })` runs the complete 512px SAM 2.1 Tiny video pipeline, including temporal memory. `comp.vision(layer, { sam3 })` runs SAM 3 / 3.1 quantized temporal tracking with optional CLIP/tokenizer-driven concept text selection. The pinned model assets download dynamically to the local cache (`~/.cache/framefields/models`) with size/SHA-256 integrity verification.+`comp.vision(layer, { sam2 })` runs the complete 512px SAM 2.1 Tiny video pipeline, including temporal memory. `comp.vision(layer, { sam3 })` runs SAM 3 / 3.1 temporal tracking with optional CLIP/tokenizer-driven concept text selection. The pinned model assets download dynamically to the local cache (`~/.cache/framefields/models`) with size/SHA-256 integrity verification, and only the image encoder the backend runs fastest is fetched: fp16 on WebGPU (Macs, the preview) and CUDA, q4 on the CPU (`SAM3_ENCODER`; override with `sam3.precision: "fp16" | "q4f16" | "q4"`). Every SAM 3 clip in a process shares that encoder and its result per frame, so tracking several clips of one source costs one encoder pass per frame (about 3.6 s at 1008² on an M1 Pro; CPU-only tracking is roughly 10× slower).```tsconst subject = Layer.video("assets/source.mp4", {