Code-first video & image, rendered natively on WebGPU.
fix(vision-sam): commit the SAM model registry and restore core model pins
okanaslb4625503b6fa
diff --git a/.gitignore b/.gitignoreindex 94261fee..baf00440 100644--- a/.gitignore+++ b/.gitignore@@ -86,6 +86,8 @@ examples/**/output/*.tflite*.taskpackages/**/models/+# Source modules named models/ are code, not weights.+!packages/**/src/models/# Local agent workspace (Hermes plans, scratch probes).hermes/diff --git a/packages/vision-sam/src/models/registry.ts b/packages/vision-sam/src/models/registry.tsnew file mode 100644index 00000000..fb36ebf6--- /dev/null+++ b/packages/vision-sam/src/models/registry.ts@@ -0,0 +1,217 @@+/**+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ * All rights reserved.+ *+ * This source code is licensed under the SAM License found in the+ * LICENSE file in the root directory of this source tree.+ */++import {+ registerVisionModel,+ VISION_MODELS,+ type VisionModelDescriptor,+} from "@framefields/vision";++/**+ * SAM model registry. Every file is pinned to an immutable Hugging Face revision with its+ * exact size and SHA-256; the shared `VisionModelStore` verifies both after download.+ * Nothing here touches the network: the descriptors are registered into the vision+ * registry on first tracker creation so `@framefields/vision` never lists SAM weights.+ */++export type Sam2ModelKey =+ | "sam2.1-tiny-constants"+ | "sam2.1-tiny-vision_encoder"+ | "sam2.1-tiny-mask_decoder"+ | "sam2.1-tiny-memory_encoder"+ | "sam2.1-tiny-memory_attention"+ | "sam2.1-tiny-pointer_tpos";++export type Sam3ModelKey =+ | "sam3-vision_encoder"+ | "sam3-vision_encoder-data"+ | "sam3-mask_decoder"+ | "sam3-memory_encoder"+ | "sam3-memory_attention"+ | "sam3-support"+ | "sam3-support-data"+ | "sam3-text-vision_encoder"+ | "sam3-text-vision_encoder-data"+ | "sam3-text-encoder"+ | "sam3-text-decoder"+ | "sam3-tokenizer"+ | "sam3-tokenizer-config";++export type SamModelKey = Sam2ModelKey | Sam3ModelKey;++export const SAM_LICENSE = "Meta SAM License";++const hf = (repo: string, revision: string, path: string): string =>+ `https://huggingface.co/${repo}/resolve/${revision}/${path}`;++type Pin = readonly [path: string, bytes: number, sha256: string];++const SAM2_REPO = "diffusionstudio/sam2.1-tiny-video-onnx-fp16";+const SAM2_REV = "66673b5db39371b7dd7847f4d3bc0d4f4179b79e";++const sam2 = (+ key: Sam2ModelKey,+ [path, bytes, sha256]: Pin,+): VisionModelDescriptor => ({+ key,+ family: "sam2",+ filename: `${key}${path.slice(path.lastIndexOf("."))}`,+ url: hf(SAM2_REPO, SAM2_REV, path),+ bytes,+ sha256,+ input: [512, 512],+ source:+ "Meta SAM 2.1 Hiera Tiny (Apache-2.0), Diffusion Studio 512px fp16 video export",+ license: "Apache-2.0",+});++const sam3 =+ (repo: string, revision: string) =>+ (+ key: Sam3ModelKey,+ filename: string,+ [path, bytes, sha256]: Pin,+ ): VisionModelDescriptor => ({+ key,+ family: "sam3",+ filename,+ url: hf(repo, revision, path),+ bytes,+ sha256,+ input: [1008, 1008],+ source: `Meta SAM 3 (${SAM_LICENSE}), pinned ${repo} ONNX export`,+ license: SAM_LICENSE,+ });++const tracker = sam3(+ "onnx-community/sam3-tracker-ONNX",+ "429305c8a5b3de597243d919a07e4e6bdcd00ef7",+);+const video = sam3(+ "jax-image-tools/sam3-video-onnx",+ "e942689a73d942f83c69b5965881cb3a0f6153ba",+);+const support = sam3(+ "RamRom/sam3-onnx",+ "68ce2746e903966a88549cdd08ab554ff9162db0",+);+const text = sam3(+ "danilobukvic/sam3-text-onnx",+ "fe8f8e123bfa42d4e57d14fa2f4928a4c60f4665",+);+const SUPPORT_GRAPH =+ "bundles/sam3-base-video-tracking-ortcuda-v2/graphs/base_tracker_preview_single1.onnx";++export const SAM_MODELS: readonly VisionModelDescriptor[] = [+ sam2("sam2.1-tiny-constants", [+ "constants.json",+ 9781,+ "67583810f4b80b762285377a5cc1066bbe19cdf8ef7fc87ff6aa78fc79830ef2",+ ]),+ sam2("sam2.1-tiny-vision_encoder", [+ "onnx/vision_encoder.onnx",+ 58385353,+ "24e9ccf5994be5c8dd93ebe771a4cc4f8bb122d0524555ac3ba4e7141710b133",+ ]),+ sam2("sam2.1-tiny-mask_decoder", [+ "onnx/mask_decoder.onnx",+ 8898805,+ "6bc158b0d028777ca2e25203ece0f27429c436de39a5c7fe301b3858d6059789",+ ]),+ sam2("sam2.1-tiny-memory_encoder", [+ "onnx/memory_encoder.onnx",+ 2807753,+ "8b2bb6810444cee684ef938b7464c5645653f236014f5497c3db3654818cc7ce",+ ]),+ sam2("sam2.1-tiny-memory_attention", [+ "onnx/memory_attention.onnx",+ 13029470,+ "67c4e4da01bc7f501b88294bb19a8200362915ece04e9d7e367a437658f342c9",+ ]),+ sam2("sam2.1-tiny-pointer_tpos", [+ "onnx/pointer_tpos.onnx",+ 34228,+ "3ab5bfce40ed01208301b2d8f596b9c9f7f493dcd7e09ef01ec22e5be3694bd7",+ ]),+ tracker("sam3-vision_encoder", "sam3-vision_encoder_q4f16.onnx", [+ "onnx/vision_encoder_q4f16.onnx",+ 1307816,+ "4b96732adab3d7d70ac0964ce540a61fad097fbc10c9b27612f47d676877cefc",+ ]),+ tracker("sam3-vision_encoder-data", "sam3-vision_encoder_q4f16.onnx_data", [+ "onnx/vision_encoder_q4f16.onnx_data",+ 295622656,+ "d1d60808376dbd48e34cf89feaee708c13abed229f2398ab512149db34fdab07",+ ]),+ video("sam3-mask_decoder", "sam3-mask_decoder.onnx", [+ "mask_decoder.onnx",+ 17809322,+ "4dff9a588828eeceba4285be85fb9c2a0306447ad642d02b1d8f6bfff1f19af9",+ ]),+ video("sam3-memory_encoder", "sam3-memory_encoder.onnx", [+ "memory_encoder.onnx",+ 5587704,+ "78146118c2f91073489926bb587a81f129c391b2e0d1dba3aede6fbaa699498f",+ ]),+ video("sam3-memory_attention", "sam3-memory_attention.onnx", [+ "memory_attention.onnx",+ 34592488,+ "8b24dc20acbb485b81f0e67ed0980b747aaa836707ccff4567a87e59c0ca1c24",+ ]),+ support("sam3-support", "sam3-base_tracker_preview_single1.onnx", [+ SUPPORT_GRAPH,+ 3064674,+ "d80c1958175d5de6ca72f5b95d24d853dbdb511e0851d5918a4dcf7ed000f3ea",+ ]),+ support("sam3-support-data", "sam3-base_tracker_preview_single1.onnx.data", [+ `${SUPPORT_GRAPH}.data`,+ 63668224,+ "618f62fbac368e863ee295526efd84ca768fb9914c7e1a5d9f1fb0f28837dda1",+ ]),+ text("sam3-text-vision_encoder", "sam3-vision_encoder_int4_fp16attn.onnx", [+ "vision_encoder_int4_fp16attn.onnx",+ 5216609,+ "9d148134938a6a284a829733774d3b1356a706a3397bb1d4be240bf041e3ac0e",+ ]),+ text(+ "sam3-text-vision_encoder-data",+ "sam3-vision_encoder_int4_fp16attn.onnx.data",+ [+ "vision_encoder_int4_fp16attn.onnx.data",+ 292836352,+ "b89c9156064e926761f29be3f87b160fd34f4c93f1de46593295d155621829a2",+ ],+ ),+ text("sam3-text-encoder", "sam3-text_encoder_int8.onnx", [+ "text_encoder_int8.onnx",+ 357021801,+ "c0baf8a4165ecc4039bee9389903723e74a68d617d685a4f1dbfc21c58bb4121",+ ]),+ text("sam3-text-decoder", "sam3-decoder_int8.onnx", [+ "decoder_int8.onnx",+ 26804694,+ "d721e7b643bb3ee2c48a59d520401c035c702cfbb13cb7f50fa103f0beb27af8",+ ]),+ text("sam3-tokenizer", "sam3-tokenizer.json", [+ "tokenizer.json",+ 3642073,+ "6d9109cc838977f3ca94a379eec36aecc7c807e1785cd729660ca2fc0171fb35",+ ]),+ text("sam3-tokenizer-config", "sam3-tokenizer_config.json", [+ "tokenizer_config.json",+ 799,+ "39670ad98457fe8f14ca59f6bb74591e9fc850974380a63993e5b8ffc865baa2",+ ]),+];++/** Registers the SAM descriptors into the shared vision registry (idempotent). */+export function ensureSamModelsRegistered(): void {+ for (const descriptor of SAM_MODELS) {+ if (!VISION_MODELS[descriptor.key]) registerVisionModel(descriptor);+ }+}diff --git a/packages/vision/src/decode/preprocess.ts b/packages/vision/src/decode/preprocess.tsindex 32c2c590..b8636037 100644--- a/packages/vision/src/decode/preprocess.ts+++ b/packages/vision/src/decode/preprocess.ts@@ -77,8 +77,15 @@ export const PREPROCESS_BY_FAMILY: Readonly<},};+export type PreprocessFamily = keyof typeof PREPROCESS_BY_FAMILY;++/** True for the families `VisionRunner` can preprocess and decode itself. */+export function isPreprocessFamily(family: string): family is PreprocessFamily {+ return Object.hasOwn(PREPROCESS_BY_FAMILY, family);+}+export function preprocessSpec(- family: keyof typeof PREPROCESS_BY_FAMILY,+ family: PreprocessFamily,input: readonly [number, number],): PreprocessSpec {return { ...PREPROCESS_BY_FAMILY[family], width: input[0], height: input[1] };diff --git a/packages/vision/src/model/model-store.ts b/packages/vision/src/model/model-store.tsindex 1f6e69cf..acb4ab8c 100644--- a/packages/vision/src/model/model-store.ts+++ b/packages/vision/src/model/model-store.ts@@ -132,11 +132,6 @@ export class VisionModelStore {}}- /** Dynamically registers an external model descriptor into the store registry. */- public registerModel(descriptor: VisionModelDescriptor): void {- VISION_MODELS[descriptor.key] = descriptor;- }-/** Drops a cached model (file + memory) so the next `ensure` re-downloads it. */public async evict(key: VisionModelKey): Promise<void> {this._buffers.delete(key);diff --git a/packages/vision/src/model/registry.ts b/packages/vision/src/model/registry.tsindex 0a0aa104..a24ec3b1 100644--- a/packages/vision/src/model/registry.ts+++ b/packages/vision/src/model/registry.ts@@ -4,8 +4,10 @@ import type { VisionTask, VisionVariant } from "@framefields/core";* Vision model registry — single source of truth for what exists, where it lives, and which* model serves which task + variant.*- * Every model in this package is Apache-2.0 and pinned to an immutable Hugging Face revision.- * The store verifies the SHA-256 below after download, so a moved or tampered file fails loudly.+ * Every model is Apache-2.0 and pinned to an immutable Hugging Face revision; the store+ * verifies the SHA-256 below after download, so a moved or tampered file fails loudly.+ * Separately licensed packages (e.g. `@framefields/vision-sam`) add their own pinned+ * models with `registerVisionModel`.** Downloads are ALWAYS lazy: nothing here touches the network. The store (model-store.ts)* fetches a model only when an inference function that needs it is first called.@@ -55,7 +57,7 @@ export interface VisionModelDescriptor {readonly input: readonly [number, number];/** Upstream project and license, surfaced in docs and errors. */readonly source: string;- /** License identifier (e.g. "Apache-2.0" or "Meta SAM License"). Defaults to "Apache-2.0". */+ /** License identifier when not Apache-2.0; the store requires acceptance to download it. */readonly license?: string;}@@ -70,7 +72,7 @@ const RTMO_SOURCE = "RTMO (OpenMMLab mmpose, Apache-2.0), ONNX export";const SELFIE_SOURCE ="MediaPipe Selfie Segmenter (Google, Apache-2.0), ONNX export";-export const VISION_MODELS: Record<string, VisionModelDescriptor> = {+const models: Record<VisionModelKey, VisionModelDescriptor> = {// ── Detection + instance segmentation (COCO 80) ─────────────────────"rtmdet-ins-t": {key: "rtmdet-ins-t",@@ -85,7 +87,6 @@ export const VISION_MODELS: Record<string, VisionModelDescriptor> = {sha256: "99ee670820c4493aa13e20578b6a8e39781c319846dc41796a77765734beac5c",input: [640, 640],source: RTMDET_INS_SOURCE,- license: "Apache-2.0",},"rtmdet-ins-s": {key: "rtmdet-ins-s",@@ -94,13 +95,12 @@ export const VISION_MODELS: Record<string, VisionModelDescriptor> = {url: hf(RTMDET_INS_REPO,RTMDET_INS_REV,- "models/small/rtmdet-ins_s_640x640.onnx",+ "models/s/rtmdet-ins_s_640x640.onnx",),- bytes: 38_745_918,- sha256: "0faef2be711666ff726d40026e6d11e5fc24c6c45ea4e4cb3ddf295b77e83492",+ bytes: 43_236_976,+ sha256: "9cd1787fbf3eb2bd64cc2d8154f41c036f5484b85599e1cbe0227a2dc5a4a08b",input: [640, 640],source: RTMDET_INS_SOURCE,- license: "Apache-2.0",},"rtmdet-ins-m": {key: "rtmdet-ins-m",@@ -109,77 +109,70 @@ export const VISION_MODELS: Record<string, VisionModelDescriptor> = {url: hf(RTMDET_INS_REPO,RTMDET_INS_REV,- "models/medium/rtmdet-ins_m_640x640.onnx",+ "models/m/rtmdet-ins_m_640x640.onnx",),- bytes: 104_139_518,- sha256: "c18fa83ee42a5c37894a896d8e8b615c4d3ddf8b7ffda62fa1e1b12b591b61c9",+ bytes: 115_735_979,+ sha256: "a003bce80316e03467c2c2b483df2d5c0398220c2357a44f9bf985929f7eb052",input: [640, 640],source: RTMDET_INS_SOURCE,- license: "Apache-2.0",},-- // ── Top-down pose estimation (COCO 17 keypoints) ──────────────────+ // ── Multi-person pose (COCO-17) ─────────────────────────────────────"rtmo-t": {key: "rtmo-t",family: "rtmo",filename: "rtmo-t.onnx",url: hf(- "tori29umai/rtmo-onnx-pose",- "550cbb91cfc29013b35eb65ce90c42d3c907d884",- "models/tiny/rtmo_tiny_640x640.onnx",+ "Xenova/RTMO-t",+ "f29d13ce5ee291fbe505e9d06a049992f05699dc",+ "onnx/model.onnx",),- bytes: 20_814_230,- sha256: "a1a8c08198f8a14b534958ea346ecce43867c2966bfb578c74d39f40c74ceca5",- input: [640, 640],+ bytes: 27_360_569,+ sha256: "cb54dd042bf997df86d938721c94238940f2bff98ba8d2ccc834430de5c65c13",+ input: [416, 416],source: RTMO_SOURCE,- license: "Apache-2.0",},"rtmo-s": {key: "rtmo-s",family: "rtmo",filename: "rtmo-s.onnx",url: hf(- "tori29umai/rtmo-onnx-pose",- "550cbb91cfc29013b35eb65ce90c42d3c907d884",- "models/small/rtmo_s_640x640.onnx",+ "Xenova/RTMO-s",+ "d8c526187f341d287753831c9c8b1ecc4855bba1",+ "onnx/model.onnx",),- bytes: 69_097_650,- sha256: "25064e7a8585e509aa09fb6fbff3eeae0c1c875d9e512cb59bf4ceab90604b38",+ bytes: 39_636_400,+ sha256: "1cd6a3517658903233e9ccb78666554862f57f0944b6886e2048093594aef4e7",input: [640, 640],source: RTMO_SOURCE,- license: "Apache-2.0",},"rtmo-m": {key: "rtmo-m",family: "rtmo",filename: "rtmo-m.onnx",url: hf(- "tori29umai/rtmo-onnx-pose",- "550cbb91cfc29013b35eb65ce90c42d3c907d884",- "models/medium/rtmo_m_640x640.onnx",+ "Xenova/RTMO-m",+ "3aba1280472b98ee2bf663482e27e243038d23fe",+ "onnx/model.onnx",),- bytes: 139_067_314,- sha256: "36009f4dd45a271701b2ca7226f318357f4fb2bc13d702d849509df6c2763f35",+ bytes: 89_291_929,+ sha256: "76d82c45e5c4810baf587ecf2638d15cbf7ed196afb2055e977382532021b59e",input: [640, 640],source: RTMO_SOURCE,- license: "Apache-2.0",},-- // ── Binary selfie segmenter (matte) ───────────────────────────────+ // ── Person matte ────────────────────────────────────────────────────"selfie-square": {key: "selfie-square",family: "selfie",filename: "selfie-square.onnx",url: hf(- "onnx-community/mediapipe_selfie_segmentation_square",- "b5066db2bc1be0c926a84f331ea06eb6e91f0a14",+ "onnx-community/mediapipe_selfie_segmentation",+ "be49485c8e027524be38591817fc5cd31bd9d00e","onnx/model.onnx",),- bytes: 462_338,- sha256: "13511eb85ca8eb90b9f0bc94be6bc961448b30efab31f49635beea7b7a2bbcd2",+ bytes: 462_352,+ sha256: "3241ac4ad8aa35bdaf33946776db29f7c283a413aa0b0dacb9483594b4531aad",input: [256, 256],source: SELFIE_SOURCE,- license: "Apache-2.0",},"selfie-landscape": {key: "selfie-landscape",@@ -194,13 +187,16 @@ export const VISION_MODELS: Record<string, VisionModelDescriptor> = {sha256: "7a0adcfdb1715d3b0ff0f61486d8a181c33f4a208343a5abfccbc6540e872d24",input: [256, 144],source: SELFIE_SOURCE,- license: "Apache-2.0",},};-/** Register external or plugin vision models into the runtime registry. */+export const VISION_MODELS: Readonly<+ Record<VisionModelKey, VisionModelDescriptor>+> = models;++/** Adds a pinned model from a separately licensed package to the registry. */export function registerVisionModel(descriptor: VisionModelDescriptor): void {- VISION_MODELS[descriptor.key] = descriptor;+ models[descriptor.key] = descriptor;}/**diff --git a/packages/vision/src/models.e2e.test.ts b/packages/vision/src/models.e2e.test.tsindex 8b54e2bb..111a23d7 100644--- a/packages/vision/src/models.e2e.test.ts+++ b/packages/vision/src/models.e2e.test.ts@@ -62,9 +62,7 @@ describe.skipIf(!enabled)("vision models (real weights)", () => {});it.each(- (Object.keys(VISION_MODELS) as VisionModelKey[]).filter(- (key) => VISION_MODELS[key].family !== "sam2",- ),+ Object.keys(VISION_MODELS) as VisionModelKey[],)("%s downloads, verifies and matches its registry input shape", async (key) => {const bytes = await new VisionModelStore().ensure(key);const ort = await import("onnxruntime-node");diff --git a/packages/vision/src/runner/vision-runner.ts b/packages/vision/src/runner/vision-runner.tsindex 4013a5ea..4124c3fd 100644--- a/packages/vision/src/runner/vision-runner.ts+++ b/packages/vision/src/runner/vision-runner.ts@@ -2,6 +2,7 @@ import type { DetectedObject } from "@framefields/core";import {type InputTransform,imageToTensor,+ isPreprocessFamily,preprocessSpec,} from "../decode/preprocess.js";import {@@ -361,9 +362,9 @@ export class VisionRunner {): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {const session = await this.session(key);const desc = VISION_MODELS[key];- if (desc.family === "sam2" || desc.family === "sam3")+ if (!isPreprocessFamily(desc.family))throw new Error(- "Use Sam2Tracker or Sam3Tracker for prompted video segmentation.",+ `VisionRunner cannot run '${desc.family}' models; use the family's own tracker (e.g. Sam2Tracker or Sam3Tracker from @framefields/vision-sam).`,);const spec = preprocessSpec(desc.family, desc.input);const dims = [1, 3, spec.height, spec.width];