Skip to content

Code-first video & image, rendered natively on WebGPU.

fix(vision-sam): commit the SAM model registry and restore core model pins

okanaslb4625503b6fa
diff --git a/.gitignore b/.gitignore
index 94261fee..baf00440 100644
--- a/.gitignore
+++ b/.gitignore
@@ -86,6 +86,8 @@ examples/**/output/
*.tflite
*.task
packages/**/models/
+# Source modules named models/ are code, not weights.
+!packages/**/src/models/
# Local agent workspace (Hermes plans, scratch probes)
.hermes/
diff --git a/packages/vision-sam/src/models/registry.ts b/packages/vision-sam/src/models/registry.ts
new file mode 100644
index 00000000..fb36ebf6
--- /dev/null
+++ b/packages/vision-sam/src/models/registry.ts
@@ -0,0 +1,217 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the SAM License found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {
+ registerVisionModel,
+ VISION_MODELS,
+ type VisionModelDescriptor,
+} from "@framefields/vision";
+
+/**
+ * SAM model registry. Every file is pinned to an immutable Hugging Face revision with its
+ * exact size and SHA-256; the shared `VisionModelStore` verifies both after download.
+ * Nothing here touches the network: the descriptors are registered into the vision
+ * registry on first tracker creation so `@framefields/vision` never lists SAM weights.
+ */
+
+export type Sam2ModelKey =
+ | "sam2.1-tiny-constants"
+ | "sam2.1-tiny-vision_encoder"
+ | "sam2.1-tiny-mask_decoder"
+ | "sam2.1-tiny-memory_encoder"
+ | "sam2.1-tiny-memory_attention"
+ | "sam2.1-tiny-pointer_tpos";
+
+export type Sam3ModelKey =
+ | "sam3-vision_encoder"
+ | "sam3-vision_encoder-data"
+ | "sam3-mask_decoder"
+ | "sam3-memory_encoder"
+ | "sam3-memory_attention"
+ | "sam3-support"
+ | "sam3-support-data"
+ | "sam3-text-vision_encoder"
+ | "sam3-text-vision_encoder-data"
+ | "sam3-text-encoder"
+ | "sam3-text-decoder"
+ | "sam3-tokenizer"
+ | "sam3-tokenizer-config";
+
+export type SamModelKey = Sam2ModelKey | Sam3ModelKey;
+
+export const SAM_LICENSE = "Meta SAM License";
+
+const hf = (repo: string, revision: string, path: string): string =>
+ `https://huggingface.co/${repo}/resolve/${revision}/${path}`;
+
+type Pin = readonly [path: string, bytes: number, sha256: string];
+
+const SAM2_REPO = "diffusionstudio/sam2.1-tiny-video-onnx-fp16";
+const SAM2_REV = "66673b5db39371b7dd7847f4d3bc0d4f4179b79e";
+
+const sam2 = (
+ key: Sam2ModelKey,
+ [path, bytes, sha256]: Pin,
+): VisionModelDescriptor => ({
+ key,
+ family: "sam2",
+ filename: `${key}${path.slice(path.lastIndexOf("."))}`,
+ url: hf(SAM2_REPO, SAM2_REV, path),
+ bytes,
+ sha256,
+ input: [512, 512],
+ source:
+ "Meta SAM 2.1 Hiera Tiny (Apache-2.0), Diffusion Studio 512px fp16 video export",
+ license: "Apache-2.0",
+});
+
+const sam3 =
+ (repo: string, revision: string) =>
+ (
+ key: Sam3ModelKey,
+ filename: string,
+ [path, bytes, sha256]: Pin,
+ ): VisionModelDescriptor => ({
+ key,
+ family: "sam3",
+ filename,
+ url: hf(repo, revision, path),
+ bytes,
+ sha256,
+ input: [1008, 1008],
+ source: `Meta SAM 3 (${SAM_LICENSE}), pinned ${repo} ONNX export`,
+ license: SAM_LICENSE,
+ });
+
+const tracker = sam3(
+ "onnx-community/sam3-tracker-ONNX",
+ "429305c8a5b3de597243d919a07e4e6bdcd00ef7",
+);
+const video = sam3(
+ "jax-image-tools/sam3-video-onnx",
+ "e942689a73d942f83c69b5965881cb3a0f6153ba",
+);
+const support = sam3(
+ "RamRom/sam3-onnx",
+ "68ce2746e903966a88549cdd08ab554ff9162db0",
+);
+const text = sam3(
+ "danilobukvic/sam3-text-onnx",
+ "fe8f8e123bfa42d4e57d14fa2f4928a4c60f4665",
+);
+const SUPPORT_GRAPH =
+ "bundles/sam3-base-video-tracking-ortcuda-v2/graphs/base_tracker_preview_single1.onnx";
+
+export const SAM_MODELS: readonly VisionModelDescriptor[] = [
+ sam2("sam2.1-tiny-constants", [
+ "constants.json",
+ 9781,
+ "67583810f4b80b762285377a5cc1066bbe19cdf8ef7fc87ff6aa78fc79830ef2",
+ ]),
+ sam2("sam2.1-tiny-vision_encoder", [
+ "onnx/vision_encoder.onnx",
+ 58385353,
+ "24e9ccf5994be5c8dd93ebe771a4cc4f8bb122d0524555ac3ba4e7141710b133",
+ ]),
+ sam2("sam2.1-tiny-mask_decoder", [
+ "onnx/mask_decoder.onnx",
+ 8898805,
+ "6bc158b0d028777ca2e25203ece0f27429c436de39a5c7fe301b3858d6059789",
+ ]),
+ sam2("sam2.1-tiny-memory_encoder", [
+ "onnx/memory_encoder.onnx",
+ 2807753,
+ "8b2bb6810444cee684ef938b7464c5645653f236014f5497c3db3654818cc7ce",
+ ]),
+ sam2("sam2.1-tiny-memory_attention", [
+ "onnx/memory_attention.onnx",
+ 13029470,
+ "67c4e4da01bc7f501b88294bb19a8200362915ece04e9d7e367a437658f342c9",
+ ]),
+ sam2("sam2.1-tiny-pointer_tpos", [
+ "onnx/pointer_tpos.onnx",
+ 34228,
+ "3ab5bfce40ed01208301b2d8f596b9c9f7f493dcd7e09ef01ec22e5be3694bd7",
+ ]),
+ tracker("sam3-vision_encoder", "sam3-vision_encoder_q4f16.onnx", [
+ "onnx/vision_encoder_q4f16.onnx",
+ 1307816,
+ "4b96732adab3d7d70ac0964ce540a61fad097fbc10c9b27612f47d676877cefc",
+ ]),
+ tracker("sam3-vision_encoder-data", "sam3-vision_encoder_q4f16.onnx_data", [
+ "onnx/vision_encoder_q4f16.onnx_data",
+ 295622656,
+ "d1d60808376dbd48e34cf89feaee708c13abed229f2398ab512149db34fdab07",
+ ]),
+ video("sam3-mask_decoder", "sam3-mask_decoder.onnx", [
+ "mask_decoder.onnx",
+ 17809322,
+ "4dff9a588828eeceba4285be85fb9c2a0306447ad642d02b1d8f6bfff1f19af9",
+ ]),
+ video("sam3-memory_encoder", "sam3-memory_encoder.onnx", [
+ "memory_encoder.onnx",
+ 5587704,
+ "78146118c2f91073489926bb587a81f129c391b2e0d1dba3aede6fbaa699498f",
+ ]),
+ video("sam3-memory_attention", "sam3-memory_attention.onnx", [
+ "memory_attention.onnx",
+ 34592488,
+ "8b24dc20acbb485b81f0e67ed0980b747aaa836707ccff4567a87e59c0ca1c24",
+ ]),
+ support("sam3-support", "sam3-base_tracker_preview_single1.onnx", [
+ SUPPORT_GRAPH,
+ 3064674,
+ "d80c1958175d5de6ca72f5b95d24d853dbdb511e0851d5918a4dcf7ed000f3ea",
+ ]),
+ support("sam3-support-data", "sam3-base_tracker_preview_single1.onnx.data", [
+ `${SUPPORT_GRAPH}.data`,
+ 63668224,
+ "618f62fbac368e863ee295526efd84ca768fb9914c7e1a5d9f1fb0f28837dda1",
+ ]),
+ text("sam3-text-vision_encoder", "sam3-vision_encoder_int4_fp16attn.onnx", [
+ "vision_encoder_int4_fp16attn.onnx",
+ 5216609,
+ "9d148134938a6a284a829733774d3b1356a706a3397bb1d4be240bf041e3ac0e",
+ ]),
+ text(
+ "sam3-text-vision_encoder-data",
+ "sam3-vision_encoder_int4_fp16attn.onnx.data",
+ [
+ "vision_encoder_int4_fp16attn.onnx.data",
+ 292836352,
+ "b89c9156064e926761f29be3f87b160fd34f4c93f1de46593295d155621829a2",
+ ],
+ ),
+ text("sam3-text-encoder", "sam3-text_encoder_int8.onnx", [
+ "text_encoder_int8.onnx",
+ 357021801,
+ "c0baf8a4165ecc4039bee9389903723e74a68d617d685a4f1dbfc21c58bb4121",
+ ]),
+ text("sam3-text-decoder", "sam3-decoder_int8.onnx", [
+ "decoder_int8.onnx",
+ 26804694,
+ "d721e7b643bb3ee2c48a59d520401c035c702cfbb13cb7f50fa103f0beb27af8",
+ ]),
+ text("sam3-tokenizer", "sam3-tokenizer.json", [
+ "tokenizer.json",
+ 3642073,
+ "6d9109cc838977f3ca94a379eec36aecc7c807e1785cd729660ca2fc0171fb35",
+ ]),
+ text("sam3-tokenizer-config", "sam3-tokenizer_config.json", [
+ "tokenizer_config.json",
+ 799,
+ "39670ad98457fe8f14ca59f6bb74591e9fc850974380a63993e5b8ffc865baa2",
+ ]),
+];
+
+/** Registers the SAM descriptors into the shared vision registry (idempotent). */
+export function ensureSamModelsRegistered(): void {
+ for (const descriptor of SAM_MODELS) {
+ if (!VISION_MODELS[descriptor.key]) registerVisionModel(descriptor);
+ }
+}
diff --git a/packages/vision/src/decode/preprocess.ts b/packages/vision/src/decode/preprocess.ts
index 32c2c590..b8636037 100644
--- a/packages/vision/src/decode/preprocess.ts
+++ b/packages/vision/src/decode/preprocess.ts
@@ -77,8 +77,15 @@ export const PREPROCESS_BY_FAMILY: Readonly<
},
};
+export type PreprocessFamily = keyof typeof PREPROCESS_BY_FAMILY;
+
+/** True for the families `VisionRunner` can preprocess and decode itself. */
+export function isPreprocessFamily(family: string): family is PreprocessFamily {
+ return Object.hasOwn(PREPROCESS_BY_FAMILY, family);
+}
+
export function preprocessSpec(
- family: keyof typeof PREPROCESS_BY_FAMILY,
+ family: PreprocessFamily,
input: readonly [number, number],
): PreprocessSpec {
return { ...PREPROCESS_BY_FAMILY[family], width: input[0], height: input[1] };
diff --git a/packages/vision/src/model/model-store.ts b/packages/vision/src/model/model-store.ts
index 1f6e69cf..acb4ab8c 100644
--- a/packages/vision/src/model/model-store.ts
+++ b/packages/vision/src/model/model-store.ts
@@ -132,11 +132,6 @@ export class VisionModelStore {
}
}
- /** Dynamically registers an external model descriptor into the store registry. */
- public registerModel(descriptor: VisionModelDescriptor): void {
- VISION_MODELS[descriptor.key] = descriptor;
- }
-
/** Drops a cached model (file + memory) so the next `ensure` re-downloads it. */
public async evict(key: VisionModelKey): Promise<void> {
this._buffers.delete(key);
diff --git a/packages/vision/src/model/registry.ts b/packages/vision/src/model/registry.ts
index 0a0aa104..a24ec3b1 100644
--- a/packages/vision/src/model/registry.ts
+++ b/packages/vision/src/model/registry.ts
@@ -4,8 +4,10 @@ import type { VisionTask, VisionVariant } from "@framefields/core";
* Vision model registry — single source of truth for what exists, where it lives, and which
* model serves which task + variant.
*
- * Every model in this package is Apache-2.0 and pinned to an immutable Hugging Face revision.
- * The store verifies the SHA-256 below after download, so a moved or tampered file fails loudly.
+ * Every model is Apache-2.0 and pinned to an immutable Hugging Face revision; the store
+ * verifies the SHA-256 below after download, so a moved or tampered file fails loudly.
+ * Separately licensed packages (e.g. `@framefields/vision-sam`) add their own pinned
+ * models with `registerVisionModel`.
*
* Downloads are ALWAYS lazy: nothing here touches the network. The store (model-store.ts)
* fetches a model only when an inference function that needs it is first called.
@@ -55,7 +57,7 @@ export interface VisionModelDescriptor {
readonly input: readonly [number, number];
/** Upstream project and license, surfaced in docs and errors. */
readonly source: string;
- /** License identifier (e.g. "Apache-2.0" or "Meta SAM License"). Defaults to "Apache-2.0". */
+ /** License identifier when not Apache-2.0; the store requires acceptance to download it. */
readonly license?: string;
}
@@ -70,7 +72,7 @@ const RTMO_SOURCE = "RTMO (OpenMMLab mmpose, Apache-2.0), ONNX export";
const SELFIE_SOURCE =
"MediaPipe Selfie Segmenter (Google, Apache-2.0), ONNX export";
-export const VISION_MODELS: Record<string, VisionModelDescriptor> = {
+const models: Record<VisionModelKey, VisionModelDescriptor> = {
// ── Detection + instance segmentation (COCO 80) ─────────────────────
"rtmdet-ins-t": {
key: "rtmdet-ins-t",
@@ -85,7 +87,6 @@ export const VISION_MODELS: Record<string, VisionModelDescriptor> = {
sha256: "99ee670820c4493aa13e20578b6a8e39781c319846dc41796a77765734beac5c",
input: [640, 640],
source: RTMDET_INS_SOURCE,
- license: "Apache-2.0",
},
"rtmdet-ins-s": {
key: "rtmdet-ins-s",
@@ -94,13 +95,12 @@ export const VISION_MODELS: Record<string, VisionModelDescriptor> = {
url: hf(
RTMDET_INS_REPO,
RTMDET_INS_REV,
- "models/small/rtmdet-ins_s_640x640.onnx",
+ "models/s/rtmdet-ins_s_640x640.onnx",
),
- bytes: 38_745_918,
- sha256: "0faef2be711666ff726d40026e6d11e5fc24c6c45ea4e4cb3ddf295b77e83492",
+ bytes: 43_236_976,
+ sha256: "9cd1787fbf3eb2bd64cc2d8154f41c036f5484b85599e1cbe0227a2dc5a4a08b",
input: [640, 640],
source: RTMDET_INS_SOURCE,
- license: "Apache-2.0",
},
"rtmdet-ins-m": {
key: "rtmdet-ins-m",
@@ -109,77 +109,70 @@ export const VISION_MODELS: Record<string, VisionModelDescriptor> = {
url: hf(
RTMDET_INS_REPO,
RTMDET_INS_REV,
- "models/medium/rtmdet-ins_m_640x640.onnx",
+ "models/m/rtmdet-ins_m_640x640.onnx",
),
- bytes: 104_139_518,
- sha256: "c18fa83ee42a5c37894a896d8e8b615c4d3ddf8b7ffda62fa1e1b12b591b61c9",
+ bytes: 115_735_979,
+ sha256: "a003bce80316e03467c2c2b483df2d5c0398220c2357a44f9bf985929f7eb052",
input: [640, 640],
source: RTMDET_INS_SOURCE,
- license: "Apache-2.0",
},
-
- // ── Top-down pose estimation (COCO 17 keypoints) ──────────────────
+ // ── Multi-person pose (COCO-17) ─────────────────────────────────────
"rtmo-t": {
key: "rtmo-t",
family: "rtmo",
filename: "rtmo-t.onnx",
url: hf(
- "tori29umai/rtmo-onnx-pose",
- "550cbb91cfc29013b35eb65ce90c42d3c907d884",
- "models/tiny/rtmo_tiny_640x640.onnx",
+ "Xenova/RTMO-t",
+ "f29d13ce5ee291fbe505e9d06a049992f05699dc",
+ "onnx/model.onnx",
),
- bytes: 20_814_230,
- sha256: "a1a8c08198f8a14b534958ea346ecce43867c2966bfb578c74d39f40c74ceca5",
- input: [640, 640],
+ bytes: 27_360_569,
+ sha256: "cb54dd042bf997df86d938721c94238940f2bff98ba8d2ccc834430de5c65c13",
+ input: [416, 416],
source: RTMO_SOURCE,
- license: "Apache-2.0",
},
"rtmo-s": {
key: "rtmo-s",
family: "rtmo",
filename: "rtmo-s.onnx",
url: hf(
- "tori29umai/rtmo-onnx-pose",
- "550cbb91cfc29013b35eb65ce90c42d3c907d884",
- "models/small/rtmo_s_640x640.onnx",
+ "Xenova/RTMO-s",
+ "d8c526187f341d287753831c9c8b1ecc4855bba1",
+ "onnx/model.onnx",
),
- bytes: 69_097_650,
- sha256: "25064e7a8585e509aa09fb6fbff3eeae0c1c875d9e512cb59bf4ceab90604b38",
+ bytes: 39_636_400,
+ sha256: "1cd6a3517658903233e9ccb78666554862f57f0944b6886e2048093594aef4e7",
input: [640, 640],
source: RTMO_SOURCE,
- license: "Apache-2.0",
},
"rtmo-m": {
key: "rtmo-m",
family: "rtmo",
filename: "rtmo-m.onnx",
url: hf(
- "tori29umai/rtmo-onnx-pose",
- "550cbb91cfc29013b35eb65ce90c42d3c907d884",
- "models/medium/rtmo_m_640x640.onnx",
+ "Xenova/RTMO-m",
+ "3aba1280472b98ee2bf663482e27e243038d23fe",
+ "onnx/model.onnx",
),
- bytes: 139_067_314,
- sha256: "36009f4dd45a271701b2ca7226f318357f4fb2bc13d702d849509df6c2763f35",
+ bytes: 89_291_929,
+ sha256: "76d82c45e5c4810baf587ecf2638d15cbf7ed196afb2055e977382532021b59e",
input: [640, 640],
source: RTMO_SOURCE,
- license: "Apache-2.0",
},
-
- // ── Binary selfie segmenter (matte) ───────────────────────────────
+ // ── Person matte ────────────────────────────────────────────────────
"selfie-square": {
key: "selfie-square",
family: "selfie",
filename: "selfie-square.onnx",
url: hf(
- "onnx-community/mediapipe_selfie_segmentation_square",
- "b5066db2bc1be0c926a84f331ea06eb6e91f0a14",
+ "onnx-community/mediapipe_selfie_segmentation",
+ "be49485c8e027524be38591817fc5cd31bd9d00e",
"onnx/model.onnx",
),
- bytes: 462_338,
- sha256: "13511eb85ca8eb90b9f0bc94be6bc961448b30efab31f49635beea7b7a2bbcd2",
+ bytes: 462_352,
+ sha256: "3241ac4ad8aa35bdaf33946776db29f7c283a413aa0b0dacb9483594b4531aad",
input: [256, 256],
source: SELFIE_SOURCE,
- license: "Apache-2.0",
},
"selfie-landscape": {
key: "selfie-landscape",
@@ -194,13 +187,16 @@ export const VISION_MODELS: Record<string, VisionModelDescriptor> = {
sha256: "7a0adcfdb1715d3b0ff0f61486d8a181c33f4a208343a5abfccbc6540e872d24",
input: [256, 144],
source: SELFIE_SOURCE,
- license: "Apache-2.0",
},
};
-/** Register external or plugin vision models into the runtime registry. */
+export const VISION_MODELS: Readonly<
+ Record<VisionModelKey, VisionModelDescriptor>
+> = models;
+
+/** Adds a pinned model from a separately licensed package to the registry. */
export function registerVisionModel(descriptor: VisionModelDescriptor): void {
- VISION_MODELS[descriptor.key] = descriptor;
+ models[descriptor.key] = descriptor;
}
/**
diff --git a/packages/vision/src/models.e2e.test.ts b/packages/vision/src/models.e2e.test.ts
index 8b54e2bb..111a23d7 100644
--- a/packages/vision/src/models.e2e.test.ts
+++ b/packages/vision/src/models.e2e.test.ts
@@ -62,9 +62,7 @@ describe.skipIf(!enabled)("vision models (real weights)", () => {
});
it.each(
- (Object.keys(VISION_MODELS) as VisionModelKey[]).filter(
- (key) => VISION_MODELS[key].family !== "sam2",
- ),
+ Object.keys(VISION_MODELS) as VisionModelKey[],
)("%s downloads, verifies and matches its registry input shape", async (key) => {
const bytes = await new VisionModelStore().ensure(key);
const ort = await import("onnxruntime-node");
diff --git a/packages/vision/src/runner/vision-runner.ts b/packages/vision/src/runner/vision-runner.ts
index 4013a5ea..4124c3fd 100644
--- a/packages/vision/src/runner/vision-runner.ts
+++ b/packages/vision/src/runner/vision-runner.ts
@@ -2,6 +2,7 @@ import type { DetectedObject } from "@framefields/core";
import {
type InputTransform,
imageToTensor,
+ isPreprocessFamily,
preprocessSpec,
} from "../decode/preprocess.js";
import {
@@ -361,9 +362,9 @@ export class VisionRunner {
): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {
const session = await this.session(key);
const desc = VISION_MODELS[key];
- if (desc.family === "sam2" || desc.family === "sam3")
+ if (!isPreprocessFamily(desc.family))
throw new Error(
- "Use Sam2Tracker or Sam3Tracker for prompted video segmentation.",
+ `VisionRunner cannot run '${desc.family}' models; use the family's own tracker (e.g. Sam2Tracker or Sam3Tracker from @framefields/vision-sam).`,
);
const spec = preprocessSpec(desc.family, desc.input);
const dims = [1, 3, spec.height, spec.width];