Code-first video & image, rendered natively on WebGPU.
import {
registerVisionModel,
VISION_MODELS,
type VisionCompute,
type VisionModelDescriptor,
} from "@framefields/vision";
/**
* SAM model registry. Every file is pinned to an immutable Hugging Face revision with its
* exact size and SHA-256; the shared `VisionModelStore` verifies both after download.
* Nothing here touches the network: the descriptors are registered into the vision
* registry on first tracker creation so `@framefields/vision` never lists SAM weights.
*/
/** SAM 2.1 checkpoints with a pinned video export. */
export type Sam2Model = "tiny" | "small";
export type Sam2Graph =
| "vision_encoder"
| "mask_decoder"
| "memory_encoder"
| "memory_attention"
| "pointer_tpos";
export type Sam2ModelKey = `sam2.1-${Sam2Model}-${Sam2Graph | "constants"}`;
/**
* Diffusion Studio's fp16 video exports of Meta SAM 2.1 (Apache-2.0). Tiny is built for
* speed at 512² (32×32 features); Small runs SAM 2's native 1024² (64×64), resolving small
* objects, thin structures and identity after cuts better at roughly 4× the encoder and
* 16× the memory-attention work.
*/
export const SAM2_EXPORTS: Readonly<
Record<
Sam2Model,
{ repo: string; revision: string; size: number; name: string }
>
> = {
tiny: {
repo: "diffusionstudio/sam2.1-tiny-video-onnx-fp16",
revision: "66673b5db39371b7dd7847f4d3bc0d4f4179b79e",
size: 512,
name: "Hiera Tiny",
},
small: {
repo: "diffusionstudio/sam2.1-small-video-onnx-fp16",
revision: "927ecec6e6ae2727d0af7720eefdb07f2efe4689",
size: 1024,
name: "Hiera Small",
},
};
/** Weight formats published for the SAM 3 tracker's image encoder. */
export type Sam3Precision = "fp16" | "q4f16" | "q4";
/**
* The encoder precision each backend runs fastest. Measured on an M1 Pro (one 1008² frame):
* WebGPU fp16 3.65 s, q4f16 4.43 s, q4 6.63 s, int8 140 s (its ops fall back to the CPU);
* fp16 is also the closest to full precision. On WebGPU, 4-bit weights are dequantized
* before every matmul, so they only save memory. CUDA runs fp16 on tensor cores. CPUs have
* no fast fp16 path, so they get fp32-activation q4.
*/
export const SAM3_ENCODER: Readonly<Record<VisionCompute, Sam3Precision>> = {
webgpu: "fp16",
cuda: "fp16",
cpu: "q4",
};
export type Sam3ModelKey =
| `sam3-vision_encoder-${Sam3Precision}`
| `sam3-vision_encoder-${Sam3Precision}-data`
| "sam3-mask_decoder"
| "sam3-memory_encoder"
| "sam3-memory_attention"
| "sam3-support"
| "sam3-support-data"
| "sam3-text-vision_encoder"
| "sam3-text-vision_encoder-data"
| "sam3-text-encoder"
| "sam3-text-decoder"
| "sam3-tokenizer"
| "sam3-tokenizer-config";
export type SamModelKey = Sam2ModelKey | Sam3ModelKey;
export const SAM_LICENSE = "Meta SAM License";
const hf = (repo: string, revision: string, path: string): string =>
`https://huggingface.co/${repo}/resolve/${revision}/${path}`;
type Pin = readonly [path: string, bytes: number, sha256: string];
const sam2 = (
model: Sam2Model,
graph: Sam2Graph | "constants",
[path, bytes, sha256]: Pin,
): VisionModelDescriptor => {
const { repo, revision, size, name } = SAM2_EXPORTS[model];
const key: Sam2ModelKey = `sam2.1-${model}-${graph}`;
return {
key,
family: "sam2",
filename: `${key}${path.slice(path.lastIndexOf("."))}`,
url: hf(repo, revision, path),
bytes,
sha256,
input: [size, size],
source: `Meta SAM 2.1 ${name} (Apache-2.0), Diffusion Studio ${size}px fp16 video export`,
license: "Apache-2.0",
};
};
const sam3 =
(repo: string, revision: string) =>
(
key: Sam3ModelKey,
filename: string,
[path, bytes, sha256]: Pin,
): VisionModelDescriptor => ({
key,
family: "sam3",
filename,
url: hf(repo, revision, path),
bytes,
sha256,
input: [1008, 1008],
source: `Meta SAM 3 (${SAM_LICENSE}), pinned ${repo} ONNX export`,
license: SAM_LICENSE,
});
const tracker = sam3(
"onnx-community/sam3-tracker-ONNX",
"429305c8a5b3de597243d919a07e4e6bdcd00ef7",
);
/** The tracker encoder at `precision`: its graph and the weights file the graph names. */
function encoder(
precision: Sam3Precision,
[graphBytes, graphSha]: readonly [number, string],
[dataBytes, dataSha]: readonly [number, string],
): VisionModelDescriptor[] {
const { graph, data } = sam3EncoderFiles(precision);
return [
tracker(`sam3-vision_encoder-${precision}`, `sam3-${graph}`, [
`onnx/${graph}`,
graphBytes,
graphSha,
]),
tracker(`sam3-vision_encoder-${precision}-data`, `sam3-${data}`, [
`onnx/${data}`,
dataBytes,
dataSha,
]),
];
}
/** File names of the encoder at `precision`; the graph refers to its weights by `data`. */
export function sam3EncoderFiles(precision: Sam3Precision): {
graph: string;
data: string;
} {
const graph = `vision_encoder_${precision}.onnx`;
return { graph, data: `${graph}_data` };
}
const video = sam3(
"jax-image-tools/sam3-video-onnx",
"e942689a73d942f83c69b5965881cb3a0f6153ba",
);
const support = sam3(
"RamRom/sam3-onnx",
"68ce2746e903966a88549cdd08ab554ff9162db0",
);
const text = sam3(
"danilobukvic/sam3-text-onnx",
"fe8f8e123bfa42d4e57d14fa2f4928a4c60f4665",
);
const SUPPORT_GRAPH =
"bundles/sam3-base-video-tracking-ortcuda-v2/graphs/base_tracker_preview_single1.onnx";
export const SAM_MODELS: readonly VisionModelDescriptor[] = [
sam2("tiny", "constants", [
"constants.json",
9781,
"67583810f4b80b762285377a5cc1066bbe19cdf8ef7fc87ff6aa78fc79830ef2",
]),
sam2("tiny", "vision_encoder", [
"onnx/vision_encoder.onnx",
58385353,
"24e9ccf5994be5c8dd93ebe771a4cc4f8bb122d0524555ac3ba4e7141710b133",
]),
sam2("tiny", "mask_decoder", [
"onnx/mask_decoder.onnx",
8898805,
"6bc158b0d028777ca2e25203ece0f27429c436de39a5c7fe301b3858d6059789",
]),
sam2("tiny", "memory_encoder", [
"onnx/memory_encoder.onnx",
2807753,
"8b2bb6810444cee684ef938b7464c5645653f236014f5497c3db3654818cc7ce",
]),
sam2("tiny", "memory_attention", [
"onnx/memory_attention.onnx",
13029470,
"67c4e4da01bc7f501b88294bb19a8200362915ece04e9d7e367a437658f342c9",
]),
sam2("tiny", "pointer_tpos", [
"onnx/pointer_tpos.onnx",
34228,
"3ab5bfce40ed01208301b2d8f596b9c9f7f493dcd7e09ef01ec22e5be3694bd7",
]),
sam2("small", "constants", [
"constants.json",
9820,
"09397f7251c159b64a2d581cc3ea0dbb3cf720c2e60777cb14e244405ed1c305",
]),
sam2("small", "vision_encoder", [
"onnx/vision_encoder.onnx",
83679474,
"fb570036f5afac59ef20848958b075c84da746e333efd863104d94604652efc4",
]),
sam2("small", "mask_decoder", [
"onnx/mask_decoder.onnx",
8910345,
"999b3b47bf2e924d522ed6cc011fa2c9b651754cc07895eb0da16a28a5ce7c4d",
]),
sam2("small", "memory_encoder", [
"onnx/memory_encoder.onnx",
2807884,
"f36324e383dbf21fcd549169aedf698db00c307fd9a49538c0e5f0aa168e4f99",
]),
sam2("small", "memory_attention", [
"onnx/memory_attention.onnx",
16175204,
"6666be8ca502b5c16e9a113d3d3934996d32f27e93bee5e66a8317fcceec9985",
]),
sam2("small", "pointer_tpos", [
"onnx/pointer_tpos.onnx",
34228,
"f9c1b36e0aa9dc6b7c05b02a8fdaa17cc671d967ffcacc14fcd3177e65ec25e9",
]),
...encoder(
"fp16",
[
1263005,
"f8c52be6de99124bb17f25792054406abc482ce3502166f948d5c5849cd14d02",
],
[
934729728,
"4b021a4d3068c9e9153f2f2fcc9e6d280156228671c00ab6c2ddfa737da7f151",
],
),
...encoder(
"q4f16",
[
1307816,
"4b96732adab3d7d70ac0964ce540a61fad097fbc10c9b27612f47d676877cefc",
],
[
295622656,
"d1d60808376dbd48e34cf89feaee708c13abed229f2398ab512149db34fdab07",
],
),
...encoder(
"q4",
[
1320396,
"b80cb1cb6ab80efe646ad49ffa11d398af76e8912eb966971932ef2d8e7fe11b",
],
[
368954368,
"be632cacb4a82ef8be28285e49821b9dd7f96e3231cb90df631890f367110555",
],
),
video("sam3-mask_decoder", "sam3-mask_decoder.onnx", [
"mask_decoder.onnx",
17809322,
"4dff9a588828eeceba4285be85fb9c2a0306447ad642d02b1d8f6bfff1f19af9",
]),
video("sam3-memory_encoder", "sam3-memory_encoder.onnx", [
"memory_encoder.onnx",
5587704,
"78146118c2f91073489926bb587a81f129c391b2e0d1dba3aede6fbaa699498f",
]),
video("sam3-memory_attention", "sam3-memory_attention.onnx", [
"memory_attention.onnx",
34592488,
"8b24dc20acbb485b81f0e67ed0980b747aaa836707ccff4567a87e59c0ca1c24",
]),
support("sam3-support", "sam3-base_tracker_preview_single1.onnx", [
SUPPORT_GRAPH,
3064674,
"d80c1958175d5de6ca72f5b95d24d853dbdb511e0851d5918a4dcf7ed000f3ea",
]),
support("sam3-support-data", "sam3-base_tracker_preview_single1.onnx.data", [
`${SUPPORT_GRAPH}.data`,
63668224,
"618f62fbac368e863ee295526efd84ca768fb9914c7e1a5d9f1fb0f28837dda1",
]),
text("sam3-text-vision_encoder", "sam3-vision_encoder_int4_fp16attn.onnx", [
"vision_encoder_int4_fp16attn.onnx",
5216609,
"9d148134938a6a284a829733774d3b1356a706a3397bb1d4be240bf041e3ac0e",
]),
text(
"sam3-text-vision_encoder-data",
"sam3-vision_encoder_int4_fp16attn.onnx.data",
[
"vision_encoder_int4_fp16attn.onnx.data",
292836352,
"b89c9156064e926761f29be3f87b160fd34f4c93f1de46593295d155621829a2",
],
),
text("sam3-text-encoder", "sam3-text_encoder_int8.onnx", [
"text_encoder_int8.onnx",
357021801,
"c0baf8a4165ecc4039bee9389903723e74a68d617d685a4f1dbfc21c58bb4121",
]),
text("sam3-text-decoder", "sam3-decoder_int8.onnx", [
"decoder_int8.onnx",
26804694,
"d721e7b643bb3ee2c48a59d520401c035c702cfbb13cb7f50fa103f0beb27af8",
]),
text("sam3-tokenizer", "sam3-tokenizer.json", [
"tokenizer.json",
3642073,
"6d9109cc838977f3ca94a379eec36aecc7c807e1785cd729660ca2fc0171fb35",
]),
text("sam3-tokenizer-config", "sam3-tokenizer_config.json", [
"tokenizer_config.json",
799,
"39670ad98457fe8f14ca59f6bb74591e9fc850974380a63993e5b8ffc865baa2",
]),
];
/** Registers the SAM descriptors into the shared vision registry (idempotent). */
export function ensureSamModelsRegistered(): void {
for (const descriptor of SAM_MODELS) {
if (!VISION_MODELS[descriptor.key]) registerVisionModel(descriptor);
}
}