Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add fp16 conversion script and update model repository links

konstantin-paulus6d275f1a619b
diff --git a/packages/sam2/scripts/convert_fp16.py b/packages/sam2/scripts/convert_fp16.py
new file mode 100644
index 00000000..a665cc9e
--- /dev/null
+++ b/packages/sam2/scripts/convert_fp16.py
@@ -0,0 +1,105 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = ["huggingface_hub", "numpy", "onnx==1.23.0", "onnxslim==0.1.96"]
+# ///
+"""
+Builds the fp16 graphs the package loads from the fp32 SAM 2.1 video export.
+
+Constants are folded in fp32 first, so positional encodings are computed at
+full precision before they are stored as halves. The graphs then compute in
+fp16 but keep their fp32 inputs and outputs, with a cast at each boundary, so
+the pipeline feeding them is the same for either precision.
+
+ uv run packages/sam2/scripts/convert_fp16.py <out-dir>
+"""
+
+import shutil
+import sys
+from pathlib import Path
+
+import numpy as np
+import onnx
+import onnxslim
+from huggingface_hub import hf_hub_download
+from onnx import TensorProto, helper, numpy_helper
+
+SOURCE_REPO = 'square-zero-labs/sam2.1-tiny-video-onnx'
+SOURCE_REVISION = '3b2984dd865f6e9d2cc6aed0be6a5a5c2eb352ce'
+GRAPHS = ['vision_encoder', 'mask_decoder', 'memory_encoder', 'memory_attention', 'pointer_tpos']
+
+# Folding memory attention's RoPE would store the same table once per layer, doubling the download.
+UNFOLDED = {'memory_attention'}
+FOLD_LIMIT_BYTES = 8 << 20
+
+# Inputs the ONNX spec types as float32 whatever the op computes in, by (op, input index).
+FLOAT32_INPUTS = {('Resize', 1), ('Resize', 2)}
+
+
+def to_fp16(model: onnx.ModelProto) -> onnx.ModelProto:
+ graph = model.graph
+ keep_float = {
+ node.input[i]
+ for node in graph.node
+ for op, i in FLOAT32_INPUTS
+ if node.op_type == op and len(node.input) > i
+ }
+
+ for init in graph.initializer:
+ if init.data_type == TensorProto.FLOAT and init.name not in keep_float:
+ init.CopyFrom(numpy_helper.from_array(to_half(numpy_helper.to_array(init)), init.name))
+
+ for node in graph.node:
+ for attr in node.attribute:
+ if node.op_type == 'Cast' and attr.name == 'to' and attr.i == TensorProto.FLOAT:
+ attr.i = TensorProto.FLOAT16
+ elif (
+ attr.type == onnx.AttributeProto.TENSOR
+ and attr.t.data_type == TensorProto.FLOAT
+ and node.op_type in ('Constant', 'ConstantOfShape')
+ and node.output[0] not in keep_float
+ ):
+ attr.t.CopyFrom(numpy_helper.from_array(to_half(numpy_helper.to_array(attr.t)), attr.t.name))
+
+ # Float inputs are cast down on entry and float outputs cast back up on exit.
+ entry, exit = [], []
+ for vi in graph.input:
+ if vi.type.tensor_type.elem_type == TensorProto.FLOAT:
+ half = f'{vi.name}__fp16'
+ for node in graph.node:
+ node.input[:] = [half if name == vi.name else name for name in node.input]
+ entry.append(helper.make_node('Cast', [vi.name], [half], name=f'{vi.name}__to_fp16', to=TensorProto.FLOAT16))
+ for vi in graph.output:
+ if vi.type.tensor_type.elem_type == TensorProto.FLOAT:
+ half = f'{vi.name}__fp16'
+ for node in graph.node:
+ node.output[:] = [half if name == vi.name else name for name in node.output]
+ node.input[:] = [half if name == vi.name else name for name in node.input]
+ exit.append(helper.make_node('Cast', [half], [vi.name], name=f'{vi.name}__to_fp32', to=TensorProto.FLOAT))
+
+ nodes = entry + list(graph.node) + exit
+ del graph.node[:]
+ graph.node.extend(nodes)
+ del graph.value_info[:]
+ onnx.checker.check_model(model)
+ return model
+
+
+def to_half(array: np.ndarray) -> np.ndarray:
+ return np.clip(array, -65504, 65504).astype(np.float16)
+
+
+def main(out: Path) -> None:
+ (out / 'onnx').mkdir(parents=True, exist_ok=True)
+ fetch = lambda path: hf_hub_download(SOURCE_REPO, path, revision=SOURCE_REVISION)
+
+ shutil.copy(fetch('constants.json'), out / 'constants.json')
+ for name in GRAPHS:
+ model = onnx.load(fetch(f'onnx/{name}.onnx'))
+ if name not in UNFOLDED:
+ model = onnxslim.slim(model, size_threshold=FOLD_LIMIT_BYTES)
+ onnx.save(to_fp16(model), out / 'onnx' / f'{name}.onnx')
+ print(f'{name}: {(out / "onnx" / f"{name}.onnx").stat().st_size:,} bytes')
+
+
+if __name__ == '__main__':
+ main(Path(sys.argv[1]))
diff --git a/packages/sam2/src/constants.ts b/packages/sam2/src/constants.ts
index 36550a45..a2ee6eff 100644
--- a/packages/sam2/src/constants.ts
+++ b/packages/sam2/src/constants.ts
@@ -2,15 +2,20 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-export const MODEL_REPO = 'https://huggingface.co/square-zero-labs/sam2.1-tiny-video-onnx/resolve/main';
+/**
+ * The graphs compute in fp16 but take and return fp32, so the pipeline is the
+ * same as for the fp32 export they are built from; `scripts/convert_fp16.py`
+ * makes them. Pinned to a commit, which is part of the cache key.
+ */
+export const MODEL_REPO = 'https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/9a1ecaa5b194cc9c5bd7840036301037fe3f6b13';
export const MODEL_FILES = {
constants: { path: 'constants.json', size: 9_922 },
- visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 134_335_567 },
- maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 17_794_355 },
- memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 5_616_496 },
- memoryAttention: { path: 'onnx/memory_attention.onnx', size: 32_259_165 },
- pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 67_289 },
+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 67_221_881 },
+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_928_987 },
+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_822_478 },
+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 16_220_704 },
+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_236 },
} as const;
export const IMAGE_SIZE = 1024;
diff --git a/packages/sam2/src/weights.ts b/packages/sam2/src/weights.ts
index c3d9c9d7..56092ed8 100644
--- a/packages/sam2/src/weights.ts
+++ b/packages/sam2/src/weights.ts
@@ -23,6 +23,7 @@ export async function fetchModelFiles(
signal?: AbortSignal,
): Promise<ModelFiles> {
const store = await openFileStore();
+ await store?.retain(Object.values(MODEL_FILES).map((file) => `${MODEL_REPO}/${file.path}`));
const loaded = new Map<ModelFileKey, number>();
const report = () => onProgress?.({ loaded: [...loaded.values()].reduce((a, b) => a + b, 0), total: TOTAL_BYTES });
@@ -73,6 +74,8 @@ async function download(url: string, signal: AbortSignal | undefined, onBytes: (
type FileStore = {
read(key: string): Promise<Blob | undefined>;
write(key: string, blob: Blob): Promise<void>;
+ /** Deletes every file but these, such as those of a model revision no longer loaded. */
+ retain(keys: string[]): Promise<void>;
};
async function openFileStore(): Promise<FileStore | null> {
@@ -97,6 +100,11 @@ async function openFileStore(): Promise<FileStore | null> {
read: (key) => run<Blob | undefined>('readonly', (store) => store.get(key)),
// A cache that cannot be written (quota, private mode) is not a reason to fail the load.
write: (key, blob) => run('readwrite', (store) => store.put(blob, key)).then(() => undefined, () => undefined),
+ // Like a failed write, a failed cleanup only costs disk space.
+ retain: (keys) =>
+ run('readonly', (store) => store.getAllKeys())
+ .then((all) => Promise.all(all.filter((key) => !keys.includes(String(key))).map((key) => run('readwrite', (store) => store.delete(key)))))
+ .then(() => undefined, () => undefined),
};
} catch {
return null;