The VS Code of video editing, built for agents.
TypeScript
Add fp16 conversion script and update model repository links
konstantin-paulus6d275f1a619b
diff --git a/packages/sam2/scripts/convert_fp16.py b/packages/sam2/scripts/convert_fp16.pynew file mode 100644index 00000000..a665cc9e--- /dev/null+++ b/packages/sam2/scripts/convert_fp16.py@@ -0,0 +1,105 @@+# /// script+# requires-python = ">=3.10"+# dependencies = ["huggingface_hub", "numpy", "onnx==1.23.0", "onnxslim==0.1.96"]+# ///+"""+Builds the fp16 graphs the package loads from the fp32 SAM 2.1 video export.++Constants are folded in fp32 first, so positional encodings are computed at+full precision before they are stored as halves. The graphs then compute in+fp16 but keep their fp32 inputs and outputs, with a cast at each boundary, so+the pipeline feeding them is the same for either precision.++ uv run packages/sam2/scripts/convert_fp16.py <out-dir>+"""++import shutil+import sys+from pathlib import Path++import numpy as np+import onnx+import onnxslim+from huggingface_hub import hf_hub_download+from onnx import TensorProto, helper, numpy_helper++SOURCE_REPO = 'square-zero-labs/sam2.1-tiny-video-onnx'+SOURCE_REVISION = '3b2984dd865f6e9d2cc6aed0be6a5a5c2eb352ce'+GRAPHS = ['vision_encoder', 'mask_decoder', 'memory_encoder', 'memory_attention', 'pointer_tpos']++# Folding memory attention's RoPE would store the same table once per layer, doubling the download.+UNFOLDED = {'memory_attention'}+FOLD_LIMIT_BYTES = 8 << 20++# Inputs the ONNX spec types as float32 whatever the op computes in, by (op, input index).+FLOAT32_INPUTS = {('Resize', 1), ('Resize', 2)}+++def to_fp16(model: onnx.ModelProto) -> onnx.ModelProto:+ graph = model.graph+ keep_float = {+ node.input[i]+ for node in graph.node+ for op, i in FLOAT32_INPUTS+ if node.op_type == op and len(node.input) > i+ }++ for init in graph.initializer:+ if init.data_type == TensorProto.FLOAT and init.name not in keep_float:+ init.CopyFrom(numpy_helper.from_array(to_half(numpy_helper.to_array(init)), init.name))++ for node in graph.node:+ for attr in node.attribute:+ if node.op_type == 'Cast' and attr.name == 'to' and attr.i == TensorProto.FLOAT:+ attr.i = TensorProto.FLOAT16+ elif (+ attr.type == onnx.AttributeProto.TENSOR+ and attr.t.data_type == TensorProto.FLOAT+ and node.op_type in ('Constant', 'ConstantOfShape')+ and node.output[0] not in keep_float+ ):+ attr.t.CopyFrom(numpy_helper.from_array(to_half(numpy_helper.to_array(attr.t)), attr.t.name))++ # Float inputs are cast down on entry and float outputs cast back up on exit.+ entry, exit = [], []+ for vi in graph.input:+ if vi.type.tensor_type.elem_type == TensorProto.FLOAT:+ half = f'{vi.name}__fp16'+ for node in graph.node:+ node.input[:] = [half if name == vi.name else name for name in node.input]+ entry.append(helper.make_node('Cast', [vi.name], [half], name=f'{vi.name}__to_fp16', to=TensorProto.FLOAT16))+ for vi in graph.output:+ if vi.type.tensor_type.elem_type == TensorProto.FLOAT:+ half = f'{vi.name}__fp16'+ for node in graph.node:+ node.output[:] = [half if name == vi.name else name for name in node.output]+ node.input[:] = [half if name == vi.name else name for name in node.input]+ exit.append(helper.make_node('Cast', [half], [vi.name], name=f'{vi.name}__to_fp32', to=TensorProto.FLOAT))++ nodes = entry + list(graph.node) + exit+ del graph.node[:]+ graph.node.extend(nodes)+ del graph.value_info[:]+ onnx.checker.check_model(model)+ return model+++def to_half(array: np.ndarray) -> np.ndarray:+ return np.clip(array, -65504, 65504).astype(np.float16)+++def main(out: Path) -> None:+ (out / 'onnx').mkdir(parents=True, exist_ok=True)+ fetch = lambda path: hf_hub_download(SOURCE_REPO, path, revision=SOURCE_REVISION)++ shutil.copy(fetch('constants.json'), out / 'constants.json')+ for name in GRAPHS:+ model = onnx.load(fetch(f'onnx/{name}.onnx'))+ if name not in UNFOLDED:+ model = onnxslim.slim(model, size_threshold=FOLD_LIMIT_BYTES)+ onnx.save(to_fp16(model), out / 'onnx' / f'{name}.onnx')+ print(f'{name}: {(out / "onnx" / f"{name}.onnx").stat().st_size:,} bytes')+++if __name__ == '__main__':+ main(Path(sys.argv[1]))diff --git a/packages/sam2/src/constants.ts b/packages/sam2/src/constants.tsindex 36550a45..a2ee6eff 100644--- a/packages/sam2/src/constants.ts+++ b/packages/sam2/src/constants.ts@@ -2,15 +2,20 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-export const MODEL_REPO = 'https://huggingface.co/square-zero-labs/sam2.1-tiny-video-onnx/resolve/main';+/**+ * The graphs compute in fp16 but take and return fp32, so the pipeline is the+ * same as for the fp32 export they are built from; `scripts/convert_fp16.py`+ * makes them. Pinned to a commit, which is part of the cache key.+ */+export const MODEL_REPO = 'https://huggingface.co/diffusionstudio/sam2.1-tiny-video-onnx-fp16/resolve/9a1ecaa5b194cc9c5bd7840036301037fe3f6b13';export const MODEL_FILES = {constants: { path: 'constants.json', size: 9_922 },- visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 134_335_567 },- maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 17_794_355 },- memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 5_616_496 },- memoryAttention: { path: 'onnx/memory_attention.onnx', size: 32_259_165 },- pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 67_289 },+ visionEncoder: { path: 'onnx/vision_encoder.onnx', size: 67_221_881 },+ maskDecoder: { path: 'onnx/mask_decoder.onnx', size: 8_928_987 },+ memoryEncoder: { path: 'onnx/memory_encoder.onnx', size: 2_822_478 },+ memoryAttention: { path: 'onnx/memory_attention.onnx', size: 16_220_704 },+ pointerTpos: { path: 'onnx/pointer_tpos.onnx', size: 34_236 },} as const;export const IMAGE_SIZE = 1024;diff --git a/packages/sam2/src/weights.ts b/packages/sam2/src/weights.tsindex c3d9c9d7..56092ed8 100644--- a/packages/sam2/src/weights.ts+++ b/packages/sam2/src/weights.ts@@ -23,6 +23,7 @@ export async function fetchModelFiles(signal?: AbortSignal,): Promise<ModelFiles> {const store = await openFileStore();+ await store?.retain(Object.values(MODEL_FILES).map((file) => `${MODEL_REPO}/${file.path}`));const loaded = new Map<ModelFileKey, number>();const report = () => onProgress?.({ loaded: [...loaded.values()].reduce((a, b) => a + b, 0), total: TOTAL_BYTES });@@ -73,6 +74,8 @@ async function download(url: string, signal: AbortSignal | undefined, onBytes: (type FileStore = {read(key: string): Promise<Blob | undefined>;write(key: string, blob: Blob): Promise<void>;+ /** Deletes every file but these, such as those of a model revision no longer loaded. */+ retain(keys: string[]): Promise<void>;};async function openFileStore(): Promise<FileStore | null> {@@ -97,6 +100,11 @@ async function openFileStore(): Promise<FileStore | null> {read: (key) => run<Blob | undefined>('readonly', (store) => store.get(key)),// A cache that cannot be written (quota, private mode) is not a reason to fail the load.write: (key, blob) => run('readwrite', (store) => store.put(blob, key)).then(() => undefined, () => undefined),+ // Like a failed write, a failed cleanup only costs disk space.+ retain: (keys) =>+ run('readonly', (store) => store.getAllKeys())+ .then((all) => Promise.all(all.filter((key) => !keys.includes(String(key))).map((key) => run('readwrite', (store) => store.delete(key)))))+ .then(() => undefined, () => undefined),};} catch {return null;