Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance model loading and session creation for SAM 2.1

konstantin-paulus37f6195ba022
diff --git a/packages/sam2/src/load.ts b/packages/sam2/src/load.ts
index a6b90db1..42a7f71a 100644
--- a/packages/sam2/src/load.ts
+++ b/packages/sam2/src/load.ts
@@ -3,6 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { IMAGE_SIZE, MEMORY_FRAMES } from './constants';
+import { readFloat16Initializer } from './onnx';
import { configureRuntime, createSession, runtimeDevice } from './sessions';
import { Sam2Video } from './tracker';
import { fetchModelFiles } from './weights';
@@ -41,12 +42,14 @@ async function load({ onProgress, signal }: Sam2LoadOptions): Promise<Sam2Video>
onProgress?.({ phase: 'compile' });
// The runtime builds one WebGPU session at a time.
- const visionEncoder = await createSession(files.visionEncoder, ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed']);
+ const visionEncoder = await createSession(files.visionEncoder, ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed'], true);
const maskDecoder = await createSession(files.maskDecoder, ['high_res_mask', 'object_pointer']);
const memoryEncoder = await createSession(files.memoryEncoder, ['memory_tokens', 'memory_pos']);
- const memoryAttention = await createSession(files.memoryAttention, ['conditioned_feats']);
+ const memoryAttention = await createSession(files.memoryAttention, ['conditioned_feats'], { enableGraphCapture: true });
const pointerTpos = await createSession(files.pointerTpos, ['pointer_pos']);
const device = await runtimeDevice();
- return new Sam2Video(device, { visionEncoder, maskDecoder, memoryEncoder, memoryAttention, pointerTpos }, constants);
+ const positions = readFloat16Initializer(files.visionEncoder, 'pos_embed');
+
+ return new Sam2Video(device, { visionEncoder, maskDecoder, memoryEncoder, memoryAttention, pointerTpos }, constants, positions);
}
diff --git a/packages/sam2/src/onnx.ts b/packages/sam2/src/onnx.ts
new file mode 100644
index 00000000..67741af6
--- /dev/null
+++ b/packages/sam2/src/onnx.ts
@@ -0,0 +1,97 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/** Protobuf field numbers on the path to a stored tensor: ModelProto.graph, GraphProto.initializer, TensorProto.{data_type, name, raw_data}. */
+const MODEL_GRAPH = 7;
+const GRAPH_INITIALIZER = 5;
+const TENSOR_DATA_TYPE = 2;
+const TENSOR_NAME = 8;
+const TENSOR_RAW_DATA = 9;
+const FLOAT16 = 10;
+
+const VARINT = 0;
+const FIXED64 = 1;
+const LENGTH_DELIMITED = 2;
+const FIXED32 = 5;
+
+type Field = { number: number; value: number; bytes: Uint8Array };
+
+/**
+ * Reads a float16 initializer out of a model's bytes, as float32: enough of
+ * the protobuf to walk to it, and nothing else.
+ */
+export function readFloat16Initializer(model: ArrayBuffer, name: string): Float32Array {
+ const graph = [...fields(new Uint8Array(model))].find((field) => field.number === MODEL_GRAPH);
+ if (!graph) throw new Error('The model has no graph');
+
+ for (const field of fields(graph.bytes)) {
+ if (field.number !== GRAPH_INITIALIZER) continue;
+
+ let tensorName = '';
+ let dataType = 0;
+ let raw: Uint8Array | null = null;
+ for (const part of fields(field.bytes)) {
+ if (part.number === TENSOR_NAME) tensorName = new TextDecoder().decode(part.bytes);
+ else if (part.number === TENSOR_DATA_TYPE) dataType = part.value;
+ else if (part.number === TENSOR_RAW_DATA) raw = part.bytes;
+ }
+ if (tensorName !== name) continue;
+ if (dataType !== FLOAT16 || !raw) throw new Error(`The initializer ${name} is not stored as raw float16`);
+
+ const halves = new DataView(raw.buffer, raw.byteOffset, raw.byteLength);
+ const floats = new Float32Array(raw.byteLength / 2);
+ for (let i = 0; i < floats.length; i++) floats[i] = halfToFloat(halves.getUint16(i * 2, true));
+ return floats;
+ }
+ throw new Error(`The model has no initializer ${name}`);
+}
+
+function* fields(bytes: Uint8Array): Generator<Field> {
+ let offset = 0;
+ const varint = (): number => {
+ let value = 0;
+ let scale = 1;
+ for (;;) {
+ const byte = bytes[offset++]!;
+ value += (byte & 0x7f) * scale;
+ if (byte < 0x80) return value;
+ scale *= 0x80;
+ }
+ };
+
+ while (offset < bytes.length) {
+ const key = varint();
+ const number = Math.floor(key / 8);
+ switch (key % 8) {
+ case VARINT:
+ yield { number, value: varint(), bytes: EMPTY };
+ break;
+ case FIXED64:
+ offset += 8;
+ break;
+ case LENGTH_DELIMITED: {
+ const length = varint();
+ yield { number, value: length, bytes: bytes.subarray(offset, offset + length) };
+ offset += length;
+ break;
+ }
+ case FIXED32:
+ offset += 4;
+ break;
+ default:
+ throw new Error('The model is not a valid ONNX file');
+ }
+ }
+}
+
+const EMPTY = new Uint8Array(0);
+
+function halfToFloat(half: number): number {
+ const sign = half & 0x8000 ? -1 : 1;
+ const exponent = (half >> 10) & 0x1f;
+ const fraction = half & 0x3ff;
+ if (exponent === 0) return sign * fraction * 2 ** -24;
+ if (exponent === 0x1f) return fraction ? NaN : sign * Infinity;
+ return sign * (1 + fraction / 1024) * 2 ** (exponent - 15);
+}
diff --git a/packages/sam2/src/sessions.ts b/packages/sam2/src/sessions.ts
index 29547cd1..2b1ca9ca 100644
--- a/packages/sam2/src/sessions.ts
+++ b/packages/sam2/src/sessions.ts
@@ -21,11 +21,13 @@ export function runtimeDevice(): Promise<GPUDevice> {
}
/** A WebGPU session whose listed outputs stay on the GPU; the rest are downloaded after each run. */
-export function createSession(model: ArrayBuffer, gpuOutputs: readonly string[]): Promise<Session> {
+export function createSession(model: ArrayBuffer, gpuOutputs: readonly string[], options: Partial<ort.InferenceSession.SessionOptions> = {}): Promise<Session> {
return ort.InferenceSession.create(model, {
- executionProviders: ['webgpu'],
+
+ executionProviders: [{ name: 'webgpu' }],
graphOptimizationLevel: 'all',
preferredOutputLocation: Object.fromEntries(gpuOutputs.map((name) => [name, 'gpu-buffer' as const])),
+ ...options,
});
}
diff --git a/packages/sam2/src/tracker.ts b/packages/sam2/src/tracker.ts
index fa1558b7..49efae6a 100644
--- a/packages/sam2/src/tracker.ts
+++ b/packages/sam2/src/tracker.ts
@@ -43,16 +43,19 @@ export class Sam2Video {
/** The frame kept encoded for `preview` and `seedHeld`, so a prompt costs the decoder alone. */
private held: Outputs | null = null;
+ /** `positions` is the vision encoder's `pos_embed`, [HIDDEN_DIM, FEAT_TOKENS]. */
constructor(
device: GPUDevice,
private readonly sessions: Sam2Sessions,
constants: ModelConstants,
+ positions: Float32Array,
) {
this.preprocessor = new FramePreprocessor(device, constants.image_mean, constants.image_std);
this.transposer = new TokenTransposer(device);
this.bank = new MemoryBank(device, constants.memory_temporal_positional_encoding);
this.featureTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 vision tokens');
this.positionTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 position tokens');
+ device.queue.writeBuffer(this.positionTokens, 0, toTokens(positions));
}
/** Segments the object at `points` and makes this the frame every later one is tracked from. */
@@ -68,12 +71,10 @@ export class Sam2Video {
/**
* Encodes a frame and keeps it hot: `preview` and `seedHeld` then run
* the mask decoder alone, which is what makes prompting feel immediate.
- * The frame held before is dropped.
+ * It stays held until another frame is encoded, by this or `seed` or `track`.
*/
async hold(frame: VideoFrame, rotation: Rotation): Promise<void> {
- const vision = await this.encode(frame, rotation);
- this.release();
- this.held = vision;
+ this.held = await this.encode(frame, rotation);
}
get holding(): boolean {
@@ -120,7 +121,6 @@ export class Sam2Video {
const vision = await this.encode(frame, rotation);
try {
this.transposer.run(gpuBuffer(vision, 'feats2'), this.featureTokens);
- this.transposer.run(gpuBuffer(vision, 'vision_pos_embed'), this.positionTokens);
const distances = this.bank.assemble(index, totalFrames);
const temporal = await this.sessions.pointerTpos.run({ normalized_diffs: floatTensor(distances, [MAX_POINTERS]) });
@@ -161,11 +161,14 @@ export class Sam2Video {
for (const session of Object.values(this.sessions)) void session.release();
}
+ /** The encoder is captured, so its outputs are one set of buffers: encoding a frame overwrites the one held. */
private async encode(frame: VideoFrame, rotation: Rotation): Promise<Outputs> {
+ this.release();
this.preprocessor.run(frame, rotation);
- return this.sessions.visionEncoder.run({
- pixel_values: gpuTensor(this.preprocessor.pixels, [1, 3, IMAGE_SIZE, IMAGE_SIZE]),
- });
+ return this.sessions.visionEncoder.run(
+ { pixel_values: gpuTensor(this.preprocessor.pixels, [1, 3, IMAGE_SIZE, IMAGE_SIZE]) },
+ VISION_OUTPUTS,
+ );
}
/**
@@ -220,6 +223,22 @@ export class Sam2Video {
}
}
+/**
+ * What the pipeline reads of the vision encoder. Its `vision_pos_embed` is a
+ * constant the runtime folds away, and a captured graph does not write folded
+ * outputs; `positionTokens` holds it instead, read from the model's weights.
+ */
+const VISION_OUTPUTS = ['feats0', 'feats1', 'feats2', 'feats2_no_mem'];
+
+/** Rewrites [C, N] channels as the [N, 1, C] tokens memory attention reads. */
+function toTokens(channels: Float32Array): Float32Array<ArrayBuffer> {
+ const tokens = new Float32Array(channels.length);
+ for (let c = 0; c < HIDDEN_DIM; c++) {
+ for (let n = 0; n < FEAT_TOKENS; n++) tokens[n * HIDDEN_DIM + c] = channels[c * FEAT_TOKENS + n]!;
+ }
+ return tokens;
+}
+
/**
* Tracked frames enter memory only when the model sees the object and trusts
* its mask, as SAMURAI selects them. Around a cut or a lookalike the