The VS Code of video editing, built for agents.
TypeScript
Enhance model loading and session creation for SAM 2.1
konstantin-paulus37f6195ba022
diff --git a/packages/sam2/src/load.ts b/packages/sam2/src/load.tsindex a6b90db1..42a7f71a 100644--- a/packages/sam2/src/load.ts+++ b/packages/sam2/src/load.ts@@ -3,6 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { IMAGE_SIZE, MEMORY_FRAMES } from './constants';+import { readFloat16Initializer } from './onnx';import { configureRuntime, createSession, runtimeDevice } from './sessions';import { Sam2Video } from './tracker';import { fetchModelFiles } from './weights';@@ -41,12 +42,14 @@ async function load({ onProgress, signal }: Sam2LoadOptions): Promise<Sam2Video>onProgress?.({ phase: 'compile' });// The runtime builds one WebGPU session at a time.- const visionEncoder = await createSession(files.visionEncoder, ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed']);+ const visionEncoder = await createSession(files.visionEncoder, ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed'], true);const maskDecoder = await createSession(files.maskDecoder, ['high_res_mask', 'object_pointer']);const memoryEncoder = await createSession(files.memoryEncoder, ['memory_tokens', 'memory_pos']);- const memoryAttention = await createSession(files.memoryAttention, ['conditioned_feats']);+ const memoryAttention = await createSession(files.memoryAttention, ['conditioned_feats'], { enableGraphCapture: true });const pointerTpos = await createSession(files.pointerTpos, ['pointer_pos']);const device = await runtimeDevice();- return new Sam2Video(device, { visionEncoder, maskDecoder, memoryEncoder, memoryAttention, pointerTpos }, constants);+ const positions = readFloat16Initializer(files.visionEncoder, 'pos_embed');++ return new Sam2Video(device, { visionEncoder, maskDecoder, memoryEncoder, memoryAttention, pointerTpos }, constants, positions);}diff --git a/packages/sam2/src/onnx.ts b/packages/sam2/src/onnx.tsnew file mode 100644index 00000000..67741af6--- /dev/null+++ b/packages/sam2/src/onnx.ts@@ -0,0 +1,97 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/** Protobuf field numbers on the path to a stored tensor: ModelProto.graph, GraphProto.initializer, TensorProto.{data_type, name, raw_data}. */+const MODEL_GRAPH = 7;+const GRAPH_INITIALIZER = 5;+const TENSOR_DATA_TYPE = 2;+const TENSOR_NAME = 8;+const TENSOR_RAW_DATA = 9;+const FLOAT16 = 10;++const VARINT = 0;+const FIXED64 = 1;+const LENGTH_DELIMITED = 2;+const FIXED32 = 5;++type Field = { number: number; value: number; bytes: Uint8Array };++/**+ * Reads a float16 initializer out of a model's bytes, as float32: enough of+ * the protobuf to walk to it, and nothing else.+ */+export function readFloat16Initializer(model: ArrayBuffer, name: string): Float32Array {+ const graph = [...fields(new Uint8Array(model))].find((field) => field.number === MODEL_GRAPH);+ if (!graph) throw new Error('The model has no graph');++ for (const field of fields(graph.bytes)) {+ if (field.number !== GRAPH_INITIALIZER) continue;++ let tensorName = '';+ let dataType = 0;+ let raw: Uint8Array | null = null;+ for (const part of fields(field.bytes)) {+ if (part.number === TENSOR_NAME) tensorName = new TextDecoder().decode(part.bytes);+ else if (part.number === TENSOR_DATA_TYPE) dataType = part.value;+ else if (part.number === TENSOR_RAW_DATA) raw = part.bytes;+ }+ if (tensorName !== name) continue;+ if (dataType !== FLOAT16 || !raw) throw new Error(`The initializer ${name} is not stored as raw float16`);++ const halves = new DataView(raw.buffer, raw.byteOffset, raw.byteLength);+ const floats = new Float32Array(raw.byteLength / 2);+ for (let i = 0; i < floats.length; i++) floats[i] = halfToFloat(halves.getUint16(i * 2, true));+ return floats;+ }+ throw new Error(`The model has no initializer ${name}`);+}++function* fields(bytes: Uint8Array): Generator<Field> {+ let offset = 0;+ const varint = (): number => {+ let value = 0;+ let scale = 1;+ for (;;) {+ const byte = bytes[offset++]!;+ value += (byte & 0x7f) * scale;+ if (byte < 0x80) return value;+ scale *= 0x80;+ }+ };++ while (offset < bytes.length) {+ const key = varint();+ const number = Math.floor(key / 8);+ switch (key % 8) {+ case VARINT:+ yield { number, value: varint(), bytes: EMPTY };+ break;+ case FIXED64:+ offset += 8;+ break;+ case LENGTH_DELIMITED: {+ const length = varint();+ yield { number, value: length, bytes: bytes.subarray(offset, offset + length) };+ offset += length;+ break;+ }+ case FIXED32:+ offset += 4;+ break;+ default:+ throw new Error('The model is not a valid ONNX file');+ }+ }+}++const EMPTY = new Uint8Array(0);++function halfToFloat(half: number): number {+ const sign = half & 0x8000 ? -1 : 1;+ const exponent = (half >> 10) & 0x1f;+ const fraction = half & 0x3ff;+ if (exponent === 0) return sign * fraction * 2 ** -24;+ if (exponent === 0x1f) return fraction ? NaN : sign * Infinity;+ return sign * (1 + fraction / 1024) * 2 ** (exponent - 15);+}diff --git a/packages/sam2/src/sessions.ts b/packages/sam2/src/sessions.tsindex 29547cd1..2b1ca9ca 100644--- a/packages/sam2/src/sessions.ts+++ b/packages/sam2/src/sessions.ts@@ -21,11 +21,13 @@ export function runtimeDevice(): Promise<GPUDevice> {}/** A WebGPU session whose listed outputs stay on the GPU; the rest are downloaded after each run. */-export function createSession(model: ArrayBuffer, gpuOutputs: readonly string[]): Promise<Session> {+export function createSession(model: ArrayBuffer, gpuOutputs: readonly string[], options: Partial<ort.InferenceSession.SessionOptions> = {}): Promise<Session> {return ort.InferenceSession.create(model, {- executionProviders: ['webgpu'],++ executionProviders: [{ name: 'webgpu' }],graphOptimizationLevel: 'all',preferredOutputLocation: Object.fromEntries(gpuOutputs.map((name) => [name, 'gpu-buffer' as const])),+ ...options,});}diff --git a/packages/sam2/src/tracker.ts b/packages/sam2/src/tracker.tsindex fa1558b7..49efae6a 100644--- a/packages/sam2/src/tracker.ts+++ b/packages/sam2/src/tracker.ts@@ -43,16 +43,19 @@ export class Sam2Video {/** The frame kept encoded for `preview` and `seedHeld`, so a prompt costs the decoder alone. */private held: Outputs | null = null;+ /** `positions` is the vision encoder's `pos_embed`, [HIDDEN_DIM, FEAT_TOKENS]. */constructor(device: GPUDevice,private readonly sessions: Sam2Sessions,constants: ModelConstants,+ positions: Float32Array,) {this.preprocessor = new FramePreprocessor(device, constants.image_mean, constants.image_std);this.transposer = new TokenTransposer(device);this.bank = new MemoryBank(device, constants.memory_temporal_positional_encoding);this.featureTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 vision tokens');this.positionTokens = createStorageBuffer(device, FEAT_TOKENS * HIDDEN_DIM * BYTES_PER_FLOAT, 'sam2 position tokens');+ device.queue.writeBuffer(this.positionTokens, 0, toTokens(positions));}/** Segments the object at `points` and makes this the frame every later one is tracked from. */@@ -68,12 +71,10 @@ export class Sam2Video {/*** Encodes a frame and keeps it hot: `preview` and `seedHeld` then run* the mask decoder alone, which is what makes prompting feel immediate.- * The frame held before is dropped.+ * It stays held until another frame is encoded, by this or `seed` or `track`.*/async hold(frame: VideoFrame, rotation: Rotation): Promise<void> {- const vision = await this.encode(frame, rotation);- this.release();- this.held = vision;+ this.held = await this.encode(frame, rotation);}get holding(): boolean {@@ -120,7 +121,6 @@ export class Sam2Video {const vision = await this.encode(frame, rotation);try {this.transposer.run(gpuBuffer(vision, 'feats2'), this.featureTokens);- this.transposer.run(gpuBuffer(vision, 'vision_pos_embed'), this.positionTokens);const distances = this.bank.assemble(index, totalFrames);const temporal = await this.sessions.pointerTpos.run({ normalized_diffs: floatTensor(distances, [MAX_POINTERS]) });@@ -161,11 +161,14 @@ export class Sam2Video {for (const session of Object.values(this.sessions)) void session.release();}+ /** The encoder is captured, so its outputs are one set of buffers: encoding a frame overwrites the one held. */private async encode(frame: VideoFrame, rotation: Rotation): Promise<Outputs> {+ this.release();this.preprocessor.run(frame, rotation);- return this.sessions.visionEncoder.run({- pixel_values: gpuTensor(this.preprocessor.pixels, [1, 3, IMAGE_SIZE, IMAGE_SIZE]),- });+ return this.sessions.visionEncoder.run(+ { pixel_values: gpuTensor(this.preprocessor.pixels, [1, 3, IMAGE_SIZE, IMAGE_SIZE]) },+ VISION_OUTPUTS,+ );}/**@@ -220,6 +223,22 @@ export class Sam2Video {}}+/**+ * What the pipeline reads of the vision encoder. Its `vision_pos_embed` is a+ * constant the runtime folds away, and a captured graph does not write folded+ * outputs; `positionTokens` holds it instead, read from the model's weights.+ */+const VISION_OUTPUTS = ['feats0', 'feats1', 'feats2', 'feats2_no_mem'];++/** Rewrites [C, N] channels as the [N, 1, C] tokens memory attention reads. */+function toTokens(channels: Float32Array): Float32Array<ArrayBuffer> {+ const tokens = new Float32Array(channels.length);+ for (let c = 0; c < HIDDEN_DIM; c++) {+ for (let n = 0; n < FEAT_TOKENS; n++) tokens[n * HIDDEN_DIM + c] = channels[c * FEAT_TOKENS + n]!;+ }+ return tokens;+}+/*** Tracked frames enter memory only when the model sees the object and trusts* its mask, as SAMURAI selects them. Around a cut or a lookalike the