Skip to content

Code-first video & image, rendered natively on WebGPU.

2.5 KB
Raw Blame
import type { Sam2Memory } from "../sam2/memory.js";

/** Dynamic memory lengths: never duplicate unavailable early-frame memories. */
export class Sam3MemoryBank {
	private seed?: Sam2Memory;
	private recent: Sam2Memory[] = [];
	constructor(private readonly temporal: Float32Array) {}
	remember(memory: Sam2Memory, prompted: boolean): void {
		if (prompted) {
			this.seed = memory;
			this.recent = [];
		} else {
			this.recent.push(memory);
			if (this.recent.length > 15) this.recent.shift();
		}
	}
	assemble(
		frame: number,
		frameCount: number,
	): {
		spatial: Float32Array;
		positions: Float32Array;
		pointers: Float32Array;
		encoding: Float32Array;
		pointerCount: number;
	} {
		if (!this.seed) throw new Error("SAM 3 needs a prompt before propagation.");
		const blocks = [this.seed, ...this.recent.slice(-6)];
		const spatial = new Float32Array(blocks.length * 5184 * 64),
			positions = new Float32Array(spatial.length);
		for (const [i, block] of blocks.entries()) {
			spatial.set(block.tokens, i * 5184 * 64);
			const row =
				block === this.seed ? 6 : Math.min(5, frame - block.frame - 1);
			for (let j = 0; j < block.positions.length; j++)
				positions[i * 5184 * 64 + j] =
					(block.positions[j] ?? 0) + (this.temporal[row * 64 + (j % 64)] ?? 0);
		}
		const entries = [this.seed, ...[...this.recent].reverse()].slice(0, 16);
		const pointers = new Float32Array(entries.length * 256),
			encoding = new Float32Array(16 * 256);
		for (const [i, entry] of entries.entries()) {
			pointers.set(entry.pointer, i * 256);
			const diff =
				(frame - entry.frame) / Math.max(1, Math.min(frameCount, 16) - 1);
			for (let j = 0; j < 128; j++) {
				const angle = diff / 10000 ** ((2 * Math.floor(j / 2)) / 128);
				encoding[i * 256 + j] = Math.sin(angle);
				encoding[i * 256 + 128 + j] = Math.cos(angle);
			}
		}
		return {
			spatial,
			positions,
			pointers,
			encoding,
			pointerCount: entries.length,
		};
	}
}

/** SAM 3 PositionEmbeddingSine: y then x, normalized pixel centers at grid indices 1..72. */
export function sam3Position(): Float32Array {
	const out = new Float32Array(5184 * 256);
	for (let y = 0; y < 72; y++)
		for (let x = 0; x < 72; x++)
			for (let j = 0; j < 128; j++) {
				const divisor = 10000 ** ((2 * Math.floor(j / 2)) / 128);
				const yy = (((y + 1) / (72 + 1e-6)) * 2 * Math.PI) / divisor,
					xx = (((x + 1) / (72 + 1e-6)) * 2 * Math.PI) / divisor;
				out[(y * 72 + x) * 256 + j] = j % 2 ? Math.cos(yy) : Math.sin(yy);
				out[(y * 72 + x) * 256 + 128 + j] = j % 2 ? Math.cos(xx) : Math.sin(xx);
			}
	return out;
}