Skip to content

Code-first video & image, rendered natively on WebGPU.

8.2 KB
Raw Blame
/** SAM 3's small prompt/position heads, assembled from the pinned upstream weights.
 * The video export omits dense prompts and learned temporal constants. These heads
 * restore the original operations, without approximating a semantic mask by clicks.
 */
import {
	concat,
	encoder,
	fields,
	firstString,
	lengthDelimited as ld,
	readVarint,
	strings,
	varint,
} from "@framefields/vision";

function integer(field: number, value: number): Uint8Array {
	return new Uint8Array([...varint(field * 8), ...varint(value)]);
}
function text(field: number, value: string): Uint8Array {
	return ld(field, encoder.encode(value));
}
function attribute(name: string, values: readonly number[]): Uint8Array {
	return ld(
		5,
		concat([
			text(1, name),
			integer(20, 7),
			ld(8, new Uint8Array(values.flatMap(varint))),
		]),
	);
}
function intAttribute(name: string, value: number): Uint8Array {
	return ld(5, concat([text(1, name), integer(3, value), integer(20, 2)]));
}
function floatAttribute(name: string, value: number): Uint8Array {
	const bytes = new Uint8Array(4);
	new DataView(bytes.buffer).setFloat32(0, value, true);
	return ld(
		5,
		concat([text(1, name), new Uint8Array([21, ...bytes]), integer(20, 1)]),
	);
}
function node(
	op: string,
	inputs: readonly string[],
	output: string,
	attrs: readonly Uint8Array[] = [],
): Uint8Array {
	return ld(
		1,
		concat([
			...inputs.map((x) => text(1, x)),
			text(2, output),
			text(4, op),
			...attrs,
		]),
	);
}
function value(name: string, dims: readonly number[]): Uint8Array {
	const shape = concat(dims.map((d) => ld(1, integer(1, d))));
	return concat([
		text(1, name),
		ld(2, ld(1, concat([integer(1, 1), ld(2, shape)]))),
	]);
}
function scalar(name: string, amount: number): Uint8Array {
	const data = new Float32Array([amount]);
	return concat([
		integer(2, 1),
		text(8, name),
		ld(9, new Uint8Array(data.buffer)),
	]);
}
function model(
	nodes: readonly Uint8Array[],
	inputs: readonly [string, readonly number[]][],
	outputs: readonly [string, readonly number[]][],
	weights: readonly Uint8Array[],
): Uint8Array {
	const graph = concat([
		...nodes,
		text(2, "framefields_sam3"),
		...weights.map((w) => ld(5, w)),
		...inputs.map(([n, d]) => ld(11, value(n, d))),
		...outputs.map(([n, d]) => ld(12, value(n, d))),
	]);
	return concat([integer(1, 9), ld(7, graph), ld(8, integer(2, 18))]);
}
interface Weight {
	readonly proto: Uint8Array;
	readonly data: Float32Array;
}
function weightsFrom(
	modelBytes: Uint8Array,
	external: Uint8Array,
): Map<string, Weight> {
	const graph = fields(modelBytes).find((f) => f.number === 7);
	if (!graph) throw new Error("SAM 3 constants graph is missing.");
	const result = new Map<string, Weight>();
	for (const entry of fields(graph.body).filter((f) => f.number === 5)) {
		const fs = fields(entry.body),
			name = firstString(entry.body, 8);
		if (
			!name ||
			!/tracker\.(maskmem_tpos_enc|no_mem_embed|obj_ptr_tpos_proj|sam_prompt_encoder\.mask_downscaling)/.test(
				name,
			)
		)
			continue;
		const metadata = new Map(
			fs
				.filter((f) => f.number === 13)
				.map((f) => [firstString(f.body, 1), firstString(f.body, 2)]),
		);
		const offset = Number(metadata.get("offset") ?? 0),
			length = Number(metadata.get("length") ?? 0);
		const raw =
			fs.find((f) => f.number === 9)?.body ??
			external.subarray(offset, offset + length);
		const dataType = fs.find((f) => f.number === 2);
		const dtype = dataType ? readVarint(dataType.raw, 1)[0] : 0;
		if (![1, 10].includes(dtype))
			throw new Error(`Unsupported SAM 3 weight type: ${name}`);
		if (raw.length === 0 || raw.length % (dtype === 10 ? 2 : 4) !== 0)
			throw new Error(`Invalid SAM 3 weight: ${name}`);
		const view = new DataView(raw.buffer, raw.byteOffset, raw.byteLength);
		const data = Float32Array.from(
			{ length: raw.length / (dtype === 10 ? 2 : 4) },
			(_, i) => {
				if (dtype === 1) return view.getFloat32(i * 4, true);
				const bits = view.getUint16(i * 2, true),
					sign = bits & 0x8000 ? -1 : 1,
					exponent = (bits >> 10) & 31,
					fraction = bits & 1023;
				if (exponent === 31) throw new Error(`Non-finite SAM 3 weight ${name}`);
				return (
					sign *
					(exponent === 0
						? fraction * 2 ** -24
						: (1 + fraction / 1024) * 2 ** (exponent - 15))
				);
			},
		);
		const copy = new Uint8Array(data.buffer);
		result.set(name, {
			proto: concat([
				...fs
					.filter((f) => ![2, 9, 13, 14].includes(f.number))
					.map((f) => f.raw),
				integer(2, 1),
				ld(9, copy),
			]),
			data: new Float32Array(copy.buffer),
		});
	}
	return result;
}
export function sam3SupportGraphs(
	reference: Uint8Array,
	external: Uint8Array,
): {
	densePrompt: Uint8Array;
	pointerPosition: Uint8Array;
	temporal: Float32Array;
	noMemory: Float32Array;
} {
	const weights = weightsFrom(reference, external);
	const get = (name: string): Weight => {
		const item = weights.get(`tracker.${name}`);
		if (!item) throw new Error(`Missing SAM 3 weight ${name}`);
		return item;
	};
	const nodes: Uint8Array[] = [],
		denseWeights: Uint8Array[] = [
			scalar("sqrt2", Math.SQRT2),
			scalar("one", 1),
			scalar("half", 0.5),
		];
	let previous = "mask";
	for (const [index, layer] of [0, 3, 6].entries()) {
		const base = `sam_prompt_encoder.mask_downscaling.${layer}`;
		for (const suffix of ["weight", "bias"])
			denseWeights.push(get(`${base}.${suffix}`).proto);
		const output = `conv${index}`;
		nodes.push(
			node(
				"Conv",
				[previous, `tracker.${base}.weight`, `tracker.${base}.bias`],
				output,
				[
					attribute("kernel_shape", layer === 6 ? [1, 1] : [2, 2]),
					attribute("strides", layer === 6 ? [1, 1] : [2, 2]),
				],
			),
		);
		previous = output;
		if (layer === 6) continue;
		const norm = `sam_prompt_encoder.mask_downscaling.${layer + 1}`;
		for (const suffix of ["weight", "bias"])
			denseWeights.push(get(`${norm}.${suffix}`).proto);
		nodes.push(
			node("Transpose", [previous], `nhwc${index}`, [
				attribute("perm", [0, 2, 3, 1]),
			]),
			node(
				"LayerNormalization",
				[`nhwc${index}`, `tracker.${norm}.weight`, `tracker.${norm}.bias`],
				`norm${index}`,
				[intAttribute("axis", 3), floatAttribute("epsilon", 1e-6)],
			),
			node("Transpose", [`norm${index}`], `nchw${index}`, [
				attribute("perm", [0, 3, 1, 2]),
			]),
			node("Div", [`nchw${index}`, "sqrt2"], `div${index}`),
			node("Erf", [`div${index}`], `erf${index}`),
			node("Add", [`erf${index}`, "one"], `add${index}`),
			node("Mul", [`add${index}`, `nchw${index}`], `mul${index}`),
			node("Mul", [`mul${index}`, "half"], `gelu${index}`),
		);
		previous = `gelu${index}`;
	}
	nodes.push(node("Identity", [previous], "dense_prompt"));
	const pointerWeights = [
		get("obj_ptr_tpos_proj.weight").proto,
		get("obj_ptr_tpos_proj.bias").proto,
	];
	return {
		densePrompt: model(
			nodes,
			[["mask", [1, 1, 288, 288]]],
			[["dense_prompt", [1, 256, 72, 72]]],
			denseWeights,
		),
		pointerPosition: model(
			[
				node(
					"Gemm",
					[
						"encoding",
						"tracker.obj_ptr_tpos_proj.weight",
						"tracker.obj_ptr_tpos_proj.bias",
					],
					"position",
					[intAttribute("transB", 1)],
				),
			],
			[["encoding", [16, 256]]],
			[["position", [16, 64]]],
			pointerWeights,
		),
		temporal: get("maskmem_tpos_enc").data,
		noMemory: get("no_mem_embed").data,
	};
}

/** Replace the no-mask embedding by the actual dense prompt head's output. */
export function sam3DenseDecoder(modelBytes: Uint8Array): Uint8Array {
	const modelFields = fields(modelBytes),
		graph = modelFields.find((f) => f.number === 7);
	if (!graph) throw new Error("Missing SAM 3 decoder graph.");
	let replaced = false;
	const graphParts = fields(graph.body).map((f) => {
		if (f.number !== 1 || firstString(f.body, 3) !== "/mask_decoder/Add")
			return f.raw;
		const inputs = strings(f.body, 1);
		if (
			inputs[0] !== "feats2" ||
			inputs[1] !== "/prompt_encoder/Expand_output_0"
		)
			throw new Error("Unexpected SAM 3 dense prompt contract.");
		replaced = true;
		return ld(
			1,
			concat(
				fields(f.body)
					.filter((x) => x.number !== 1)
					.map((x) => x.raw)
					.concat([text(1, "feats2"), text(1, "dense_prompt")]),
			),
		);
	});
	if (!replaced)
		throw new Error("SAM 3 decoder has no prompt embedding input.");
	graphParts.push(ld(11, value("dense_prompt", [1, 256, 72, 72])));
	return concat(
		modelFields.map((f) => (f === graph ? ld(7, concat(graphParts)) : f.raw)),
	);
}