Skip to content

Code-first video & image, rendered natively on WebGPU.

2.9 KB
Raw Blame
import type {
	SessionProvider,
	VisionModelStore,
	VisionTensorInput,
} from "@framefields/vision";
import { describe, expect, it } from "vitest";
import {
	ensureSamModelsRegistered,
	SAM_MODELS,
	SAM3_ENCODER,
} from "../models/registry.js";
import { holdSam3Encoder } from "./encoder.js";

function fakes() {
	const stats = { sessions: 0, runs: 0, released: 0, ensured: [] as string[] };
	const output = (n: number) => ({ data: new Float32Array(n), dims: [n] });
	const provider: SessionProvider = {
		kind: "webgpu",
		createSession: async () => {
			stats.sessions++;
			return {
				run: async () => ({}),
				runMany: async (_inputs: readonly VisionTensorInput[]) => {
					stats.runs++;
					return {
						"image_embeddings.0": output(4),
						"image_embeddings.1": output(4),
						"image_embeddings.2": output(4),
					};
				},
				release: () => {
					stats.released++;
				},
			};
		},
	};
	const store = {
		ensure: async (key: string) => {
			stats.ensured.push(key);
			return new Uint8Array(1);
		},
	} as unknown as VisionModelStore;
	return { provider, store, stats };
}

const frame = (value: number) => ({
	data: new Uint8Array(4 * 4 * 4).fill(value),
	width: 4,
	height: 4,
});

describe("shared SAM 3 encoder", () => {
	it("runs once per frame for every tracker on a provider, and frees with the last", async () => {
		const { provider, store, stats } = fakes();
		const [a, b] = await Promise.all([
			holdSam3Encoder(store, provider, "fp16"),
			holdSam3Encoder(store, provider, "fp16"),
		]);
		expect(stats.sessions).toBe(1);
		expect(stats.ensured).toEqual([
			"sam3-vision_encoder-fp16",
			"sam3-vision_encoder-fp16-data",
		]);
		// Separate decodes of one frame: equal pixels, different buffers.
		await Promise.all([a.encode(frame(7)), b.encode(frame(7))]);
		expect(stats.runs).toBe(1);
		await b.encode(frame(8));
		expect(stats.runs).toBe(2);
		a.release();
		a.release();
		expect(stats.released).toBe(0);
		b.release();
		expect(stats.released).toBe(1);
		await holdSam3Encoder(store, provider, "fp16");
		expect(stats.sessions).toBe(2);
	});

	it("keeps precisions apart", async () => {
		const { provider, store, stats } = fakes();
		await holdSam3Encoder(store, provider, "fp16");
		await holdSam3Encoder(store, provider, "q4");
		expect(stats.sessions).toBe(2);
	});
});

describe("SAM 3 encoder variants", () => {
	it("pins a graph and its weights for every precision a backend picks", () => {
		ensureSamModelsRegistered();
		const keys = new Set(SAM_MODELS.map((model) => model.key));
		for (const precision of Object.values(SAM3_ENCODER)) {
			expect(keys).toContain(`sam3-vision_encoder-${precision}`);
			expect(keys).toContain(`sam3-vision_encoder-${precision}-data`);
		}
		// q4f16 keeps the cache file names it was first downloaded under.
		expect(
			SAM_MODELS.find((m) => m.key === "sam3-vision_encoder-q4f16")?.filename,
		).toBe("sam3-vision_encoder_q4f16.onnx");
	});
});