Skip to content

Code-first video & image, rendered natively on WebGPU.

14.2 KB
Raw Blame
/**
 * Real-model end-to-end checks — opt-in, because they download ~380 MB of weights:
 *
 *   pnpm --filter @framefields/vision test:models
 *
 * Verifies every pinned model downloads, passes its size + SHA-256 check and matches the
 * registry's input shape, then runs each size on reference frames from `examples/` and checks
 * behaviour that regressed during development (subject selection, merging, pose duplicates).
 * Models are cached in `$FRAMEFIELDS_MODELS_DIR` (default `~/.cache/framefields/models`).
 */

import { fileURLToPath } from "node:url";
import { Canvas, loadImage } from "skia-canvas";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { imageToTensor, preprocessSpec } from "./decode/preprocess.js";
import { decodeRtmdetIns } from "./decode/rtmdet-ins.js";
import { FrameTensorPipeline } from "./gpu/frame-tensor.js";
import { needsNchwLayout } from "./model/conv-layout.js";
import { VisionModelStore } from "./model/model-store.js";
import {
	COCO_CLASSES,
	VISION_MODELS,
	VISION_VARIANTS,
	type VisionModelKey,
} from "./model/registry.js";
import { VisionRunner } from "./runner/vision-runner.js";
import { NodeWebGPUProvider } from "./runtime/node-webgpu-provider.js";
import { mergeSubjectMask } from "./segmentation/subject.js";
import type { VisionImageInput } from "./types.js";

const enabled = process.env.FRAMEFIELDS_VISION_E2E === "1";
const EXAMPLES = fileURLToPath(new URL("../../../examples/", import.meta.url));
const ASSETS = {
	dancer: "19_framefields_film/assets/dancer.png",
	portrait: "19_framefields_film/assets/portrait.png",
	ink: "19_framefields_film/assets/ink.png",
	crema: "21_full_circle/assets/crema.png",
	eclipse: "21_full_circle/assets/eclipse.png",
	spotlight: "21_full_circle/assets/spotlight.png",
	tunnel: "21_full_circle/assets/tunnel.png",
} as const;

async function loadFrame(path: string): Promise<VisionImageInput> {
	const image = await loadImage(EXAMPLES + path);
	const canvas = new Canvas(image.width, image.height);
	const ctx = canvas.getContext("2d");
	ctx.drawImage(image, 0, 0);
	return {
		data: ctx.getImageData(0, 0, image.width, image.height).data,
		width: image.width,
		height: image.height,
	};
}

describe.skipIf(!enabled)("vision models (real weights)", () => {
	const frames = {} as Record<keyof typeof ASSETS, VisionImageInput>;

	beforeAll(async () => {
		for (const [name, path] of Object.entries(ASSETS)) {
			frames[name as keyof typeof ASSETS] = await loadFrame(path);
		}
	});

	it.each(
		Object.keys(VISION_MODELS) as VisionModelKey[],
	)("%s downloads, verifies and matches its registry input shape", async (key) => {
		const bytes = await new VisionModelStore().ensure(key);
		const ort = await import("onnxruntime-node");
		const session = await ort.InferenceSession.create(bytes);
		const meta = session.inputMetadata[0];
		const shape = meta?.isTensor ? meta.shape : [];
		const [w, h] = VISION_MODELS[key].input;
		const spatial = shape.slice(2);
		// Graphs with named spatial dims take any size; `input` is then the runner's choice.
		if (spatial.every((d) => typeof d === "string"))
			expect(spatial).toHaveLength(2);
		else expect(spatial).toEqual([h, w]);
		await session.release();
	}, 600_000);

	it("runs only RTMO in NCHW (its grouped convs break the NHWC kernels)", async () => {
		const store = new VisionModelStore();
		expect(needsNchwLayout(await store.ensure("rtmo-s"))).toBe(true);
		expect(needsNchwLayout(await store.ensure("rtmdet-ins-s"))).toBe(false);
		expect(needsNchwLayout(await store.ensure("selfie-landscape"))).toBe(false);
	}, 600_000);

	describe("GPU frames", () => {
		let device: GPUDevice;
		// Held for the test's lifetime: Dawn crashes if its instance is collected
		// while a device is alive.
		const keepAlive: unknown[] = [];
		const upload = (frame: VisionImageInput) => {
			const texture = device.createTexture({
				size: [frame.width, frame.height],
				format: "rgba8unorm",
				usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST,
			});
			device.queue.writeTexture(
				{ texture },
				frame.data as Uint8Array<ArrayBuffer>,
				{ bytesPerRow: frame.width * 4 },
				[frame.width, frame.height],
			);
			return { texture, device, width: frame.width, height: frame.height };
		};

		beforeAll(async () => {
			const { create, globals } = await import("webgpu");
			Object.assign(globalThis, globals);
			const instance = create([]);
			keepAlive.push(instance);
			const adapter = await instance.requestAdapter();
			if (!adapter) throw new Error("no WebGPU adapter");
			device = await adapter.requestDevice();
		});

		// Dawn can crash the worker at exit while a device is alive.
		afterAll(() => {
			device?.destroy();
			keepAlive.length = 0;
		});

		it("preprocesses on the GPU like imageToTensor", async () => {
			for (const family of ["rtmdet-ins", "rtmo", "selfie"] as const) {
				const spec = preprocessSpec(
					family,
					family === "selfie" ? [256, 144] : [640, 640],
				);
				const frame = frames.dancer;
				const expected = imageToTensor(frame, spec).tensor;
				const out = device.createBuffer({
					size: FrameTensorPipeline.tensorBytes(spec),
					usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
				});
				const staging = device.createBuffer({
					size: out.size,
					usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
				});
				FrameTensorPipeline.for(device).toTensor(upload(frame), spec, out);
				const encoder = device.createCommandEncoder();
				encoder.copyBufferToBuffer(out, 0, staging, 0, out.size);
				device.queue.submit([encoder.finish()]);
				await staging.mapAsync(GPUMapMode.READ);
				const actual = new Float32Array(staging.getMappedRange().slice(0));
				let worst = 0;
				for (let i = 0; i < expected.length; i++) {
					worst = Math.max(
						worst,
						Math.abs((actual[i] as number) - (expected[i] as number)),
					);
				}
				// A step of one 8-bit level is 1/std ≈ 0.017 (rtmdet), 1 (rtmo), 1/255 (selfie).
				expect(worst).toBeLessThan(family === "rtmo" ? 0.05 : 1e-3);
			}
		}, 120_000);

		it("gives the same detections, masks and poses as CPU frames", async () => {
			const gpu = VisionRunner.create({
				provider: new NodeWebGPUProvider({ device }),
			});
			const cpu = VisionRunner.create();
			for (const name of ["dancer", "crema"] as const) {
				const a = await gpu.segment(upload(frames[name]));
				const b = await cpu.segment(frames[name]);
				expect(a.detections.map((d) => d.category)).toEqual(
					b.detections.map((d) => d.category),
				);
				a.detections.forEach((d, i) => {
					expect(d.score).toBeCloseTo((b.detections[i] as typeof d).score, 2);
				});
				expect(a.masks.map((m) => Math.round(m.coverage * 1000))).toEqual(
					b.masks.map((m) => Math.round(m.coverage * 1000)),
				);
			}
			const pose = await gpu.pose(upload(frames.dancer));
			expect(pose.people).toHaveLength(1);
			gpu.close();
			cpu.close();
		}, 600_000);
	});

	describe.each(VISION_VARIANTS)("variant %s", (variant) => {
		const runner = VisionRunner.create({ variant });

		it("matches the end2end graph (head cut + own NMS and masks)", async () => {
			const key = runner.modelKeyFor("segment");
			const ort = await import("onnxruntime-node");
			const full = await ort.InferenceSession.create(
				await new VisionModelStore().ensure(key),
			);
			const decodeOptions = {
				confidence: 0.3,
				classNames: COCO_CLASSES,
				maskThreshold: 0.5,
			};
			for (const name of ["dancer", "crema", "tunnel", "spotlight"] as const) {
				const frame = frames[name];
				const spec = preprocessSpec("rtmdet-ins", VISION_MODELS[key].input);
				const { tensor, transform } = imageToTensor(frame, spec);
				const out = await full.run({
					input: new ort.Tensor("float32", tensor, [
						1,
						3,
						spec.height,
						spec.width,
					]),
				});
				const dets = out.dets as unknown as {
					data: Float32Array;
					dims: readonly number[];
				};
				const masks = out.masks as unknown as {
					data: Float32Array;
					dims: readonly number[];
				};
				const expected = decodeRtmdetIns(
					{
						dets: dets.data,
						labels: (out.labels as { data: BigInt64Array }).data,
						masks: masks.data,
						count: dets.dims[1] ?? 0,
						maskHeight: masks.dims[2] ?? 0,
						maskWidth: masks.dims[3] ?? 0,
					},
					{
						...decodeOptions,
						transform,
						sourceWidth: frame.width,
						sourceHeight: frame.height,
					},
				);
				const actual = await runner.segment(frame);

				expect(actual.detections.map((d) => d.category)).toEqual(
					expected.detections.map((d) => d.category),
				);
				actual.detections.forEach((d, i) => {
					const e = expected.detections[i] as (typeof expected.detections)[0];
					expect(d.score).toBeCloseTo(e.score, 4);
					expect(d.boundingBox.originX).toBeCloseTo(e.boundingBox.originX, 2);
					expect(d.boundingBox.width).toBeCloseTo(e.boundingBox.width, 2);
				});
				expect(actual.masks).toHaveLength(expected.masks.length);
				actual.masks.forEach((m, i) => {
					const e = expected.masks[i] as (typeof expected.masks)[0];
					let inter = 0;
					let union = 0;
					for (let p = 0; p < m.mask.length; p++) {
						const a = (m.mask[p] as number) >= 128;
						const b = (e.mask[p] as number) >= 128;
						if (a && b) inter++;
						if (a || b) union++;
					}
					// Edges differ slightly: end2end upsamples twice (graph ×8, then to
					// the frame) with the sigmoid in between; the head path upsamples the
					// logits once, straight to the frame.
					expect(union === 0 ? 1 : inter / union).toBeGreaterThan(0.98);
				});
			}
			await full.release();
		}, 600_000);

		it("cuts out the whole dancer, dress included", async () => {
			const seg = await runner.segment(frames.dancer);
			const subject = mergeSubjectMask(seg.masks);
			expect(subject?.category).toBe("person");
			// the bare person mask is ~3% of the frame; with the dress merged it is ~12.5%
			expect(subject?.coverage).toBeGreaterThan(0.08);
			expect(subject?.coverage).toBeLessThan(0.2);
		}, 120_000);

		it("cuts out the whole dancer when the subject is limited to people", async () => {
			// The dress comes back as other classes ("kite", "umbrella"); a person-only
			// filter before the merge leaves just the bare person (~3%).
			const seg = await runner.segment(frames.dancer);
			const subject = mergeSubjectMask(seg.masks, { classes: ["person"] });
			expect(subject?.category).toBe("person");
			expect(subject?.coverage).toBeGreaterThan(0.08);
			expect(subject?.coverage).toBeLessThan(0.2);
		}, 120_000);

		it("finds one dancer with a full skeleton", async () => {
			const { people } = await runner.pose(frames.dancer);
			expect(people).toHaveLength(1);
			const visible = people[0].keypoints.filter((k) => k.visibility > 0.5);
			expect(visible.length).toBeGreaterThanOrEqual(12);
			const nose = people[0].keypoints[0];
			expect(nose.x).toBeGreaterThan(1000);
			expect(nose.x).toBeLessThan(1120);
			expect(nose.y).toBeGreaterThan(250);
			expect(nose.y).toBeLessThan(360);
		}, 120_000);

		it("reports the spotlight dancer once (no duplicate skeletons)", async () => {
			const { people } = await runner.pose(frames.spotlight);
			expect(people).toHaveLength(1);
		}, 120_000);

		it("picks the cup, not the table, as a non-person subject", async () => {
			const seg = await runner.segment(frames.crema);
			expect(mergeSubjectMask(seg.masks)?.category).toBe("cup");
		}, 120_000);

		it("keeps a tiny figure separate from the tunnel around it", async () => {
			const seg = await runner.segment(frames.tunnel);
			const subject = mergeSubjectMask(seg.masks);
			expect(subject?.category).toBe("person");
			expect(subject?.coverage).toBeLessThan(0.02);
		}, 120_000);

		it("sees no confident person in abstract frames", async () => {
			for (const frame of [frames.ink, frames.eclipse]) {
				const detections = await runner.detect(frame);
				const people = detections.filter(
					(d) => d.category === "person" && d.score >= 0.5,
				);
				expect(people).toHaveLength(0);
			}
		}, 120_000);

		it("mattes a portrait with the Selfie Segmenter", async () => {
			const matte = await runner.matte(frames.portrait);
			expect(matte.width).toBe(frames.portrait.width);
			expect(matte.coverage).toBeGreaterThan(0.3);
			expect(matte.coverage).toBeLessThan(0.6);
		}, 120_000);
	});

	describe("alpha matting (ViTMatte Small)", () => {
		const runner = VisionRunner.create();
		afterAll(() => runner.close());

		it("recovers a soft edge a hard mask cuts off", async () => {
			// A textured disc whose alpha falls off over 12 px, over a textured background.
			const width = 320;
			const height = 240;
			const truth = new Float32Array(width * height);
			const data = new Uint8Array(width * height * 4);
			for (let y = 0; y < height; y++) {
				for (let x = 0; x < width; x++) {
					const i = y * width + x;
					const r = Math.hypot(x - 160, y - 120);
					const a = Math.min(1, Math.max(0, (80 - r) / 12));
					truth[i] = a;
					const fg = [220, 170 + 40 * Math.sin(x / 3), 60];
					const bg = [40 + 30 * Math.sin(y / 5), 90, 200];
					for (let c = 0; c < 3; c++)
						data[i * 4 + c] =
							(fg[c] as number) * a + (bg[c] as number) * (1 - a);
					data[i * 4 + 3] = 255;
				}
			}
			const hard = Uint8Array.from(truth, (a) => (a >= 0.5 ? 255 : 0));
			const matte = await runner.refineMatte({ data, width, height }, hard, {
				band: 12,
			});
			let hardError = 0;
			let softError = 0;
			for (let i = 0; i < truth.length; i++) {
				hardError += Math.abs((hard[i] as number) / 255 - (truth[i] as number));
				softError += Math.abs(
					(matte.mask[i] as number) / 255 - (truth[i] as number),
				);
			}
			expect(softError).toBeLessThan(hardError * 0.5);
		}, 600_000);

		it("softens a portrait's segmentation edge and keeps its core", async () => {
			const frame = frames.portrait;
			const subject = mergeSubjectMask((await runner.segment(frame)).masks);
			if (!subject) throw new Error("no subject in the portrait");
			const hard = Uint8Array.from(subject.mask, (v) => (v >= 128 ? 255 : 0));
			const matte = await runner.refineMatte(frame, hard);
			let soft = 0;
			let inside = 0;
			let kept = 0;
			for (let i = 0; i < hard.length; i++) {
				const a = matte.mask[i] as number;
				if (a > 10 && a < 245) soft++;
				if (hard[i]) {
					inside++;
					if (a >= 128) kept++;
				}
			}
			expect(soft).toBeGreaterThan(frame.width);
			expect(kept / inside).toBeGreaterThan(0.95);
		}, 600_000);
	});
});