Skip to content

Code-first video & image, rendered natively on WebGPU.

4.9 KB
Raw Blame
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import {
	type Sam2Point,
	Sam2Tracker,
	Sam3Tracker,
} from "@framefields/vision-sam";
import { mediaDecoderCache } from "@framefields/webgpu-renderers";
import { afterEach, describe, expect, it, vi } from "vitest";
import { ClipVision } from "./clip-vision.js";

const points = [{ x: 0.25, y: 0.5, label: 1 as const }];
afterEach(() => vi.restoreAllMocks());

describe("SAM source scheduling and persistence", () => {
	it("propagates skipped frames, applies correction prompts, and reuses disk masks on seek", async () => {
		const dir = await mkdtemp(path.join(tmpdir(), "framefields-sam-clip-"));
		const source = path.join(dir, "source.mp4");
		await writeFile(source, "decoder test fixture");
		const process = vi.fn(
			async (
				_image: unknown,
				frame: number,
				_points?: readonly Sam2Point[],
			) => ({
				mask: Uint8Array.from({ length: 32 }, (_, i) =>
					i % 8 === frame ? 255 : 0,
				),
				width: 8,
				height: 4,
				score: 4,
				iou: 0.9,
			}),
		);
		const release = vi.fn();
		vi.spyOn(Sam2Tracker, "create").mockResolvedValue({
			process,
			release,
		} as unknown as Sam2Tracker);
		vi.spyOn(mediaDecoderCache, "getVideo").mockReturnValue({
			durationSec: async () => 10,
			getFrame: async () => ({
				width: 8,
				height: 4,
				buffer: new Uint8Array(128),
			}),
		} as unknown as ReturnType<typeof mediaDecoderCache.getVideo>);
		const options = {
			cacheDir: dir,
			sam2: {
				prompts: [
					{ frame: 1, points },
					{ frame: 3, points },
				],
			},
		};
		try {
			const clip = new ClipVision(source, 24, options);
			const signals = clip.signals;
			await clip.ready(4);
			expect(process.mock.calls.map((call) => call[1])).toEqual([1, 2, 3, 4]);
			expect(process.mock.calls[0]?.[2]).toEqual(points);
			expect(process.mock.calls[1]?.[2]).toBeUndefined();
			expect(process.mock.calls[2]?.[2]).toEqual(points);
			expect(clip.subject(0, 24)).toBeNull();
			expect(signals.getMaskResult(4)[0]?.mask[4]).toBe(255);
			clip.release();
			const restored = new ClipVision(source, 24, options);
			await restored.ready(2);
			expect(restored.subject(2, 24)?.mask[2]).toBe(255);
			expect(process).toHaveBeenCalledTimes(4);
			const invalid = new ClipVision(source, 24, {
				cacheDir: false,
				sam2: {
					prompts: [
						{ frame: 0, points },
						{ frame: 300, points },
					],
				},
			});
			await expect(invalid.ready(0)).rejects.toThrow("inside the source video");
		} finally {
			await rm(dir, { recursive: true, force: true });
		}
	});

	it("gives concurrent clips of one source their own decoders", async () => {
		// A decoder drops a pending request when a newer one arrives, so shared
		// decoders made one clip read `null` frames while another advanced.
		vi.spyOn(Sam2Tracker, "create").mockResolvedValue({
			process: async () => ({
				mask: new Uint8Array(32),
				width: 8,
				height: 4,
				score: 1,
				iou: 1,
			}),
			release: () => {},
		} as unknown as Sam2Tracker);
		const getVideo = vi.spyOn(mediaDecoderCache, "getVideo").mockReturnValue({
			durationSec: async () => 1,
			getFrame: async () => ({
				width: 8,
				height: 4,
				buffer: new Uint8Array(128),
			}),
		} as unknown as ReturnType<typeof mediaDecoderCache.getVideo>);
		const clips = [0.25, 0.75].map(
			(x) =>
				new ClipVision("shared.mp4", 24, {
					cacheDir: false,
					sam2: { prompts: [{ frame: 0, points: [{ x, y: 0.5, label: 1 }] }] },
				}),
		);
		await Promise.all(clips.map((clip) => clip.ready(2)));
		const ids = new Set(getVideo.mock.calls.map((call) => call[2]));
		expect(ids.size).toBe(2);
	});

	it("publishes every SAM 3 track under its own ID", async () => {
		const columns = (from: number) =>
			Uint8Array.from({ length: 32 }, (_, i) =>
				i % 8 >= from && i % 8 < from + 2 ? 255 : 0,
			);
		vi.spyOn(Sam3Tracker, "create").mockResolvedValue({
			process: async () => ({
				width: 8,
				height: 4,
				objects: [
					{ id: 1, mask: columns(0), score: 3, iou: 1 },
					{ id: 2, mask: columns(5), score: 2, iou: 1 },
				],
			}),
			release: () => {},
		} as unknown as Sam3Tracker);
		vi.spyOn(mediaDecoderCache, "getVideo").mockReturnValue({
			durationSec: async () => 1,
			getFrame: async () => ({
				width: 8,
				height: 4,
				buffer: new Uint8Array(128),
			}),
		} as unknown as ReturnType<typeof mediaDecoderCache.getVideo>);
		const clip = new ClipVision("crowd.mp4", 24, {
			cacheDir: false,
			sam3: { category: "person", maxObjects: 2, text: "student" },
		});
		const signals = clip.signals;
		await clip.ready(0);
		const objects = signals.getObjectResult(0).objects;
		expect(objects.map((o) => [o.trackId, o.boundingBox.originX])).toEqual([
			[1, 0],
			[2, 5],
		]);
		expect(signals.getMaskResult(0).map((m) => m.trackId)).toEqual([1, 2]);
		expect(signals.segmentation.matte.at(0)?.coverage).toBe(0.5);
		clip.release();
	});
});