Skip to content

Code-first video & image, rendered natively on WebGPU.

2.8 KB
Raw Blame
import { describe, expect, it } from "vitest";
import { ClipVision, seedClipVision } from "./clip-vision.js";

function selectedClip(reverse = false) {
	const clip = new ClipVision("selected.mp4", 24, {
		cacheDir: false,
		sam2: {
			category: "person",
			prompts: [{ frame: 0, points: [{ x: 0.2, y: 0.5, label: 1 }] }],
		},
		...(reverse
			? {
					timeRemap: {
						kind: "rate" as const,
						sourceStartSec: 2 / 24,
						rate: -1,
					},
				}
			: {}),
	});
	seedClipVision(
		clip,
		{ width: 8, height: 4 },
		Array.from({ length: 3 }, (_, frame) => {
			const mask = Uint8Array.from({ length: 32 }, (_, i) =>
				i % 8 >= frame && i % 8 < frame + 2 ? 255 : 0,
			);
			return { kp: null, mask: null, subject: mask };
		}),
	);
	return clip;
}

describe("prompted clip vision signals", () => {
	it("reuses the selected mask at fractional analysis timestamps", () => {
		const clip = new ClipVision("selected.mp4", 24, {
			fps: 12,
			sam2: { prompts: [{ frame: 0, points: [{ x: 0.2, y: 0.5, label: 1 }] }] },
		});
		seedClipVision(clip, { width: 2, height: 1 }, [
			{ kp: null, mask: null, subject: new Uint8Array([255, 0]) },
			{ kp: null, mask: null, subject: new Uint8Array([0, 255]) },
		]);
		expect(clip.subject(1, 24)?.mask).toEqual(new Uint8Array([255, 0]));
		expect(clip.subject(3, 24)?.mask).toEqual(new Uint8Array([0, 255]));
	});

	it("exposes selected-object geometry and frame-exact lazy masks", async () => {
		const clip = selectedClip();
		const signals = clip.signals;
		await clip.ready(0, 2);
		expect(signals.getObjectResult(0).objects[0]).toMatchObject({
			trackId: 1,
			boundingBox: { originX: 0, width: 2 },
			active: true,
		});
		expect(signals.getObjectResult(2).objects[0]?.velocity.vx).toBe(24);
		expect(signals.getMaskResult(2)[0]?.mask[2]).toBe(255);
		expect(signals.getMaskResult(0)[0]?.mask[2]).toBe(0);
		expect(signals.segmentation.matte.at(2)?.coverage).toBe(0.25);
		expect(
			signals.objects.get(1).active.get({ frame: 2, fps: 24 } as never),
		).toBe(1);
		clip.release();
		expect(signals.getMaskResult(1)[0]?.mask[1]).toBe(255);
	});

	it("maps reverse playback into masks and object velocity without rewinding model state", async () => {
		const clip = selectedClip(true);
		const signals = clip.signals;
		await clip.ready(0, 2);
		expect(signals.getObjectResult(0).objects[0]?.boundingBox.originX).toBe(2);
		expect(signals.getObjectResult(1).objects[0]?.velocity.vx).toBe(-24);
		expect(signals.segmentation.matte.at(2)?.mask[0]).toBe(255);
	});

	it("copies prompt inputs so later edits cannot silently change the cache recipe", () => {
		const points = [{ x: 0.2, y: 0.5, label: 1 as const }];
		const clip = new ClipVision("selected.mp4", 24, {
			sam2: { prompts: [{ frame: 0, points }] },
		});
		const point = points[0];
		if (point) point.x = 0.9;
		expect(clip.options.sam2?.prompts[0]?.points[0]?.x).toBe(0.2);
	});
});