Skip to content

Code-first video & image, rendered natively on WebGPU.

7.5 KB
Raw Blame
import { collectNodeOps } from "@framefields/compositions/program";
import {
	type MediaSignalRecorder,
	type ResolvedMediaTimeRemap,
	sampleMediaSignal,
} from "@framefields/core";
import { describe, expect, it } from "vitest";
import { clipVisionFromResults } from "./clip-vision.js";
import { Composition, Layer, Signal } from "./index.js";

/** A 160×80 clip: frame f has the left wrist at (20 + 20f, 10) and a 40×40 square silhouette moving right. */
function clip(frames = 4, options = {}) {
	const cell = 4;
	const w = 40;
	const h = 20;
	return clipVisionFromResults(
		24,
		{ width: 160, height: 80, cell },
		Array.from({ length: frames }, (_, f) => {
			const kp = new Array(51).fill(0);
			kp.splice(9 * 3, 3, 20 + 20 * f, 10, 1);
			const mask = new Uint8Array(w * h);
			for (let y = 5; y < 15; y++)
				for (let x = 5 + f * 10; x < 15 + f * 10 && x < w; x++)
					mask[y * w + x] = 255;
			return { kp, mask };
		}),
		{ smoothing: 0, ...options },
	);
}

describe("ClipVision", () => {
	it("reads keypoints at composition frames and maps them into a layer box", () => {
		const person = clip().person;
		expect(person.point("leftWrist", 1.5)).toMatchObject({
			x: 50,
			y: 10,
			visibility: 1,
		});
		// 160×80 covering 320×320 at (100, 0): scale 4, centered.
		const placed = person.place({
			x: 100,
			y: 0,
			width: 320,
			height: 320,
			fit: "cover",
		});
		expect(placed.point("leftWrist", 0)).toMatchObject({
			x: 100 - 160 + 80,
			y: 40,
		});
		expect(
			placed.signals("leftWrist").x.get({ frame: 1, fps: 24 } as never),
		).toBeCloseTo(100 - 160 + 160);
	});

	it("offsets the clip by its start frame", () => {
		const late = clip(4, { startFrame: 10 }).person;
		expect(late.point("leftWrist", 12).x).toBe(60);
	});

	it("smooths over neighbouring frames, weighted by confidence", () => {
		const person = clip(5, { smoothing: 1 }).person;
		// Frames 1..3 at x 40, 60, 80 with weights 1, 2, 1 → 60.
		expect(person.point("leftWrist", 2).x).toBeCloseTo(60);
	});

	it("samples the silhouette with a gradient pointing inward", () => {
		const person = clip().person;
		expect(person.sample(0, 40, 40).alpha).toBeGreaterThan(0.9);
		expect(person.sample(0, 120, 40).alpha).toBe(0);
		const edge = person.sample(0, 62, 40);
		expect(edge.alpha).toBeGreaterThan(0);
		expect(edge.gx).toBeLessThan(0);
	});

	it("gives render-time mattes the full-resolution subject per source frame", () => {
		const subject = new Uint8Array(160 * 80);
		subject.fill(200, 0, 160 * 10);
		const c = clipVisionFromResults(24, { width: 160, height: 80, cell: 4 }, [
			{ kp: null, mask: null, subject },
			{ kp: null, mask: null, subject: null },
			{ kp: null, mask: null },
		]);
		const s = c.subject(0, 24);
		expect(s).toMatchObject({ width: 160, height: 80, category: "person" });
		expect(s?.mask).toEqual(subject);
		expect(s?.area).toBe(Math.round((200 * 160 * 10) / 255));
		// Source frame 2 at 48 fps is clip frame 1: analysed, nobody there.
		expect(c.subject(2, 48)).toBeNull();
		// Not analysed with subjects, or between clip frames: the caller segments.
		expect(c.subject(2, 24)).toBeUndefined();
		expect(c.subject(1, 48)).toBeUndefined();
	});

	it("explains a read of a frame that was never analysed", () => {
		expect(() => clip(2).person.point("leftWrist", 30)).toThrow(
			/not analysed yet/,
		);
	});
});

describe("remapped clip vision", () => {
	it("uses recorded signal history for the same source clock", () => {
		const history = sampleMediaSignal({
			mode: "source-time",
			fps: 24,
			startFrame: 0,
			durationFrames: 2,
			sample: (frame) => frame / 24,
		});
		const c = clip(4, { timeRemap: { kind: "sampled", history } });
		expect(c.person.point("leftWrist", 1).x).toBe(40);
	});
	it("samples reverse motion and covers intermediate turns in a readiness window", () => {
		const c = clip(4, {
			timeRemap: {
				kind: "keyframes",
				keyframes: [
					{ frame: 0, sourceSec: 0 },
					{ frame: 3, sourceSec: 3 / 24, interpolation: "smooth" },
					{ frame: 6, sourceSec: 1 / 24 },
				],
			},
		});
		expect(c.person.point("leftWrist", 3).x).toBe(80);
		expect(c.person.point("leftWrist", 6).x).toBe(40);
		const range = (
			c as unknown as { window(from: number, to: number): [number, number] }
		).window(0, 6);
		expect(range[1]).toBeGreaterThanOrEqual(3);
	});
	it("requires a layer to be in the composition before binding vision", () => {
		const comp = new Composition({
			width: 64,
			height: 64,
			fps: 24,
			durationFrames: 24,
		});
		expect(() =>
			comp.vision(Layer.video("clip.mp4", { durationFrames: 24 })),
		).toThrow(/Add the media layer/);
	});
	it("binds to the absolute start of a nested media layer", () => {
		const comp = new Composition({
			width: 64,
			height: 64,
			fps: 24,
			durationFrames: 100,
		});
		const layer = Layer.video("clip.mp4", {
			startFrame: 7,
			durationFrames: 24,
			timeRemap: { kind: "rate", sourceStartSec: 1, rate: 0.5 },
		});
		comp.add(
			Layer.box({ startFrame: 10, durationFrames: 60, children: [layer] }),
		);
		const vision = comp.vision(layer);
		expect(vision.startFrame).toBe(17);
		expect(vision.clipFrame(19)).toBe(25);
	});
	it("binds a layer inside a 3D scene on the composition's clock", () => {
		const comp = new Composition({
			width: 64,
			height: 64,
			fps: 24,
			durationFrames: 200,
		});
		// The scene starts at 90 (an act at 60, the scene 30 into it); the
		// clip starts 4 frames into the scene, on the scene's own clock.
		const layer = Layer.video("clip.mp4", {
			id: "plate",
			is3D: true,
			startFrame: 4,
			durationFrames: 40,
			timeRemap: { kind: "rate", sourceStartSec: 1, rate: 0.5 },
		});
		comp.add(
			Layer.box({
				startFrame: 60,
				durationFrames: 120,
				children: [
					Layer.box({
						width: 64,
						height: 64,
						startFrame: 30,
						durationFrames: 60,
						children: [
							Layer.camera({ x: 32, y: 32, z: -100, targetX: 32, targetY: 32 }),
							layer,
						],
					}),
				],
			}),
		);
		const vision = comp.vision(layer);
		expect(vision.startFrame).toBe(94);
		// Its time remap comes along: 2 frames in, 1 s + 2/24 × 0.5 s of source.
		expect(vision.clipFrame(96)).toBeCloseTo((1 + (2 / 24) * 0.5) * 24, 6);
	});
	it("shares a layer's live signal time with its render", () => {
		const comp = new Composition({
			width: 64,
			height: 64,
			fps: 24,
			durationFrames: 48,
		});
		const speed = Signal.state(2);
		const layer = Layer.video("clip.mp4", {
			startFrame: 4,
			durationFrames: 24,
			timeRemap: { kind: "signal", mode: "speed", signal: speed },
		});
		comp.add(layer);
		const vision = comp.vision(layer);
		const op = collectNodeOps(comp.toVirtualMedia().children).find(
			(node) => node.operation.dataType === "Video",
		)?.operation as unknown as { timeRemap: ResolvedMediaTimeRemap };
		expect(vision.options.timeRemap).toBe(op.timeRemap);
		// No look-ahead: the clip's time is known once the clock reaches it.
		expect(() => vision.clipFrame(5)).toThrow(/not been recorded/);
		(op.timeRemap as { recorder: MediaSignalRecorder }).recorder.advance(5);
		expect(vision.clipFrame(5)).toBe(2);
	});
	it("samples the same source clock as the media layer", () => {
		const c = clip(4, {
			startFrame: 10,
			timeRemap: {
				kind: "keyframes",
				keyframes: [
					{ frame: 0, sourceSec: 0 },
					{ frame: 2, sourceSec: 1 / 24 },
					{ frame: 4, sourceSec: 1 / 24 },
					{ frame: 6, sourceSec: 3 / 24 },
				],
			},
		});
		expect(c.person.point("leftWrist", 12).x).toBe(40);
		expect(c.person.point("leftWrist", 13).x).toBe(40);
		expect(c.person.point("leftWrist", 16).x).toBe(80);
	});
});