Skip to content

Code-first video & image, rendered natively on WebGPU.

3.0 KB
Raw Blame
import { describe, expect, it } from "vitest";
import type { InstanceMask } from "../types.js";
import { maskBounds, mergeSubjectMask, selectSubjectMask } from "./subject.js";

const W = 20;
const H = 20;

function rect(
	category: string,
	x0: number,
	y0: number,
	x1: number,
	y1: number,
	detectionIndex = 0,
): InstanceMask {
	const mask = new Uint8Array(W * H);
	let area = 0;
	for (let y = y0; y <= y1; y++) {
		for (let x = x0; x <= x1; x++) {
			mask[y * W + x] = 255;
			area++;
		}
	}
	return {
		category,
		mask,
		width: W,
		height: H,
		area,
		coverage: area / (W * H),
		detectionIndex,
	};
}

describe("subject selection", () => {
	it("prefers a person over a larger non-person instance", () => {
		const person = rect("person", 0, 0, 2, 2);
		const car = rect("car", 5, 5, 15, 15);
		expect(selectSubjectMask([car, person])).toBe(person);
		expect(selectSubjectMask([car])).toBe(car);
		expect(selectSubjectMask([])).toBeUndefined();
	});

	it("without a person, prefers the most confident instance over a large backdrop", () => {
		const table = rect("dining table", 0, 0, 19, 19, 1);
		const cup = rect("cup", 6, 6, 12, 12, 0);
		expect(selectSubjectMask([table, cup])).toBe(cup);
	});

	it("does not merge a container much larger than the subject", () => {
		const figure = rect("person", 9, 9, 10, 11);
		const tunnel = rect("clock", 0, 0, 19, 19, 1);
		expect(mergeSubjectMask([tunnel, figure])).toBe(figure);
	});

	it("merges instances overlapping the subject (e.g. a dress detected as another class)", () => {
		const body = rect("person", 8, 2, 12, 17);
		const dress = rect("horse", 4, 6, 16, 16);
		const farCar = rect("car", 0, 18, 3, 19);
		const merged = mergeSubjectMask([farCar, dress, body]);
		expect(merged?.category).toBe("person");
		expect(merged?.mask[10 * W + 5]).toBe(255); // dress pixel joined the subject
		expect(merged?.mask[19 * W + 1]).toBe(0); // unrelated car stays out
		expect(merged?.area).toBe(body.area + dress.area - 5 * 11);
	});

	it("limits the subject to `classes` but merges parts of any class", () => {
		// The smoke project's cutout: classes ["person"], a dress detected as a "kite".
		const body = rect("person", 8, 2, 12, 17, 1);
		const dress = rect("kite", 4, 6, 16, 16, 0);
		const merged = mergeSubjectMask([dress, body], { classes: ["Person"] });
		expect(merged?.category).toBe("person");
		expect(merged?.mask[10 * W + 5]).toBe(255);
		expect(merged?.area).toBe(body.area + dress.area - 5 * 11);
	});

	it("finds no subject when no instance is of `classes`", () => {
		const car = rect("car", 5, 5, 15, 15);
		expect(selectSubjectMask([car], ["person"])).toBeUndefined();
		expect(mergeSubjectMask([car], { classes: ["person"] })).toBeUndefined();
		expect(selectSubjectMask([car], ["car"])).toBe(car);
	});

	it("maskBounds returns tight bounds or null", () => {
		expect(maskBounds(rect("x", 3, 4, 7, 9))).toEqual({
			x0: 3,
			y0: 4,
			x1: 7,
			y1: 9,
		});
		expect(
			maskBounds({ ...rect("x", 0, 0, 0, 0), mask: new Uint8Array(W * H) }),
		).toBeNull();
	});
});