Skip to content

Code-first video & image, rendered natively on WebGPU.

21.0 KB
Raw Blame
import type { DetectedObject } from "@framefields/core";
import { decodeDepth, depthInputSize } from "../decode/depth.js";
import {
	type InputTransform,
	imageToTensor,
	isPreprocessFamily,
	preprocessSpec,
} from "../decode/preprocess.js";
import {
	decodeRtmdetIns,
	type RtmdetInsOutputs,
} from "../decode/rtmdet-ins.js";
import {
	maskLogits,
	RTMDET_INS_HEAD,
	type RtmdetInsHeadOutputs,
	selectInstances,
} from "../decode/rtmdet-ins-head.js";
import { decodeRtmo } from "../decode/rtmo.js";
import { decodeSelfie } from "../decode/selfie.js";
import { FrameTensorPipeline, readTexturePixels } from "../gpu/frame-tensor.js";
import { defaultMatteBand, trimapFromMask } from "../matting/trimap.js";
import {
	composeMatte,
	matteWindow,
	vitmatteInput,
} from "../matting/vitmatte.js";
import {
	type ModelDownloadStatus,
	VisionModelStore,
} from "../model/model-store.js";
import { extractSubgraph } from "../model/onnx-subgraph.js";
import {
	COCO_CLASSES,
	modelKeyFor,
	VISION_MODELS,
	type VisionModelKey,
	type VisionTask,
	type VisionVariant,
} from "../model/registry.js";
import {
	createDefaultSessionProvider,
	type SessionProvider,
	type VisionSession,
	type VisionTensor,
	type VisionTensorOutputs,
} from "../runtime/session-provider.js";
import {
	type DepthMap,
	isGpuImage,
	type MatteRefineOptions,
	type PersonMatte,
	type PoseResult,
	type RefinedMatte,
	type SegmentationResult,
	type VisionFrame,
	type VisionImageInput,
} from "../types.js";

export interface VisionRunnerOptions {
	/** Model size for detect/segment/pose. Default "s". */
	readonly variant?: VisionVariant;
	/** Minimum detection / person score, 0..1. Default 0.3. */
	readonly confidence?: number;
	/** COCO class-name filter for detect/segment (all classes when omitted). */
	readonly classes?: readonly string[];
	readonly maskThreshold?: number;
	readonly featherRadius?: number;
	readonly modelsDir?: string;
	/** Mirror origin for model downloads (see VisionModelStore). */
	readonly baseUrl?: string;
	readonly timeoutMs?: number;
	/** Session backend. Default: WebGPU when possible, else onnxruntime-node (CPU); see `AutoSessionProvider`. Browser: `createWebGPUProvider()`. */
	readonly provider?: SessionProvider;
	/** Inject a model store (tests, shared caches). Overrides modelsDir/baseUrl/timeoutMs. */
	readonly store?: VisionModelStore;
	readonly onProgress?: (
		key: VisionModelKey,
		bytesLoaded: number,
		totalBytes: number,
	) => void;
}

/** Options that only change how outputs are decoded — not which model runs. */
export type VisionDecodeOptions = Pick<
	VisionRunnerOptions,
	"confidence" | "classes" | "maskThreshold" | "featherRadius"
>;

/**
 * Model-side state: sessions, buffers and the per-frame RTMDet-Ins pass. Shared
 * by a runner and every runner `derive()`d from it.
 */
interface RunnerCore {
	readonly store: VisionModelStore;
	readonly provider: SessionProvider;
	readonly sessions: Map<VisionModelKey, Promise<VisionSession>>;
	readonly tensorScratch: Map<VisionModelKey, Float32Array>;
	/** Tail of each model's run queue: one inference per session at a time. */
	readonly runQueue: Map<VisionModelKey, Promise<unknown>>;
	/** Each model's input on the GPU (GPU frames); runs are queued, so one each. */
	readonly gpuInputs: Map<
		VisionModelKey,
		{ readonly device: GPUDevice; readonly buffer: GPUBuffer }
	>;
	/**
	 * Recent RTMDet-Ins passes, newest last, shared by detect() and segment()
	 * and matched by content — layers showing the same pixels share one pass.
	 */
	readonly instancePasses: {
		readonly content: string;
		readonly result: Promise<InstancePass>;
	}[];
}

const MAX_INSTANCE_PASSES = 4;

/**
 * Lazy vision runner.
 *
 * `create()` performs ZERO I/O: no downloads, no sessions. Each inference method downloads
 * its model and warms its session on FIRST USE ONLY, then reuses them for the process
 * lifetime. `detect()` and `segment()` share one RTMDet-Ins forward pass per frame.
 * Call `close()` to release sessions.
 */
export class VisionRunner {
	public readonly variant: VisionVariant;
	public readonly confidence: number;
	public readonly classes?: readonly string[];
	public readonly maskThreshold: number;
	public readonly featherRadius?: number;

	private readonly _core: RunnerCore;
	/** False for derived runners: their sessions belong to the runner they came from. */
	private readonly _ownsCore: boolean;

	private constructor(options: VisionRunnerOptions, core?: RunnerCore) {
		this.variant = options.variant ?? "s";
		this.confidence = options.confidence ?? 0.3;
		this.classes = options.classes;
		this.maskThreshold = options.maskThreshold ?? 0.5;
		this.featherRadius = options.featherRadius;
		this._ownsCore = core === undefined;
		this._core = core ?? {
			store:
				options.store ??
				new VisionModelStore({
					modelsDir: options.modelsDir,
					baseUrl: options.baseUrl,
					timeoutMs: options.timeoutMs,
					onProgress: options.onProgress,
				}),
			provider: options.provider ?? createDefaultSessionProvider(),
			sessions: new Map(),
			tensorScratch: new Map(),
			runQueue: new Map(),
			gpuInputs: new Map(),
			instancePasses: [],
		};
	}

	/** Cheap: no downloads, no sessions, no side effects. */
	public static create(options: VisionRunnerOptions = {}): VisionRunner {
		return new VisionRunner(options);
	}

	/**
	 * A runner with other decode options (confidence, classes, mask threshold,
	 * feather) that shares this runner's models, sessions and per-frame RTMDet-Ins
	 * pass — e.g. one per layer, without loading a model twice. Its `close()` is a
	 * no-op; closing this runner releases the shared sessions.
	 */
	public derive(options: VisionDecodeOptions): VisionRunner {
		return new VisionRunner(
			{
				variant: this.variant,
				confidence: options.confidence ?? this.confidence,
				classes: "classes" in options ? options.classes : this.classes,
				maskThreshold: options.maskThreshold ?? this.maskThreshold,
				featherRadius:
					"featherRadius" in options
						? options.featherRadius
						: this.featherRadius,
			},
			this._core,
		);
	}

	public get modelsDir(): string {
		return this._core.store.modelsDir;
	}

	/** Every registry model keyed to its current download state (downloaded → "ready"). */
	public get downloadStatus(): ReadonlyMap<
		VisionModelKey,
		ModelDownloadStatus
	> {
		const statuses = new Map<VisionModelKey, ModelDownloadStatus>();
		for (const key of Object.keys(VISION_MODELS) as VisionModelKey[]) {
			statuses.set(key, "pending");
		}
		for (const [key, status] of this._core.store.statuses()) {
			statuses.set(key, status);
		}
		return statuses;
	}

	/** The model serving `task` (matte depends on frame aspect; defaults to square). */
	public modelKeyFor(task: VisionTask, aspect = 1): VisionModelKey {
		return modelKeyFor(task, this.variant, aspect);
	}

	/**
	 * Explicit warm-up: downloads + sessions for the given tasks, ahead of first use.
	 * `matte` warms both aspect variants (the frame aspect is unknown until inference).
	 */
	public async preload(
		tasks: readonly VisionTask[] = ["detect"],
	): Promise<void> {
		const keys = new Set<VisionModelKey>();
		for (const task of tasks) {
			if (task === "matte") {
				keys.add("selfie-square");
				keys.add("selfie-landscape");
			} else {
				keys.add(this.modelKeyFor(task));
			}
		}
		await this._core.store.preload([...keys]);
		await Promise.all([...keys].map((key) => this.session(key)));
	}

	/** COCO-80 boxes (sorted by score, source pixels). Masks are not decoded. */
	public async detect(image: VisionFrame): Promise<readonly DetectedObject[]> {
		const pass = await this.instancePass(image);
		return decodeRtmdetIns(
			{ ...pass.outputs, masks: undefined, maskPlane: undefined },
			this.instanceDecodeOptions(image, pass.transform),
		).detections;
	}

	/** COCO-80 boxes plus a frame-aligned soft mask per instance. */
	public async segment(image: VisionFrame): Promise<SegmentationResult> {
		const pass = await this.instancePass(image);
		return decodeRtmdetIns(
			pass.outputs,
			this.instanceDecodeOptions(image, pass.transform),
		);
	}

	/** COCO-17 keypoints per person (source pixels), highest score first. */
	public async pose(image: VisionFrame): Promise<PoseResult> {
		const key = this.modelKeyFor("pose");
		const { outputs, transform } = await this.infer(key, image);
		const dets = output(outputs, "dets", key);
		const keypoints = output(outputs, "keypoints", key);
		return decodeRtmo(
			{
				dets: dets.data as Float32Array,
				keypoints: keypoints.data as Float32Array,
				count: dets.dims[1] ?? 0,
				keypointStride: keypoints.dims[3] ?? 3,
			},
			{
				confidence: this.confidence,
				transform,
				sourceWidth: image.width,
				sourceHeight: image.height,
			},
		);
	}

	/** Person-vs-background alpha for the whole frame (Selfie Segmenter). */
	public async matte(image: VisionFrame): Promise<PersonMatte> {
		const key = this.modelKeyFor("matte", image.width / image.height);
		const [w, h] = VISION_MODELS[key].input;
		const { outputs } = await this.infer(key, image);
		const alphas = output(outputs, "alphas", key);
		return decodeSelfie(alphas.data as Float32Array, w, h, {
			sourceWidth: image.width,
			sourceHeight: image.height,
			maskThreshold: this.maskThreshold,
			...(this.featherRadius !== undefined
				? { featherRadius: this.featherRadius }
				: {}),
		});
	}

	/** Relative depth of the whole frame (Depth Anything V2 Small), near = 1, at the model's resolution. */
	public async depth(image: VisionFrame): Promise<DepthMap> {
		const key: VisionModelKey = "depth-anything-v2-s";
		const input = depthInputSize(
			image.width,
			image.height,
			VISION_MODELS[key].input[1],
		);
		const { outputs } = await this.infer(key, image, input);
		const predicted = output(outputs, "predicted_depth", key);
		const dims = predicted.dims;
		return decodeDepth(
			predicted.data as Float32Array,
			dims[dims.length - 1] ?? input[0],
			dims[dims.length - 2] ?? input[1],
		);
	}

	/**
	 * Resolves a hard mask's edge to soft alpha (ViTMatte Small): hair, motion
	 * blur and see-through parts. `mask` is frame-sized, inside where ≥ 128.
	 * Only a band around the edge changes; one model pass covers every part of
	 * the mask, cropped to the band and scaled to `resolution`.
	 */
	public async refineMatte(
		image: VisionFrame,
		mask: ArrayLike<number>,
		options: MatteRefineOptions = {},
	): Promise<RefinedMatte> {
		const { width, height } = image;
		if (mask.length !== width * height)
			throw new Error(
				`refineMatte: the mask has ${mask.length} pixels; the frame has ${width}×${height}.`,
			);
		const band = Math.max(
			1,
			Math.round(options.band ?? defaultMatteBand(width, height)),
		);
		const key: VisionModelKey = "vitmatte-s";
		const trimap = trimapFromMask(mask, width, height, band);
		const window = matteWindow(
			trimap,
			// Enough of either side for the model to tell subject from background.
			Math.max(32, band * 2),
			options.resolution ?? VISION_MODELS[key].input[0],
		);
		let alphas: ArrayLike<number> = [];
		if (window) {
			const pixels = isGpuImage(image) ? await readTexturePixels(image) : image;
			const outputs = await this.queued(key, async () => {
				const session = await this.session(key);
				const scratch = this._core.tensorScratch;
				const data = vitmatteInput(pixels, trimap, window, scratch.get(key));
				if (data.buffer !== scratch.get(key)?.buffer) scratch.set(key, data);
				return session.run({
					name: "pixel_values",
					data,
					dims: [1, 4, window.inputHeight, window.inputWidth],
				});
			});
			alphas = output(outputs, "alphas", key).data as Float32Array;
		}
		return {
			mask: composeMatte(trimap, window, alphas),
			width,
			height,
			nearest: trimap.nearest,
		};
	}

	public close(): void {
		if (!this._ownsCore) return;
		const core = this._core;
		for (const promise of core.sessions.values()) {
			void promise.then((session) => session.release()).catch(() => undefined);
		}
		core.sessions.clear();
		core.runQueue.clear();
		for (const { buffer } of core.gpuInputs.values()) buffer.destroy();
		core.gpuInputs.clear();
		core.instancePasses.length = 0;
		core.store.clearMemoryCache();
	}

	// ── internals ──────────────────────────────────────────────────────────

	private async instancePass(image: VisionFrame): Promise<InstancePass> {
		const passes = this._core.instancePasses;
		const content = isGpuImage(image)
			? await FrameTensorPipeline.for(image.device).contentKey(image)
			: contentKey(image);
		// Checked after the await: a caller that hashed the same frame meanwhile
		// has registered its pass by now.
		const byContent = passes.find((pass) => pass.content === content);
		if (byContent) return byContent.result;
		const key = this.modelKeyFor("segment");
		const result = this.infer(key, image).then(({ outputs, transform }) => ({
			transform,
			outputs:
				RTMDET_INS_HEAD.scores in outputs
					? headPassOutputs(outputs, key, transform)
					: end2endPassOutputs(outputs, key),
		}));
		const pass = { content, result };
		passes.push(pass);
		if (passes.length > MAX_INSTANCE_PASSES) passes.shift();
		result.catch(() => {
			const index = passes.indexOf(pass);
			if (index >= 0) passes.splice(index, 1);
		});
		return result;
	}

	private instanceDecodeOptions(image: VisionFrame, transform: InputTransform) {
		return {
			confidence: this.confidence,
			classes: this.classes,
			classNames: COCO_CLASSES,
			transform,
			sourceWidth: image.width,
			sourceHeight: image.height,
			maskThreshold: this.maskThreshold,
			...(this.featherRadius !== undefined
				? { featherRadius: this.featherRadius }
				: {}),
		};
	}

	/**
	 * Runs one model on one image. Runs of the same model are queued: they share
	 * the input scratch buffer, and one run already uses the session's threads.
	 */
	private infer(
		key: VisionModelKey,
		image: VisionFrame,
		/** Input size for graphs that take any (default: the registry's). */
		input?: readonly [number, number],
	): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {
		return this.queued(key, () => this.inferNow(key, image, input));
	}

	/** Runs `fn` after the model's earlier runs: one inference per session at a time. */
	private queued<T>(key: VisionModelKey, fn: () => Promise<T>): Promise<T> {
		const queue = this._core.runQueue;
		const previous = queue.get(key) ?? Promise.resolve();
		const run = previous.then(fn, fn);
		const tail = run.catch(() => undefined);
		queue.set(key, tail);
		void tail.then(() => {
			if (queue.get(key) === tail) queue.delete(key);
		});
		return run;
	}

	private async inferNow(
		key: VisionModelKey,
		frame: VisionFrame,
		input?: readonly [number, number],
	): Promise<{ outputs: VisionTensorOutputs; transform: InputTransform }> {
		const session = await this.session(key);
		const desc = VISION_MODELS[key];
		if (!isPreprocessFamily(desc.family))
			throw new Error(
				`VisionRunner cannot run '${desc.family}' models; use the family's own tracker (e.g. Sam2Tracker or Sam3Tracker from @framefields/vision-sam).`,
			);
		const spec = preprocessSpec(desc.family, input ?? desc.input);
		const dims = [1, 3, spec.height, spec.width];
		let image: VisionImageInput;
		if (isGpuImage(frame)) {
			if (this._core.provider.device === frame.device) {
				// Same device as the sessions: preprocess on the GPU, no readback.
				const buffer = this.gpuInput(key, frame.device, spec);
				const transform = FrameTensorPipeline.for(frame.device).toTensor(
					frame,
					spec,
					buffer,
				);
				const outputs = await session.run({
					name: "input",
					data: EMPTY_TENSOR,
					gpuBuffer: buffer,
					dims,
				});
				return { outputs, transform };
			}
			image = await readTexturePixels(frame);
		} else {
			image = frame;
		}
		const size = 3 * spec.width * spec.height;
		const scratchByKey = this._core.tensorScratch;
		let scratch = scratchByKey.get(key);
		if (!scratch || scratch.length < size) {
			scratch = new Float32Array(size);
			scratchByKey.set(key, scratch);
		}
		const { tensor, transform } = imageToTensor(image, spec, scratch);
		const outputs = await session.run({ name: "input", data: tensor, dims });
		return { outputs, transform };
	}

	private gpuInput(
		key: VisionModelKey,
		device: GPUDevice,
		spec: ReturnType<typeof preprocessSpec>,
	): GPUBuffer {
		const inputs = this._core.gpuInputs;
		const size = FrameTensorPipeline.tensorBytes(spec);
		const current = inputs.get(key);
		if (current?.device === device && current.buffer.size >= size) {
			return current.buffer;
		}
		current?.buffer.destroy();
		const buffer = device.createBuffer({
			label: `vision_input_${key}`,
			size,
			usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
		});
		inputs.set(key, { device, buffer });
		return buffer;
	}

	private session(key: VisionModelKey): Promise<VisionSession> {
		const { sessions, store, provider } = this._core;
		let promise = sessions.get(key);
		if (!promise) {
			promise = store
				.ensure(key)
				.then((bytes) =>
					provider.createSession(
						VISION_MODELS[key].family === "rtmdet-ins"
							? rtmdetInsBody(bytes)
							: bytes,
					),
				)
				.catch((err: unknown) => {
					// Drop both the session cache entry AND the cached file: a parse
					// failure means the on-disk bytes are corrupt — the next call must
					// re-download instead of replaying the same bad file.
					sessions.delete(key);
					void store.evict(key);
					throw err;
				});
			sessions.set(key, promise);
		}
		return promise;
	}
}

const EMPTY_TENSOR = new Float32Array(0);

interface InstancePass {
	readonly transform: InputTransform;
	readonly outputs: RtmdetInsOutputs;
}

/**
 * The RTMDet-Ins graph up to its head (see `RTMDET_INS_HEAD`): no NMS or
 * full-size masks in the graph, so the GPU part has fixed shapes and only
 * ~9 MB comes back. Models without those tensors run end to end.
 */
function rtmdetInsBody(bytes: Uint8Array): Uint8Array {
	try {
		return extractSubgraph(bytes, Object.values(RTMDET_INS_HEAD));
	} catch {
		return bytes;
	}
}

/** Outputs of the end2end graph: NMS done, one input-sized mask per detection. */
function end2endPassOutputs(
	outputs: VisionTensorOutputs,
	key: VisionModelKey,
): RtmdetInsOutputs {
	const dets = output(outputs, "dets", key);
	const masks = output(outputs, "masks", key);
	return {
		dets: dets.data as Float32Array,
		labels: output(outputs, "labels", key).data,
		masks: masks.data as Float32Array,
		count: dets.dims[1] ?? 0,
		maskHeight: masks.dims[masks.dims.length - 2] ?? 0,
		maskWidth: masks.dims[masks.dims.length - 1] ?? 0,
	};
}

/** Outputs of the head: selection here, mask logits only for detections asked for. */
function headPassOutputs(
	outputs: VisionTensorOutputs,
	key: VisionModelKey,
	transform: InputTransform,
): RtmdetInsOutputs {
	const scores = output(outputs, RTMDET_INS_HEAD.scores, key);
	const features = output(outputs, RTMDET_INS_HEAD.maskFeatures, key);
	const head: RtmdetInsHeadOutputs = {
		scores: scores.data as Float32Array,
		boxes: output(outputs, RTMDET_INS_HEAD.boxes, key).data as Float32Array,
		kernels: output(outputs, RTMDET_INS_HEAD.kernels, key).data as Float32Array,
		maskFeatures: features.data as Float32Array,
		anchors: scores.dims[1] ?? 0,
		classes: scores.dims[2] ?? 0,
		maskHeight: features.dims[2] ?? 0,
		maskWidth: features.dims[3] ?? 0,
		inputWidth: transform.inputWidth,
		inputHeight: transform.inputHeight,
	};
	const picked = selectInstances(head);
	const dets = new Float32Array(picked.length * 5);
	const labels = new Int32Array(picked.length);
	picked.forEach(({ anchor, classIndex, score }, i) => {
		dets.set(head.boxes.subarray(anchor * 4, anchor * 4 + 4), i * 5);
		dets[i * 5 + 4] = score;
		labels[i] = classIndex;
	});
	// Shared by every layer that reads this pass: each plane is computed once.
	const planes = new Map<number, Float32Array>();
	return {
		dets,
		labels,
		count: picked.length,
		maskWidth: head.maskWidth,
		maskHeight: head.maskHeight,
		maskLogits: true,
		maskPlane: (index) => {
			let plane = planes.get(index);
			if (!plane) {
				plane = maskLogits(head, (picked[index] as { anchor: number }).anchor);
				planes.set(index, plane);
			}
			return plane;
		},
	};
}

/** Size plus a 64-bit hash of the pixels: equal keys mean the same picture. */
function contentKey(image: VisionImageInput): string {
	const { data } = image;
	const words = data.byteOffset % 4 === 0 ? data.byteLength >>> 2 : 0;
	const view = new Uint32Array(data.buffer, data.byteOffset, words);
	let h1 = 0x811c9dc5;
	let h2 = 0x01000193 ^ data.byteLength;
	for (let i = 0; i < words; i++) {
		const v = view[i] as number;
		h1 = Math.imul(h1 ^ v, 0x9e3779b1);
		h2 = Math.imul(h2 ^ v, 0x85ebca77) ^ (h1 >>> 15);
	}
	for (let i = words * 4; i < data.length; i++) {
		h1 = Math.imul(h1 ^ (data[i] as number), 0x9e3779b1);
	}
	return `${image.width}x${image.height}:${(h1 >>> 0).toString(36)}:${(h2 >>> 0).toString(36)}`;
}

function output(
	outputs: VisionTensorOutputs,
	name: string,
	key: VisionModelKey,
): VisionTensor {
	const tensor = outputs[name];
	if (!tensor) {
		throw new Error(
			`Vision model '${key}' returned no '${name}' output (got: ${Object.keys(outputs).join(", ") || "none"}). ` +
				`Expected the ${VISION_MODELS[key].source}.`,
		);
	}
	return tensor;
}