Skip to content

Code-first video & image, rendered natively on WebGPU.

7.1 KB
Raw Blame
/**
 * Image → NCHW float tensor preprocessing, plus the inverse mapping back to source pixels.
 *
 * Each model family was exported with its own preprocessing contract:
 *
 * | Model       | Resize                       | Pad  | Channels | Values                     |
 * | ----------- | ---------------------------- | ---- | -------- | -------------------------- |
 * | RTMDet-Ins  | keep ratio, content top-left | 114  | BGR      | (v − mean) / std (ImageNet) |
 * | RTMO        | keep ratio, content centered | 114  | BGR      | raw 0..255                 |
 * | Selfie      | stretch to input             | —    | RGB      | v / 255                    |
 * | Depth Any.  | stretch, aspect kept by size | —    | RGB      | (v − mean) / std (ImageNet) |
 *
 * Getting any of these wrong does not crash — it silently degrades accuracy — so the specs
 * live here and every runner path goes through `imageToTensor`. Input sizes differ per model
 * (RTMO-t is 416², the rest of RTMO 640²), so they come from the registry, never from here.
 */

export interface RgbaImage {
	readonly data: Uint8ClampedArray | Uint8Array;
	readonly width: number;
	readonly height: number;
}

export interface PreprocessSpec {
	/** Model input size in pixels. */
	readonly width: number;
	readonly height: number;
	/** `topLeft` / `center` keep aspect ratio and pad; `stretch` fills the input. */
	readonly fit: "topLeft" | "center" | "stretch";
	readonly channels: "rgb" | "bgr";
	/** Per-channel mean, in output channel order, on the 0..255 scale. */
	readonly mean: readonly [number, number, number];
	/** Per-channel std, in output channel order, on the 0..255 scale. */
	readonly std: readonly [number, number, number];
	/** Padding value on the 0..255 scale (normalized like any other pixel). */
	readonly padValue: number;
}

/** Maps model-input pixels back to source pixels: `src = (input − offset) / scale`. */
export interface InputTransform {
	readonly scaleX: number;
	readonly scaleY: number;
	readonly offsetX: number;
	readonly offsetY: number;
	readonly inputWidth: number;
	readonly inputHeight: number;
}

const IMAGENET_BGR_MEAN = [103.53, 116.28, 123.675] as const;
const IMAGENET_BGR_STD = [57.375, 57.12, 58.395] as const;

type Normalization = Omit<PreprocessSpec, "width" | "height">;

/** Per-family normalization; the input size always comes from the model's registry entry. */
export const PREPROCESS_BY_FAMILY: Readonly<
	Record<"rtmdet-ins" | "rtmo" | "selfie" | "depth-anything", Normalization>
> = {
	"rtmdet-ins": {
		fit: "topLeft",
		channels: "bgr",
		mean: IMAGENET_BGR_MEAN,
		std: IMAGENET_BGR_STD,
		padValue: 114,
	},
	rtmo: {
		fit: "center",
		channels: "bgr",
		mean: [0, 0, 0],
		std: [1, 1, 1],
		padValue: 114,
	},
	selfie: {
		fit: "stretch",
		channels: "rgb",
		mean: [0, 0, 0],
		std: [255, 255, 255],
		padValue: 0,
	},
	"depth-anything": {
		fit: "stretch",
		channels: "rgb",
		mean: [123.675, 116.28, 103.53],
		std: [58.395, 57.12, 57.375],
		padValue: 0,
	},
};

export type PreprocessFamily = keyof typeof PREPROCESS_BY_FAMILY;

/** True for the families `VisionRunner` can preprocess and decode itself. */
export function isPreprocessFamily(family: string): family is PreprocessFamily {
	return Object.hasOwn(PREPROCESS_BY_FAMILY, family);
}

export function preprocessSpec(
	family: PreprocessFamily,
	input: readonly [number, number],
): PreprocessSpec {
	return { ...PREPROCESS_BY_FAMILY[family], width: input[0], height: input[1] };
}

export function computeInputTransform(
	srcW: number,
	srcH: number,
	spec: Pick<PreprocessSpec, "width" | "height" | "fit">,
): InputTransform {
	if (spec.fit === "stretch") {
		return {
			scaleX: spec.width / srcW,
			scaleY: spec.height / srcH,
			offsetX: 0,
			offsetY: 0,
			inputWidth: spec.width,
			inputHeight: spec.height,
		};
	}
	const scale = Math.min(spec.width / srcW, spec.height / srcH);
	const contentW = Math.round(srcW * scale);
	const contentH = Math.round(srcH * scale);
	const center = spec.fit === "center";
	return {
		scaleX: scale,
		scaleY: scale,
		offsetX: center ? (spec.width - contentW) / 2 : 0,
		offsetY: center ? (spec.height - contentH) / 2 : 0,
		inputWidth: spec.width,
		inputHeight: spec.height,
	};
}

/** Model-input point → source pixel point. */
export function toSourcePoint(
	x: number,
	y: number,
	t: InputTransform,
): { x: number; y: number } {
	return { x: (x - t.offsetX) / t.scaleX, y: (y - t.offsetY) / t.scaleY };
}

/**
 * Resizes RGBA → NCHW float32 `[1, 3, height, width]` with bilinear sampling, padding,
 * channel reordering and normalization per `spec`. Reuses `out` when large enough
 * (hot-path frame loop).
 */
export function imageToTensor(
	image: RgbaImage,
	spec: PreprocessSpec,
	out?: Float32Array,
): { tensor: Float32Array; transform: InputTransform } {
	const { width: srcW, height: srcH, data: src } = image;
	const { width: W, height: H } = spec;
	const transform = computeInputTransform(srcW, srcH, spec);
	const { scaleX, scaleY, offsetX, offsetY } = transform;

	const plane = W * H;
	if (!out || out.length < 3 * plane) {
		out = new Float32Array(3 * plane);
	}

	// Source channel feeding each output plane (RGBA byte offsets).
	const order = spec.channels === "bgr" ? [2, 1, 0] : [0, 1, 2];
	const [m0, m1, m2] = spec.mean;
	const inv0 = 1 / spec.std[0];
	const inv1 = 1 / spec.std[1];
	const inv2 = 1 / spec.std[2];
	const pad0 = (spec.padValue - m0) * inv0;
	const pad1 = (spec.padValue - m1) * inv1;
	const pad2 = (spec.padValue - m2) * inv2;
	const [c0, c1, c2] = order;

	const x0 = Math.ceil(offsetX);
	const y0 = Math.ceil(offsetY);
	const x1 = Math.min(W, Math.floor(offsetX + srcW * scaleX)) - 1;
	const y1 = Math.min(H, Math.floor(offsetY + srcH * scaleY)) - 1;

	for (let y = 0; y < H; y++) {
		const rowBase = y * W;
		const paddedRow = y < y0 || y > y1;
		// Pixel-center sampling keeps stretch/letterbox resizes unbiased.
		const sy = Math.max(0, (y + 0.5 - offsetY) / scaleY - 0.5);
		const iy0 = Math.min(srcH - 1, Math.floor(sy));
		const iy1 = Math.min(srcH - 1, iy0 + 1);
		const fy = sy - iy0;
		for (let x = 0; x < W; x++) {
			const o = rowBase + x;
			if (paddedRow || x < x0 || x > x1) {
				out[o] = pad0;
				out[plane + o] = pad1;
				out[2 * plane + o] = pad2;
				continue;
			}
			const sx = Math.max(0, (x + 0.5 - offsetX) / scaleX - 0.5);
			const ix0 = Math.min(srcW - 1, Math.floor(sx));
			const ix1 = Math.min(srcW - 1, ix0 + 1);
			const fx = sx - ix0;
			const i00 = (iy0 * srcW + ix0) * 4;
			const i10 = (iy0 * srcW + ix1) * 4;
			const i01 = (iy1 * srcW + ix0) * 4;
			const i11 = (iy1 * srcW + ix1) * 4;
			const w00 = (1 - fx) * (1 - fy);
			const w10 = fx * (1 - fy);
			const w01 = (1 - fx) * fy;
			const w11 = fx * fy;
			out[o] =
				(src[i00 + c0] * w00 +
					src[i10 + c0] * w10 +
					src[i01 + c0] * w01 +
					src[i11 + c0] * w11 -
					m0) *
				inv0;
			out[plane + o] =
				(src[i00 + c1] * w00 +
					src[i10 + c1] * w10 +
					src[i01 + c1] * w01 +
					src[i11 + c1] * w11 -
					m1) *
				inv1;
			out[2 * plane + o] =
				(src[i00 + c2] * w00 +
					src[i10 + c2] * w10 +
					src[i01 + c2] * w01 +
					src[i11 + c2] * w11 -
					m2) *
				inv2;
		}
	}

	return { tensor: out, transform };
}