Skip to content

Code-first video & image, rendered natively on WebGPU.

6.4 KB
Raw Blame
export * from "@framefields/core";

import type { DetectedObject, ObjectBoundingBox } from "@framefields/core";

export type VisionInputSource =
	| string
	| { inputHandleId?: string; id?: string };

/** RGBA pixel buffer accepted by the runner. */
export interface VisionImageInput {
	readonly data: Uint8ClampedArray | Uint8Array;
	readonly width: number;
	readonly height: number;
}

/**
 * A frame that is already on the GPU: an 8-bit RGBA/BGRA texture of `device`.
 * When the runner's sessions run on the same device (see `setVisionDevice`),
 * preprocessing happens on the GPU and the frame never comes back to the CPU;
 * otherwise the runner reads it back.
 */
export interface VisionGpuImage {
	readonly texture: GPUTexture;
	readonly device: GPUDevice;
	readonly width: number;
	readonly height: number;
}

export type VisionFrame = VisionImageInput | VisionGpuImage;

export function isGpuImage(image: VisionFrame): image is VisionGpuImage {
	return (image as Partial<VisionGpuImage>).texture !== undefined;
}

/** Instance segmentation mask aligned to the ORIGINAL source frame. */
export interface InstanceMask {
	readonly category: string;
	readonly mask: Uint8Array; // frame-sized, continuous alpha 0..255
	readonly width: number;
	readonly height: number;
	readonly area: number; // px
	readonly coverage: number; // area / (width * height)
	/** Index of the detection this mask belongs to (see segmentation result detections). */
	readonly detectionIndex: number;
	/** Assigned by the caller (renderer) after temporal tracking — matches TrackedObject.trackId. */
	readonly trackId?: number;
}

export interface SegmentationResult {
	readonly detections: readonly DetectedObject[];
	readonly masks: readonly InstanceMask[];
}

/**
 * Relative depth of a whole frame (Depth Anything), 0 = farthest … 1 = nearest,
 * row-major at `width × height` (the model's resolution, the frame's aspect).
 */
export interface DepthMap {
	readonly data: Float32Array;
	readonly width: number;
	readonly height: number;
}

/** How {@link VisionRunner.refineMatte} resolves a mask's edge. */
export interface MatteRefineOptions {
	/**
	 * Pixels either side of the mask's edge the model may change (default about
	 * 1.2% of the frame's short side: 13 px at 1080p). Wider reaches loose hair
	 * the mask missed; narrower keeps more of the mask's own edge.
	 */
	readonly band?: number;
	/** Longest side, in model px, the region around the edge is scaled to (default 1024). */
	readonly resolution?: number;
}

/** A mask with its edge resolved to soft alpha, aligned to the source frame. */
export interface RefinedMatte {
	/** Frame-sized alpha, 0..255. */
	readonly mask: Uint8Array;
	readonly width: number;
	readonly height: number;
	/**
	 * Per pixel, the index of the nearest pixel inside the input mask (−1 when
	 * the mask is empty), so alpha past the edge can follow labelled objects.
	 */
	readonly nearest: Int32Array;
}

/** Person-vs-background alpha (Selfie Segmenter), aligned to the ORIGINAL source frame. */
export interface PersonMatte {
	readonly mask: Uint8Array; // frame-sized, continuous alpha 0..255
	readonly width: number;
	readonly height: number;
	readonly coverage: number; // mean alpha, 0..1
}

export interface PoseKeypoint {
	readonly x: number;
	readonly y: number;
	readonly visibility: number; // 0..1
}

export interface PosePerson {
	readonly score: number;
	readonly boundingBox: ObjectBoundingBox;
	/** 17 keypoints in COCO-17 order (see COCO17_KEYPOINTS). */
	readonly keypoints: readonly PoseKeypoint[];
	readonly trackId?: number;
}

export interface PoseResult {
	readonly people: readonly PosePerson[];
}

/**
 * Deterministic, serializable snapshot of one frame's signals.
 * Built synchronously from the per-frame caches, so it is safe to call inside a frame hook.
 */
export interface VisionSummary {
	readonly frame: number;
	readonly objects: ReadonlyArray<{
		readonly trackId: number;
		readonly category: string;
		readonly score: number;
		/** Source-space pixel center, `[x, y]`. */
		readonly center: readonly [number, number];
		readonly speed: number;
		readonly active: boolean;
	}>;
	readonly classes: readonly string[];
	/** Per-class instance count for the frame (only non-zero classes present). */
	readonly masks: Readonly<Record<string, number>>;
}

/** Per-track aggregate in an analysis report. */
export interface VisionAnalysisTrackSummary {
	readonly trackId: number;
	readonly category: string;
	/** Inclusive frame range the track was active over, `[start, end]`. */
	readonly frames: readonly [number, number];
	readonly speedMeanPxS: number;
	/** Center path sampled every `pathStride` frames, `[x, y]` pairs. */
	readonly centerPath: ReadonlyArray<readonly [number, number]>;
}

export interface VisionAnalysisClassSummary {
	readonly framesPresent: number;
	readonly totalFrames: number;
	readonly maxConfidence: number;
	readonly tracks: number;
}

export interface VisionAnalysisReport {
	readonly source: string;
	readonly totalFrames: number;
	readonly fps: number;
	readonly durationSec: number;
	readonly tasks: readonly string[];
	/** What was fetched, and how long it took — surfaced so agents can budget cold starts. */
	readonly modelDownloads: Readonly<
		Record<string, { readonly bytes: number; readonly ms: number }>
	>;
	readonly inferenceMs: number;
	readonly trackCount: number;
	readonly classes: Readonly<Record<string, VisionAnalysisClassSummary>>;
	readonly tracks: readonly VisionAnalysisTrackSummary[];
	readonly masks: Readonly<Record<string, { readonly meanCoverage: number }>>;
}

export interface PinToLandmarkOptions {
	readonly offsetX?: number;
	readonly offsetY?: number;
	readonly offsetZ?: number;
	readonly trackRotation?: boolean;
	readonly scaleWithPerspective?: boolean;
}

export interface LandmarkCoordinateSignals {
	readonly x: import("@framefields/core").ProgrammaticSignal;
	readonly y: import("@framefields/core").ProgrammaticSignal;
	readonly z: import("@framefields/core").ProgrammaticSignal;
	readonly screenX: import("@framefields/core").ProgrammaticSignal;
	readonly screenY: import("@framefields/core").ProgrammaticSignal;
}

export type ObjectAnchorName =
	| "center"
	| "topCenter"
	| "bottomCenter"
	| "topLeft"
	| "topRight"
	| "centerLeft"
	| "centerRight"
	| "bottomLeft"
	| "bottomRight";

export interface PinToObjectOptions extends PinToLandmarkOptions {
	readonly anchor?: ObjectAnchorName;
	readonly matchWidth?: boolean;
	readonly matchHeight?: boolean;
	readonly smoothFrames?: number;
	readonly hideWhenLost?: boolean;
}