Code-first video & image, rendered natively on WebGPU.
export * from "@framefields/core";
import type { DetectedObject, ObjectBoundingBox } from "@framefields/core";
export type VisionInputSource =
| string
| { inputHandleId?: string; id?: string };
/** RGBA pixel buffer accepted by the runner. */
export interface VisionImageInput {
readonly data: Uint8ClampedArray | Uint8Array;
readonly width: number;
readonly height: number;
}
/**
* A frame that is already on the GPU: an 8-bit RGBA/BGRA texture of `device`.
* When the runner's sessions run on the same device (see `setVisionDevice`),
* preprocessing happens on the GPU and the frame never comes back to the CPU;
* otherwise the runner reads it back.
*/
export interface VisionGpuImage {
readonly texture: GPUTexture;
readonly device: GPUDevice;
readonly width: number;
readonly height: number;
}
export type VisionFrame = VisionImageInput | VisionGpuImage;
export function isGpuImage(image: VisionFrame): image is VisionGpuImage {
return (image as Partial<VisionGpuImage>).texture !== undefined;
}
/** Instance segmentation mask aligned to the ORIGINAL source frame. */
export interface InstanceMask {
readonly category: string;
readonly mask: Uint8Array; // frame-sized, continuous alpha 0..255
readonly width: number;
readonly height: number;
readonly area: number; // px
readonly coverage: number; // area / (width * height)
/** Index of the detection this mask belongs to (see segmentation result detections). */
readonly detectionIndex: number;
/** Assigned by the caller (renderer) after temporal tracking — matches TrackedObject.trackId. */
readonly trackId?: number;
}
export interface SegmentationResult {
readonly detections: readonly DetectedObject[];
readonly masks: readonly InstanceMask[];
}
/**
* Relative depth of a whole frame (Depth Anything), 0 = farthest … 1 = nearest,
* row-major at `width × height` (the model's resolution, the frame's aspect).
*/
export interface DepthMap {
readonly data: Float32Array;
readonly width: number;
readonly height: number;
}
/** How {@link VisionRunner.refineMatte} resolves a mask's edge. */
export interface MatteRefineOptions {
/**
* Pixels either side of the mask's edge the model may change (default about
* 1.2% of the frame's short side: 13 px at 1080p). Wider reaches loose hair
* the mask missed; narrower keeps more of the mask's own edge.
*/
readonly band?: number;
/** Longest side, in model px, the region around the edge is scaled to (default 1024). */
readonly resolution?: number;
}
/** A mask with its edge resolved to soft alpha, aligned to the source frame. */
export interface RefinedMatte {
/** Frame-sized alpha, 0..255. */
readonly mask: Uint8Array;
readonly width: number;
readonly height: number;
/**
* Per pixel, the index of the nearest pixel inside the input mask (−1 when
* the mask is empty), so alpha past the edge can follow labelled objects.
*/
readonly nearest: Int32Array;
}
/** Person-vs-background alpha (Selfie Segmenter), aligned to the ORIGINAL source frame. */
export interface PersonMatte {
readonly mask: Uint8Array; // frame-sized, continuous alpha 0..255
readonly width: number;
readonly height: number;
readonly coverage: number; // mean alpha, 0..1
}
export interface PoseKeypoint {
readonly x: number;
readonly y: number;
readonly visibility: number; // 0..1
}
export interface PosePerson {
readonly score: number;
readonly boundingBox: ObjectBoundingBox;
/** 17 keypoints in COCO-17 order (see COCO17_KEYPOINTS). */
readonly keypoints: readonly PoseKeypoint[];
readonly trackId?: number;
}
export interface PoseResult {
readonly people: readonly PosePerson[];
}
/**
* Deterministic, serializable snapshot of one frame's signals.
* Built synchronously from the per-frame caches, so it is safe to call inside a frame hook.
*/
export interface VisionSummary {
readonly frame: number;
readonly objects: ReadonlyArray<{
readonly trackId: number;
readonly category: string;
readonly score: number;
/** Source-space pixel center, `[x, y]`. */
readonly center: readonly [number, number];
readonly speed: number;
readonly active: boolean;
}>;
readonly classes: readonly string[];
/** Per-class instance count for the frame (only non-zero classes present). */
readonly masks: Readonly<Record<string, number>>;
}
/** Per-track aggregate in an analysis report. */
export interface VisionAnalysisTrackSummary {
readonly trackId: number;
readonly category: string;
/** Inclusive frame range the track was active over, `[start, end]`. */
readonly frames: readonly [number, number];
readonly speedMeanPxS: number;
/** Center path sampled every `pathStride` frames, `[x, y]` pairs. */
readonly centerPath: ReadonlyArray<readonly [number, number]>;
}
export interface VisionAnalysisClassSummary {
readonly framesPresent: number;
readonly totalFrames: number;
readonly maxConfidence: number;
readonly tracks: number;
}
export interface VisionAnalysisReport {
readonly source: string;
readonly totalFrames: number;
readonly fps: number;
readonly durationSec: number;
readonly tasks: readonly string[];
/** What was fetched, and how long it took — surfaced so agents can budget cold starts. */
readonly modelDownloads: Readonly<
Record<string, { readonly bytes: number; readonly ms: number }>
>;
readonly inferenceMs: number;
readonly trackCount: number;
readonly classes: Readonly<Record<string, VisionAnalysisClassSummary>>;
readonly tracks: readonly VisionAnalysisTrackSummary[];
readonly masks: Readonly<Record<string, { readonly meanCoverage: number }>>;
}
export interface PinToLandmarkOptions {
readonly offsetX?: number;
readonly offsetY?: number;
readonly offsetZ?: number;
readonly trackRotation?: boolean;
readonly scaleWithPerspective?: boolean;
}
export interface LandmarkCoordinateSignals {
readonly x: import("@framefields/core").ProgrammaticSignal;
readonly y: import("@framefields/core").ProgrammaticSignal;
readonly z: import("@framefields/core").ProgrammaticSignal;
readonly screenX: import("@framefields/core").ProgrammaticSignal;
readonly screenY: import("@framefields/core").ProgrammaticSignal;
}
export type ObjectAnchorName =
| "center"
| "topCenter"
| "bottomCenter"
| "topLeft"
| "topRight"
| "centerLeft"
| "centerRight"
| "bottomLeft"
| "bottomRight";
export interface PinToObjectOptions extends PinToLandmarkOptions {
readonly anchor?: ObjectAnchorName;
readonly matchWidth?: boolean;
readonly matchHeight?: boolean;
readonly smoothFrames?: number;
readonly hideWhenLost?: boolean;
}