Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

6.4 KB
Raw Blame
// SignInput — webcam ASL fingerspelling capture → text.
//
// The input half of the platform's sign-language loop (the output half is
// src/sign-speech.js). Opens the user's camera, runs MediaPipe's Holistic
// landmarker in the browser, assembles per-frame feature rows in the exact
// column order the recognition worker publishes (GET /api/asl-recognition),
// and on stop POSTs the frame matrix for transcription. Raw video never
// leaves the device — only landmark coordinates.
//
// Usage:
//   const input = new SignInput({ onState });
//   await input.start();          // camera on, capturing
//   const { text } = await input.stop();  // camera off, transcribed
//
// The recognizer reads continuous fingerspelling (Kaggle-2023 ASLFR
// 1st-place model — see workers/model-asl-recognition/README.md for the
// full provenance chain).

import { FilesetResolver, HolisticLandmarker } from '@mediapipe/tasks-vision';

import { modelUrl, visionWasmBase } from './shared/mediapipe-assets.js';
// Runtime location comes from the shared resolver (see face-mocap.js).
const MODEL_URL =
	modelUrl('holistic_landmarker/holistic_landmarker/float16/latest/holistic_landmarker.task');

const COLUMN_RE = /^([xyz])_(face|left_hand|right_hand|pose)_(\d+)$/;

// Result-field lookup per schema group. Holistic returns arrays-of-arrays
// (one entry per detected person; we take the first).
const GROUP_FIELDS = {
	face: 'faceLandmarks',
	left_hand: 'leftHandLandmarks',
	right_hand: 'rightHandLandmarks',
	pose: 'poseLandmarks',
};

/** Parse the worker's column names once into fast extractors. */
export function buildColumnExtractors(columns) {
	return columns.map((name) => {
		const m = COLUMN_RE.exec(name);
		if (!m) throw new Error(`unrecognized landmark column: ${name}`);
		const [, coord, group, idxStr] = m;
		return { field: GROUP_FIELDS[group], coord, idx: parseInt(idxStr, 10) };
	});
}

/** One Holistic result → a schema row ([number|null] per column). */
export function extractRow(result, extractors) {
	return extractors.map(({ field, coord, idx }) => {
		const lm = result?.[field]?.[0]?.[idx];
		if (!lm) return null;
		const v = lm[coord];
		return Number.isFinite(v) ? v : null;
	});
}

export class SignInput {
	/**
	 * @param {{
	 *   apiBase?: string,
	 *   onState?: (state: 'loading'|'capturing'|'transcribing'|'idle', detail?: object) => void,
	 *   maxSeconds?: number,
	 *   targetFps?: number,
	 * }} [opts]
	 */
	constructor(opts = {}) {
		this.apiBase = opts.apiBase ?? '/api/asl-recognition';
		this.onState = opts.onState ?? (() => {});
		this.maxSeconds = opts.maxSeconds ?? 20;
		this.targetFps = opts.targetFps ?? 24;

		this._landmarker = null;
		this._extractors = null;
		this._minFrames = 8;
		this._video = null;
		this._stream = null;
		this._frames = [];
		this._raf = 0;
		this._lastSampleT = 0;
		this._lastVideoTime = -1;
		this.capturing = false;
	}

	/** Camera preview element (attach it to the page while capturing). */
	get videoElement() {
		return this._video;
	}

	async _ensureReady() {
		if (this._landmarker && this._extractors) return;
		this.onState('loading');
		const [schemaRes, fileset] = await Promise.all([
			fetch(this.apiBase, { headers: { accept: 'application/json' } }),
			visionWasmBase().then((base) => FilesetResolver.forVisionTasks(base)),
		]);
		if (!schemaRes.ok) {
			const body = await schemaRes.json().catch(() => ({}));
			throw new Error(body.message || `sign recognition unavailable (${schemaRes.status})`);
		}
		const schema = await schemaRes.json();
		this._extractors = buildColumnExtractors(schema.columns);
		this._minFrames = schema.min_frames ?? 8;
		this._landmarker = await HolisticLandmarker.createFromOptions(fileset, {
			baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },
			runningMode: 'VIDEO',
		});
	}

	/** Open the camera and start collecting frames. */
	async start() {
		if (this.capturing) return;
		await this._ensureReady();
		this._stream = await navigator.mediaDevices.getUserMedia({
			video: { width: 640, height: 480, facingMode: 'user' },
			audio: false,
		});
		const video = document.createElement('video');
		video.playsInline = true;
		video.muted = true;
		video.srcObject = this._stream;
		await video.play();
		this._video = video;
		this._frames = [];
		this._lastSampleT = 0;
		this._lastVideoTime = -1;
		this.capturing = true;
		this.onState('capturing', { frames: 0 });
		const step = (t) => {
			if (!this.capturing) return;
			this._raf = requestAnimationFrame(step);
			if (t - this._lastSampleT < 1000 / this.targetFps) return;
			if (video.currentTime === this._lastVideoTime) return;
			this._lastVideoTime = video.currentTime;
			this._lastSampleT = t;
			const result = this._landmarker.detectForVideo(video, performance.now());
			this._frames.push(extractRow(result, this._extractors));
			if (this._frames.length % 12 === 0) {
				this.onState('capturing', { frames: this._frames.length });
			}
			if (this._frames.length >= this.maxSeconds * this.targetFps) {
				// Hard cap — auto-stop into transcription rather than dropping data.
				this.stop().catch(() => {});
			}
		};
		this._raf = requestAnimationFrame(step);
	}

	/** Stop the camera and transcribe what was captured. */
	async stop() {
		if (!this.capturing) return { text: '', frames: 0 };
		this.capturing = false;
		cancelAnimationFrame(this._raf);
		this._stream?.getTracks().forEach((tr) => tr.stop());
		this._stream = null;
		const frames = this._frames;
		this._frames = [];
		this._video = null;
		if (frames.length < this._minFrames) {
			this.onState('idle');
			throw new Error('Not enough signing captured — hold the sign a moment longer.');
		}
		this.onState('transcribing', { frames: frames.length });
		try {
			const res = await fetch(this.apiBase, {
				method: 'POST',
				headers: { 'content-type': 'application/json' },
				body: JSON.stringify({ frames }),
			});
			const out = await res.json().catch(() => ({}));
			if (!res.ok) throw new Error(out.message || `transcription failed (${res.status})`);
			return {
				text: out.text ?? '',
				raw: out.raw ?? out.text ?? '',
				confidence: out.confidence ?? null,
				cleaned: !!out.cleaned,
				frames: frames.length,
				ms: out.ms,
			};
		} finally {
			this.onState('idle');
		}
	}

	/** Abandon capture without transcribing. */
	cancel() {
		this.capturing = false;
		cancelAnimationFrame(this._raf);
		this._stream?.getTracks().forEach((tr) => tr.stop());
		this._stream = null;
		this._frames = [];
		this._video = null;
		this.onState('idle');
	}
}