Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/**
* /api/asl-recognition — webcam ASL fingerspelling → text.
*
* GET /api/asl-recognition
* → { columns: [390 landmark column names], max_frames, min_frames }
* The feature schema the browser must assemble per video frame
* (MediaPipe Holistic landmark selection, served by the worker).
*
* POST /api/asl-recognition { frames: [[390 numbers|null]…], clean?: boolean }
* → { text, raw, cleaned, confidence, frames, ms }
* `text` is the LLM-cleaned decode (send clean:false to skip that pass
* and get the worker's raw output); `confidence` is the recognizer's
* mean per-character softmax, which the UI uses to warn on a bad read.
*
* Thin authenticated proxy to workers/model-asl-recognition (Kaggle-2023
* ASLFR 1st-place model, Apache-2.0 weights, FSboard CC BY 4.0 corpus).
* Landmarks are extracted client-side (src/sign-input.js), so raw video
* never reaches the platform — only pose coordinates.
*/
import { cors, json, method, readJson, wrap, rateLimited } from './_lib/http.js';
import { limits, clientIp } from './_lib/rate-limit.js';
import { fetchUpstream } from './_lib/upstream-fetch.js';
// One breaker for the worker: the schema read is idempotent and retried, a
// transcribe is a GPU inference and gets a single attempt with a long deadline.
const WORKER_BREAKER = 'asl-recognition-worker';
const SCHEMA_TIMEOUT_MS = 10_000;
const TRANSCRIBE_TIMEOUT_MS = 60_000;
// 1500 frames × 390 floats serializes to ~10–20 MB; cap the body above that.
const MAX_BODY_BYTES = 24_000_000;
function config() {
const base = (process.env.GCP_ASL_RECOGNITION_URL || '').replace(/\/$/, '');
const key = process.env.GCP_RECONSTRUCTION_KEY || '';
return base && key ? { base, key } : null;
}
function unconfigured(res) {
return json(res, 503, {
error: 'unconfigured',
message:
'Sign recognition is not configured. Set GCP_ASL_RECOGNITION_URL and ' +
'GCP_RECONSTRUCTION_KEY on the API service.',
});
}
let _schemaCache = null;
let _schemaAt = 0;
const SCHEMA_TTL_MS = 10 * 60 * 1000;
export default wrap(async (req, res) => {
if (cors(req, res, { methods: 'GET,POST,OPTIONS' })) return;
if (!method(req, res, ['GET', 'POST'])) return;
const cfg = config();
if (!cfg) return unconfigured(res);
if (req.method === 'GET') {
if (_schemaCache && Date.now() - _schemaAt < SCHEMA_TTL_MS) {
return json(res, 200, _schemaCache);
}
const r = await fetchUpstream(`${cfg.base}/schema`, {
headers: { authorization: `Bearer ${cfg.key}` },
}, { name: WORKER_BREAKER, timeoutMs: SCHEMA_TIMEOUT_MS, attempts: 3, okWhen: () => true });
if (!r.ok) return json(res, 502, { error: 'worker_error', status: r.status });
_schemaCache = await r.json();
_schemaAt = Date.now();
return json(res, 200, _schemaCache);
}
const rl = await limits.aslTranscribeIp(clientIp(req));
if (!rl.success) return rateLimited(res, rl, 'Sign recognition limit reached. Try again shortly.');
const body = await readJson(req, MAX_BODY_BYTES).catch(() => null);
if (!body || !Array.isArray(body.frames)) {
return json(res, 400, { error: 'bad_request', message: 'Body must be { frames: [[…]] }.' });
}
const r = await fetchUpstream(`${cfg.base}/transcribe`, {
method: 'POST',
headers: {
'content-type': 'application/json',
authorization: `Bearer ${cfg.key}`,
},
body: JSON.stringify({ frames: body.frames }),
}, { name: WORKER_BREAKER, timeoutMs: TRANSCRIBE_TIMEOUT_MS, attempts: 1, okWhen: () => true });
const out = await r.json().catch(() => ({}));
if (!r.ok) {
return json(res, r.status === 400 ? 400 : 502, {
error: r.status === 400 ? 'bad_frames' : 'worker_error',
message: out.detail || out.message || `worker returned ${r.status}`,
});
}
out.raw = out.text ?? '';
if (out.raw && body.clean !== false) {
out.text = await cleanTranscription(out.raw);
out.cleaned = out.text !== out.raw;
}
return json(res, 200, out);
});
/**
* Webcam fingerspelling decodes at a 10–20% character error rate; a
* constrained LLM pass recovers most of it ("esteredouble" → intended word).
* Fail-open: any LLM problem returns the raw decode untouched.
*/
async function cleanTranscription(raw) {
try {
const { llmComplete, llmConfigured } = await import('./_lib/llm.js');
if (!llmConfigured()) return raw;
const cleaned = await llmComplete({
system:
'You correct noisy ASL fingerspelling transcriptions. The input is a ' +
'machine decode of someone spelling letter by letter into a webcam; ' +
'roughly 1 in 6 characters may be wrong, split, or merged. Return ONLY ' +
'the most likely intended text, preserving meaning and proper nouns. ' +
'No explanations, no quotes. If the input already looks right, return it unchanged.',
user: raw,
maxTokens: 120,
timeoutMs: 8_000,
});
const text = (cleaned || '').trim();
// Guard against LLM refusals/essays: a cleanup should stay in the same
// size class as the input.
if (!text || text.length > raw.length * 2 + 20) return raw;
return text;
} catch {
return raw;
}
}