Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/**
* Audio2Face-3D playback — drive a loaded GLB from an A2F blendshape track.
*
* NVIDIA Audio2Face-3D (api/_lib/a2f-nvidia.js) returns a per-frame ARKit-52
* blendshape track aligned to the spoken audio's timeline:
*
* { fps, blendShapeNames: ["EyeBlinkLeft", …, "JawOpen", …],
* frames: [{ t: <seconds>, w: [<weight per name>] }, …] }
*
* This class maps that track onto whatever morph-target convention the avatar's
* GLB actually ships — the SAME problem glb-canonicalize.js solves for bones,
* solved here for face morphs by reusing the cross-format vocabulary in
* arkit-blendshapes.js:
*
* • ARKit / RPM / Avaturn / MetaHuman (jawOpen, JawOpen, jaw_open, …)
* → driven DIRECTLY: each A2F shape writes to its canonical morph.
* • VRM / VRoid (A, I, U, E, O / Aa, Ih, Ou, Ee, Oh, Blink, emotions)
* and Oculus / Meta visemes (viseme_aa, viseme_O, …)
* → driven by DERIVING each expression's activation from the A2F ARKit
* frame (the inverse of arkit-blendshapes' VRM/Oculus→ARKit maps), so a
* rig that exposes only monolithic vowel shapes still lip-syncs.
*
* The avatar plays the ORIGINAL audio; `update(audioTimeSec)` samples the track
* by the audio element's currentTime (interpolating between the 30 fps A2F
* frames for smooth motion at display refresh), so the lips track the real
* voice. No rig allowlist: an unknown convention degrades to "no coverage", and
* the caller falls back to amplitude lipsync — never a frozen face.
*/
import {
canonicalARKitName,
VRM_TO_ARKIT,
OCULUS_TO_ARKIT,
ARKIT_GROUPS,
} from './arkit-blendshapes.js';
// Canonical ARKit shapes that move the mouth/jaw — used to decide whether a mesh
// already has DIRECT mouth coverage (so we don't also drive its vowel morphs and
// double-stack the lips).
const MOUTH_SHAPES = new Set([...ARKIT_GROUPS.jaw, ...ARKIT_GROUPS.mouth]);
// Expression morph names we know how to derive from an ARKit frame. Built from
// the VRM + Oculus maps so the set stays in lockstep with arkit-blendshapes.js.
const EXPRESSION_COMPONENTS = buildExpressionComponents();
function buildExpressionComponents() {
const out = new Map(); // lowercased morph name → { arkitName: weight }
for (const [name, components] of Object.entries(VRM_TO_ARKIT)) {
out.set(name.toLowerCase(), components);
}
for (const [name, components] of Object.entries(OCULUS_TO_ARKIT)) {
out.set(name.toLowerCase(), components);
}
return out;
}
export class A2FPlayer {
constructor() {
/** @type {Array<{mesh:any, index:number, canonical:string}>} */
this._direct = [];
/** @type {Array<{mesh:any, index:number, components:Record<string,number>, denom:number}>} */
this._expression = [];
this._track = null; // { fps, names, frames, canonForName: (string|null)[] }
this._cursor = 0;
this._lastTime = -1;
this._attached = false;
}
/**
* Bind to a freshly loaded GLB scene. Re-callable after a model hot-swap.
* @param {import('three').Object3D} root
*/
attach(root) {
this._direct = [];
this._expression = [];
this._attached = false;
if (!root || typeof root.traverse !== 'function') return;
const meshes = [];
root.traverse((node) => {
if (node.isMesh && node.morphTargetDictionary && node.morphTargetInfluences) {
meshes.push(node);
}
});
for (const mesh of meshes) {
const dict = mesh.morphTargetDictionary;
const directHere = [];
for (const [morphName, index] of Object.entries(dict)) {
const canonical = canonicalARKitName(morphName);
if (canonical) directHere.push({ mesh, index, canonical });
}
const hasDirectMouth = directHere.some((b) => MOUTH_SHAPES.has(b.canonical));
this._direct.push(...directHere);
// Only derive vowel/viseme expressions for a mesh that lacks direct
// mouth morphs — otherwise we'd fight the ARKit mouth shapes.
if (!hasDirectMouth) {
for (const [morphName, index] of Object.entries(dict)) {
const components = EXPRESSION_COMPONENTS.get(String(morphName).toLowerCase());
if (!components) continue;
const denom = Object.values(components).reduce((s, w) => s + w, 0);
if (denom > 0) this._expression.push({ mesh, index, components, denom });
}
}
}
this._attached = true;
}
/** True when at least one morph is wired and a track is loaded. */
hasCoverage() {
return this._attached && (this._direct.length > 0 || this._expression.length > 0);
}
/** Diagnostics: which morphs are wired and by which path. */
describe() {
return {
direct: this._direct.map((b) => `${b.mesh.name}:${b.canonical}`),
expression: this._expression.map((b) => `${b.mesh.name}:#${b.index}`),
frames: this._track?.frames.length ?? 0,
fps: this._track?.fps ?? 0,
};
}
/**
* Load an A2F animation track. Precomputes the canonical ARKit name for each
* blendshape column so per-frame sampling does no string work.
* @param {{ fps:number, blendShapeNames:string[], frames:Array<{t:number,w:number[]}> }} track
*/
setTrack(track) {
if (!track || !Array.isArray(track.frames) || !track.frames.length) {
this._track = null;
return;
}
const names = Array.isArray(track.blendShapeNames) ? track.blendShapeNames : [];
this._track = {
fps: track.fps || 30,
names,
frames: track.frames,
canonForName: names.map((n) => canonicalARKitName(n)),
};
this._cursor = 0;
this._lastTime = -1;
}
get duration() {
const f = this._track?.frames;
return f && f.length ? f[f.length - 1].t : 0;
}
/**
* Sample the track at `timeSec` (the playing audio's currentTime) and write
* the resulting weights onto the bound morphs. Safe before attach/setTrack
* (no-op). Interpolates between the two bracketing A2F frames.
*
* @param {number} timeSec
*/
update(timeSec) {
if (!this._attached || !this._track) return;
const frames = this._track.frames;
const t = Number(timeSec) || 0;
// Forward playback advances the cursor; a backward seek (replay) rewinds it.
if (t < this._lastTime) this._cursor = 0;
this._lastTime = t;
let i = this._cursor;
while (i < frames.length - 1 && frames[i + 1].t <= t) i++;
this._cursor = i;
const cur = frames[i];
const nxt = frames[i + 1];
let blend = 0;
if (nxt && nxt.t > cur.t) blend = Math.min(1, Math.max(0, (t - cur.t) / (nxt.t - cur.t)));
// Build the canonical ARKit frame (name → interpolated weight) once, then
// fan out to direct + derived bindings.
const arkit = this._arkit || (this._arkit = Object.create(null));
for (const k in arkit) arkit[k] = 0;
const { canonForName } = this._track;
const cw = cur.w;
const nw = nxt ? nxt.w : null;
for (let c = 0; c < canonForName.length; c++) {
const canon = canonForName[c];
if (!canon) continue;
const a = cw[c] || 0;
const b = nw ? (nw[c] || 0) : a;
arkit[canon] = a + (b - a) * blend;
}
for (const bind of this._direct) {
bind.mesh.morphTargetInfluences[bind.index] = arkit[bind.canonical] || 0;
}
for (const bind of this._expression) {
let sum = 0;
for (const name in bind.components) sum += (arkit[name] || 0) * bind.components[name];
bind.mesh.morphTargetInfluences[bind.index] = clamp01(sum / bind.denom);
}
}
/**
* Last sampled weight for a canonical ARKit shape (e.g. 'jawOpen') — for live
* activity meters / diagnostics. Returns 0 before the first update().
* @param {string} canonical
*/
currentWeight(canonical) {
return (this._arkit && this._arkit[canonical]) || 0;
}
/** Settle every bound morph to rest. Call when playback ends/stops. */
reset() {
for (const bind of this._direct) {
if (bind.mesh.morphTargetInfluences) bind.mesh.morphTargetInfluences[bind.index] = 0;
}
for (const bind of this._expression) {
if (bind.mesh.morphTargetInfluences) bind.mesh.morphTargetInfluences[bind.index] = 0;
}
this._cursor = 0;
this._lastTime = -1;
}
dispose() {
this.reset();
this._direct = [];
this._expression = [];
this._track = null;
this._attached = false;
}
}
function clamp01(n) {
const v = Number(n);
if (!Number.isFinite(v)) return 0;
if (v < 0) return 0;
if (v > 1) return 1;
return v;
}
// Pure helper exported for tests: derive an expression morph's activation from a
// canonical ARKit frame given its component weights (the inverse of the
// VRM/Oculus→ARKit maps).
export function deriveExpressionWeight(arkitFrame, components) {
let sum = 0;
let denom = 0;
for (const name in components) {
const w = components[name];
denom += w;
sum += (arkitFrame[name] || 0) * w;
}
return denom > 0 ? clamp01(sum / denom) : 0;
}