Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/**
* ARKit 52-blendshape vocabulary + cross-format mapping.
*
* ARKit defines the de-facto industry-standard set of 52 facial blendshape
* names (mouth, eye, brow, cheek, nose, jaw, tongue). three.ws standardizes
* on these names because:
*
* 1. Most modern avatar pipelines (Avaturn, Apple Live Link, MetaHuman,
* Wolf3D, Hyprface, Polywink) export with these names already.
* 2. They're orthogonal — overlaying a smile on top of a jaw-open works
* cleanly; in contrast, VRM's monolithic "A"/"I"/"U"/"E"/"O" shapes
* conflict when blended.
* 3. They map cleanly to Preston-Blair / Disney 12-viseme phoneme sets,
* which is what real-time lipsync drivers want.
*
* This module is *pure data + helpers*: no DOM, no three.js, no async I/O.
* Anything that touches a loaded GLB lives in avatar-morph-target.js and
* imports from here.
*/
// ── Canonical 52 ARKit blendshape names ─────────────────────────────────
export const ARKIT_NAMES = [
// brow
'browDownLeft', 'browDownRight', 'browInnerUp', 'browOuterUpLeft', 'browOuterUpRight',
// cheek
'cheekPuff', 'cheekSquintLeft', 'cheekSquintRight',
// eye
'eyeBlinkLeft', 'eyeBlinkRight',
'eyeLookDownLeft', 'eyeLookDownRight',
'eyeLookInLeft', 'eyeLookInRight',
'eyeLookOutLeft', 'eyeLookOutRight',
'eyeLookUpLeft', 'eyeLookUpRight',
'eyeSquintLeft', 'eyeSquintRight',
'eyeWideLeft', 'eyeWideRight',
// jaw
'jawForward', 'jawLeft', 'jawOpen', 'jawRight',
// mouth
'mouthClose', 'mouthDimpleLeft', 'mouthDimpleRight',
'mouthFrownLeft', 'mouthFrownRight',
'mouthFunnel',
'mouthLeft', 'mouthLowerDownLeft', 'mouthLowerDownRight',
'mouthPressLeft', 'mouthPressRight',
'mouthPucker',
'mouthRight',
'mouthRollLower', 'mouthRollUpper',
'mouthShrugLower', 'mouthShrugUpper',
'mouthSmileLeft', 'mouthSmileRight',
'mouthStretchLeft', 'mouthStretchRight',
'mouthUpperUpLeft', 'mouthUpperUpRight',
// nose
'noseSneerLeft', 'noseSneerRight',
// tongue
'tongueOut',
];
export const ARKIT_GROUPS = {
brow: ['browDownLeft', 'browDownRight', 'browInnerUp', 'browOuterUpLeft', 'browOuterUpRight'],
cheek: ['cheekPuff', 'cheekSquintLeft', 'cheekSquintRight'],
eye: [
'eyeBlinkLeft', 'eyeBlinkRight',
'eyeLookDownLeft', 'eyeLookDownRight',
'eyeLookInLeft', 'eyeLookInRight',
'eyeLookOutLeft', 'eyeLookOutRight',
'eyeLookUpLeft', 'eyeLookUpRight',
'eyeSquintLeft', 'eyeSquintRight',
'eyeWideLeft', 'eyeWideRight',
],
jaw: ['jawForward', 'jawLeft', 'jawOpen', 'jawRight'],
mouth: [
'mouthClose', 'mouthDimpleLeft', 'mouthDimpleRight',
'mouthFrownLeft', 'mouthFrownRight',
'mouthFunnel',
'mouthLeft', 'mouthLowerDownLeft', 'mouthLowerDownRight',
'mouthPressLeft', 'mouthPressRight',
'mouthPucker', 'mouthRight',
'mouthRollLower', 'mouthRollUpper',
'mouthShrugLower', 'mouthShrugUpper',
'mouthSmileLeft', 'mouthSmileRight',
'mouthStretchLeft', 'mouthStretchRight',
'mouthUpperUpLeft', 'mouthUpperUpRight',
],
nose: ['noseSneerLeft', 'noseSneerRight'],
tongue: ['tongueOut'],
};
// ── VRM expression → ARKit weighted shape map ──────────────────────────
//
// VRM (Pixiv / Niconi standard) ships monolithic expression names. Map each
// to a *weighted combination* of orthogonal ARKit shapes so we can render
// VRM-tagged emotions on ARKit rigs without losing fidelity.
export const VRM_TO_ARKIT = {
// Vowel mouth shapes — VRM canonical
Aa: { jawOpen: 1.0 },
A: { jawOpen: 1.0 },
Ah: { jawOpen: 1.0 },
Ih: { mouthStretchLeft: 0.7, mouthStretchRight: 0.7, mouthSmileLeft: 0.25, mouthSmileRight: 0.25 },
I: { mouthStretchLeft: 0.7, mouthStretchRight: 0.7, mouthSmileLeft: 0.25, mouthSmileRight: 0.25 },
Ou: { mouthFunnel: 0.7, mouthPucker: 0.4, jawOpen: 0.2 },
U: { mouthFunnel: 0.7, mouthPucker: 0.4, jawOpen: 0.2 },
Ee: { mouthSmileLeft: 0.55, mouthSmileRight: 0.55, mouthStretchLeft: 0.35, mouthStretchRight: 0.35, jawOpen: 0.2 },
E: { mouthSmileLeft: 0.55, mouthSmileRight: 0.55, mouthStretchLeft: 0.35, mouthStretchRight: 0.35, jawOpen: 0.2 },
Oh: { jawOpen: 0.55, mouthFunnel: 0.55 },
O: { jawOpen: 0.55, mouthFunnel: 0.55 },
// VRM emotions
Joy: { mouthSmileLeft: 0.9, mouthSmileRight: 0.9, cheekSquintLeft: 0.4, cheekSquintRight: 0.4, eyeSquintLeft: 0.3, eyeSquintRight: 0.3 },
Happy: { mouthSmileLeft: 0.9, mouthSmileRight: 0.9, cheekSquintLeft: 0.4, cheekSquintRight: 0.4, eyeSquintLeft: 0.3, eyeSquintRight: 0.3 },
Angry: { browDownLeft: 0.9, browDownRight: 0.9, noseSneerLeft: 0.4, noseSneerRight: 0.4, mouthFrownLeft: 0.5, mouthFrownRight: 0.5 },
Sad: { browInnerUp: 0.7, mouthFrownLeft: 0.6, mouthFrownRight: 0.6, eyeLookDownLeft: 0.3, eyeLookDownRight: 0.3 },
Surprised: { eyeWideLeft: 0.9, eyeWideRight: 0.9, browInnerUp: 0.7, browOuterUpLeft: 0.6, browOuterUpRight: 0.6, jawOpen: 0.5 },
Fun: { mouthSmileLeft: 0.7, mouthSmileRight: 0.7, cheekSquintLeft: 0.3, cheekSquintRight: 0.3 },
// VRM blinks (commonly named just 'Blink')
Blink: { eyeBlinkLeft: 1.0, eyeBlinkRight: 1.0 },
BlinkL: { eyeBlinkLeft: 1.0 },
BlinkR: { eyeBlinkRight: 1.0 },
};
// ── Oculus / Meta visemes → ARKit ───────────────────────────────────────
//
// Oculus ship a 15-viseme set used by their Lipsync SDK. Many avatars produced
// for Meta apps carry these morph names verbatim.
export const OCULUS_TO_ARKIT = {
viseme_sil: {},
viseme_PP: { mouthClose: 0.9, mouthPressLeft: 0.4, mouthPressRight: 0.4 },
viseme_FF: { mouthLowerDownLeft: 0.4, mouthLowerDownRight: 0.4, mouthUpperUpLeft: 0.2, mouthUpperUpRight: 0.2 },
viseme_TH: { jawOpen: 0.25, tongueOut: 0.5 },
viseme_DD: { jawOpen: 0.3, tongueOut: 0.15 },
viseme_kk: { jawOpen: 0.35 },
viseme_CH: { mouthFunnel: 0.45, mouthPucker: 0.4, jawOpen: 0.25 },
viseme_SS: { mouthStretchLeft: 0.3, mouthStretchRight: 0.3 },
viseme_nn: { jawOpen: 0.2, tongueOut: 0.15 },
viseme_RR: { mouthFunnel: 0.4, jawOpen: 0.25 },
viseme_aa: { jawOpen: 0.95 },
viseme_E: { mouthSmileLeft: 0.55, mouthSmileRight: 0.55, jawOpen: 0.25 },
viseme_I: { mouthStretchLeft: 0.65, mouthStretchRight: 0.65 },
viseme_O: { jawOpen: 0.5, mouthFunnel: 0.55 },
viseme_U: { mouthFunnel: 0.7, mouthPucker: 0.45 },
};
// ── Preston-Blair phoneme codes → ARKit ─────────────────────────────────
//
// The classic 9-viseme cartoon-animation set, useful as a fallback when only
// rough phoneme estimates are available (e.g. from amplitude-band heuristics
// in lipsync-driver.js).
export const PHONEME_TO_ARKIT = {
// closed lips (M, B, P)
MBP: { mouthClose: 0.95, mouthPressLeft: 0.35, mouthPressRight: 0.35 },
// open vowel (A, I as in "cat")
AI: { jawOpen: 0.75, mouthSmileLeft: 0.2, mouthSmileRight: 0.2 },
// open-front (E as in "ten")
E: { jawOpen: 0.45, mouthSmileLeft: 0.45, mouthSmileRight: 0.45 },
// open-back (O as in "boat")
O: { jawOpen: 0.55, mouthFunnel: 0.55 },
// rounded (U as in "boot", W)
U: { mouthFunnel: 0.7, mouthPucker: 0.45 },
// L (tongue up)
L: { jawOpen: 0.3, tongueOut: 0.25 },
// F / V (lip-teeth)
FV: { mouthLowerDownLeft: 0.45, mouthLowerDownRight: 0.45, mouthUpperUpLeft: 0.25, mouthUpperUpRight: 0.25 },
// W / Q (puckered)
WQ: { mouthPucker: 0.7, mouthFunnel: 0.45 },
// silence / rest
REST: {},
};
// ── Name resolution ─────────────────────────────────────────────────────
//
// Different riggers spell ARKit names with different casing and separators
// (jawOpen, JawOpen, jaw_open, Jaw_Open, jaw-open, ARKit_JawOpen, …). This
// helper normalizes any incoming string to its canonical lowercase token so
// lookups across pipelines work without per-rigger branches.
function normalizeToken(name) {
return String(name || '')
.replace(/^arkit[_-]?/i, '')
.replace(/[_\-\s]+/g, '')
.toLowerCase();
}
const _canonicalIndex = new Map(ARKIT_NAMES.map((n) => [normalizeToken(n), n]));
/**
* Resolve any spelling of a morph name to its canonical ARKit name, or null
* if it doesn't match a known ARKit shape. Case- and separator-insensitive.
*/
export function canonicalARKitName(name) {
if (!name) return null;
return _canonicalIndex.get(normalizeToken(name)) || null;
}
/**
* Given a morph dictionary from a loaded mesh (mapping morph name → index),
* return a Map from canonical ARKit name → morph index. Drops any morphs
* that don't correspond to a known ARKit shape.
*/
export function indexARKitMorphs(morphDict) {
const result = new Map();
if (!morphDict) return result;
for (const [name, idx] of Object.entries(morphDict)) {
const canonical = canonicalARKitName(name);
if (canonical && !result.has(canonical)) {
result.set(canonical, idx);
}
}
return result;
}
/**
* Coverage report for an indexed mesh — useful for showing operators which
* groups a particular GLB supports (and to gate UI features that depend on
* specific shape availability, e.g. eye-look requires the eye group).
*/
export function coverageOf(arkitIndex) {
const report = {};
let total = 0;
let found = 0;
for (const [group, names] of Object.entries(ARKIT_GROUPS)) {
const present = names.filter((n) => arkitIndex.has(n));
report[group] = {
present: present.length,
total: names.length,
missing: names.filter((n) => !arkitIndex.has(n)),
};
total += names.length;
found += present.length;
}
report.overall = { present: found, total, ratio: total ? found / total : 0 };
return report;
}
// ── Weighted-shape application ───────────────────────────────────────────
/**
* Resolve a "high-level shape" — VRM name, Oculus viseme, Preston-Blair
* phoneme code, OR a raw ARKit name — to a `{ arkitName: weight }` map.
*
* Returns an empty object for unknown inputs.
*/
export function resolveShape(name) {
if (!name) return {};
// Direct ARKit hit?
const canon = canonicalARKitName(name);
if (canon) return { [canon]: 1 };
// Indirect maps — case-sensitive on purpose, the keys above are the
// canonical spellings emitted by each upstream pipeline.
if (name in VRM_TO_ARKIT) return { ...VRM_TO_ARKIT[name] };
if (name in OCULUS_TO_ARKIT) return { ...OCULUS_TO_ARKIT[name] };
if (name in PHONEME_TO_ARKIT) return { ...PHONEME_TO_ARKIT[name] };
// Case-insensitive retry on the indirect maps so callers don't have to
// match the exact studly-caps spelling.
const lower = String(name).toLowerCase();
for (const map of [VRM_TO_ARKIT, OCULUS_TO_ARKIT, PHONEME_TO_ARKIT]) {
for (const k of Object.keys(map)) {
if (k.toLowerCase() === lower) return { ...map[k] };
}
}
return {};
}
/**
* Blend any number of shape inputs (each either a string label or an
* already-resolved `{ arkitName: weight }` map) into a single ARKit map.
*
* Same name across inputs: takes the MAX weight (not sum). This mirrors how
* cinematic facial-rig software blends visemes — additive blending of
* orthogonal mouth shapes produces broken poses; max keeps the strongest
* activation per channel.
*/
export function blendShapes(...inputs) {
const out = {};
for (const input of inputs) {
if (!input) continue;
const resolved = typeof input === 'string' ? resolveShape(input) : input;
for (const [k, v] of Object.entries(resolved)) {
const canon = canonicalARKitName(k);
if (!canon) continue;
const w = clamp01(v);
if (w > (out[canon] || 0)) out[canon] = w;
}
}
return out;
}
function clamp01(n) {
const v = Number(n);
if (!Number.isFinite(v)) return 0;
if (v < 0) return 0;
if (v > 1) return 1;
return v;
}