Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// Unified avatar loader + animation controller.
// ==============================================
// One entry point used by BOTH the corner companion and the full-page
// playground, so adding an avatar to the roster makes it work everywhere at
// once. Given a roster entry it loads the GLB and returns a controller with a
// single interface, setState('idle'|'walk'|'run'|'jump') + playWave() plus
// emotes() / playEmote(name) for on-command performances, no matter how the
// rig is animated underneath:
//
// • embedded rigs play the clips baked into the GLB (robot, fox, showpieces),
// with loose name matching that always falls back to the model's first clip
// so even a one-animation GLB keeps moving and never shows a bind pose.
// • shared rigs are driven by the retargeted shared clip library through
// AnimationManager (humanoids that ship no locomotion, or only a T-pose).
//
// The caller is responsible for framing/scaling the returned `model`.
import { AnimationMixer, LoopOnce, LoopRepeat } from 'three';
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
import { getMeshoptDecoder } from './meshopt.js';
import { AnimationManager } from './runtime.js';
import { resolveClipUrls } from './manifest.js';
import { log } from './log.js';
import { resolveAvatarUrl, DEFAULT_SHARED_CLIPS, DEFAULT_EMOTES } from '../roster.js';
const DEFAULT_WAVE_MS = 1500;
// A looping emote clip (dance, headbang) performs one capped pass before the
// controller settles back to its base state: a tap is a performance, not a
// permanent mode switch.
const EMOTE_MAX_MS = 8000;
// Baked-clip name candidates per emote for embedded rigs, matched with the same
// loose case-insensitive lookup as the locomotion states. RobotExpressive (the
// default mascot) ships Dance/Punch/Wave; rigs missing a clip simply don't
// list that emote, and the UI hides its button.
const EMBEDDED_EMOTE_CANDIDATES = {
dance: ['Dance'],
punch: ['Punch'],
backflip: ['Backflip', 'BackFlip', 'Back Flip', 'back_flip'],
wave: ['Wave'],
};
/**
* Resolve an emote map ({name: clip}) against the set of clips that actually
* exist, dropping emotes whose clip is missing so a UI can render only buttons
* that work. Pure; exported for tests.
*/
export function resolveEmotes(availableClipNames, emotes = DEFAULT_EMOTES) {
const available = new Set(availableClipNames);
const out = {};
for (const [name, clip] of Object.entries(emotes)) {
if (clip && available.has(clip)) out[name] = clip;
}
return out;
}
// Thrown by buildSharedController when the attached rig can't be driven by the
// shared clip library (no skinned humanoid skeleton). loadWalkAvatar catches it
// to recover to baked clips or the default rig — never a frozen T-pose.
const RIG_UNSUPPORTED = 'WALK_RIG_UNSUPPORTED';
let _loaderPromise = null;
async function makeLoader() {
// One meshopt-only GLTFLoader, reused across loads. Draco/KTX2 are never
// emitted by the bakes these avatars come from, so we skip those decoders.
if (!_loaderPromise) {
_loaderPromise = (async () => {
const loader = new GLTFLoader();
loader.setMeshoptDecoder(await getMeshoptDecoder());
return loader;
})();
}
return _loaderPromise;
}
// Coalesce concurrent requests for the same key into one underlying load.
// three.js's FileLoader keeps its own duplicate-request table (`loading[url]`
// in node_modules/three/src/loaders/FileLoader.js), but that table's cleanup
// has an edge where a second queued callback can see the entry already
// deleted and rethrow into a promise chain FileLoader itself never returns or
// awaits: an unhandled rejection no caller-side try/catch can intercept,
// because it is not the promise loadAsync() ever handed back to us. Never
// letting two of our own calls register against the same URL concurrently
// means FileLoader's table never holds more than one callback for it. Pure
// and loader-agnostic so it can be exercised directly in tests; exported for
// that reason (see walk-sdk/test/load-avatar.test.mjs).
const _inFlight = new Map();
export function coalesceLoad(key, start) {
const existing = _inFlight.get(key);
if (existing) return existing.then((value) => ({ value, owner: false }));
const promise = start();
_inFlight.set(key, promise);
// Separate listener from the one below: cleans up the table on either
// outcome without affecting what the caller below awaits. Needs its own
// .catch(): a bare .finally() with nothing after it is exactly the
// floating-promise shape this whole fix exists to avoid.
promise.finally(() => {
if (_inFlight.get(key) === promise) _inFlight.delete(key);
}).catch(() => {});
return promise.then((value) => ({ value, owner: true }));
}
// A joiner must never share the owner's live gltf.scene: it gets attached to
// its own rig and its own AnimationMixer/AnimationManager, and Object3D.add()
// reparents (steals) a child already attached elsewhere, which would yank the
// model out from under whichever caller mounted first.
function cloneGltfForJoiner(gltf) {
return { ...gltf, scene: gltf.scene.clone(true) };
}
/**
* Load a roster entry and build its controller.
* @returns {Promise<{ model: import('three').Object3D, controller: object, gltf: object }>}
*/
export async function loadWalkAvatar(entry, opts = {}) {
const {
assetBase = '',
apiBase = '',
manifestUrl = '/animations/manifest.json',
fallbackEntry = null,
waveMs = DEFAULT_WAVE_MS,
} = opts;
const loader = await makeLoader();
const url = resolveAvatarUrl(entry, { assetBase, apiBase });
if (!url) throw new Error(`walk: cannot resolve a GLB url for avatar "${entry?.id}"`);
let active = entry;
let gltf;
try {
const primary = await coalesceLoad(url, () => loader.loadAsync(url));
gltf = primary.owner ? primary.value : cloneGltfForJoiner(primary.value);
} catch (err) {
if (fallbackEntry && fallbackEntry.id !== entry.id) {
log.warn(
`avatar "${entry?.id}" failed to load — falling back to "${fallbackEntry.id}"`,
err?.message || err,
);
active = fallbackEntry;
const fallbackUrl = resolveAvatarUrl(fallbackEntry, { assetBase, apiBase });
const fallback = await coalesceLoad(fallbackUrl, () => loader.loadAsync(fallbackUrl));
gltf = fallback.owner ? fallback.value : cloneGltfForJoiner(fallback.value);
} else {
throw err;
}
}
const model = gltf.scene;
model.traverse((n) => {
if (n.isMesh) n.frustumCulled = false;
});
let controller;
try {
if (active.rig === 'shared') {
controller = await buildSharedController(model, active.clips || DEFAULT_SHARED_CLIPS, {
manifestUrl,
waveMs,
emotes: active.emotes,
});
} else {
controller = makeEmbeddedController(model, gltf.animations || [], active.clips || {}, {
waveMs,
});
}
} catch (err) {
// A `shared` entry whose GLB turns out NOT to be a retargetable humanoid
// (no skinned mesh, too few canonical bones) can't be driven by the shared
// clip library — left alone it would freeze in its bind/T-pose, which the
// platform forbids. Recover without ever showing a T-pose:
// 1. play whatever clips are baked into the GLB itself, else
// 2. fall back to the default rig (the caller's fallbackEntry).
if (err?.code === RIG_UNSUPPORTED) {
if (gltf.animations && gltf.animations.length) {
// Same expected-fallback path: baked clips are a designed recovery,
// not a failure worth a warning on every page view.
log.debug(
`avatar "${active.id}" isn't a retargetable humanoid — driving its ${gltf.animations.length} baked clip(s) instead`,
);
// The shared `clips` map names manifest clips, not embedded ones, so
// drop it and let the embedded matcher use generic names + first-clip
// fallback (which never freezes).
controller = makeEmbeddedController(model, gltf.animations, {}, { waveMs });
} else if (fallbackEntry && fallbackEntry.id !== active.id) {
// Expected, handled behavior (non-humanoid rigs fall back by design),
// so log at debug, not warn, to keep every page view's console clean.
log.debug(
`avatar "${active.id}" can't be animated (non-humanoid rig, no baked clips) — falling back to "${fallbackEntry.id}"`,
);
disposeModel(model);
// Recurse once into the default rig; clear fallbackEntry so a broken
// default can't loop. The default (robot) is an embedded rig anyway.
return loadWalkAvatar(fallbackEntry, { ...opts, fallbackEntry: null });
} else {
disposeModel(model);
throw err;
}
} else {
// Controller build failed after the GLB parsed — free the scene's GPU
// resources so a failed avatar load doesn't leak meshes/textures.
disposeModel(model);
throw err;
}
}
return { model, controller, gltf, entry: active };
}
// Release a parsed GLB scene's GPU resources (geometries, materials, textures)
// when a load is abandoned mid-way so a failed avatar build leaks nothing.
function disposeModel(model) {
model.traverse((n) => {
if (!n.isMesh) return;
n.geometry?.dispose?.();
const mats = Array.isArray(n.material) ? n.material : [n.material];
for (const m of mats) {
if (!m) continue;
for (const v of Object.values(m)) if (v && v.isTexture) v.dispose();
m.dispose?.();
}
});
}
// ── Embedded-clip controller (rig: 'embedded') ───────────────────────────────
function makeEmbeddedController(root, clips, overrides, { waveMs }) {
const mixer = new AnimationMixer(root);
const byName = (name) => clips.find((c) => c.name.toLowerCase() === String(name).toLowerCase());
const pick = (cands) => {
for (const n of cands) {
const c = byName(n);
if (c) return c;
}
return null;
};
const ov = (k) => (Array.isArray(overrides?.[k]) ? overrides[k] : []);
// Idle must always resolve to *something* animated, so a single-clip GLB
// (a lone walk or dance loop) never stalls in its bind/T-pose.
const idleClip = pick([...ov('idle'), 'Idle', 'idle']) || clips[0] || null;
const map = {
idle: idleClip,
walk: pick([...ov('walk'), 'Walking', 'Walk', 'walk']) || idleClip,
run: pick([...ov('run'), 'Running', 'Run', 'run', 'Walking', 'walk']) || idleClip,
jump: pick([...ov('jump'), 'Jump', 'jump', 'WalkJump']) || null,
wave: pick([...ov('wave'), 'Wave', 'wave']) || null,
};
// Emotes resolve against the same baked clips; a rig without a matching clip
// simply doesn't list that emote, so UIs can hide dead buttons.
const emoteClip = {};
for (const [name, cands] of Object.entries(EMBEDDED_EMOTE_CANDIDATES)) {
const clip = pick([...ov(name), ...cands]);
if (clip) emoteClip[name] = clip;
}
const action = {};
for (const [state, clip] of Object.entries(map)) {
if (!clip) continue;
const a = mixer.clipAction(clip);
a.enabled = true;
action[state] = a;
}
const emoteAction = {};
for (const [name, clip] of Object.entries(emoteClip)) {
const a = mixer.clipAction(clip);
a.enabled = true;
emoteAction[name] = a;
}
let base = 'idle';
let requested = 'idle';
let current = null;
let oneShot = false;
function fadeTo(a, { once = false, dur = 0.3 } = {}) {
if (!a) return;
a.reset();
a.setLoop(once ? LoopOnce : LoopRepeat, once ? 1 : Infinity);
a.clampWhenFinished = once;
a.fadeIn(dur).play();
if (current && current !== a) current.fadeOut(dur);
current = a;
}
function crossfade(name, opts) {
fadeTo(action[name] || action.idle, opts);
}
mixer.addEventListener('finished', () => {
if (oneShot) {
oneShot = false;
crossfade(base, { dur: 0.25 });
}
});
crossfade('idle', { dur: 0 });
return {
setState(next) {
if (next === requested) return;
requested = next;
if (next === 'jump') {
if (action.jump) {
oneShot = true;
crossfade('jump', { once: true, dur: 0.12 });
}
return;
}
base = next;
if (!oneShot) crossfade(base, { dur: 0.22 });
},
// Names this rig can actually perform; render only these as buttons.
emotes() {
return Object.keys(emoteAction);
},
// Play an emote once, then settle back to the current base state. A tap
// mid-emote switches to the new one. Returns false when unsupported.
playEmote(name) {
const a = emoteAction[name];
if (!a) return false;
oneShot = true;
fadeTo(a, { once: true, dur: 0.2 });
// Safety net: if the 'finished' event is missed (clip stripped of its
// end key, etc.), still fall back to the base after the clip's length.
const len = a.getClip().duration * 1000 || waveMs;
clearTimeout(this._emoteGuard);
this._emoteGuard = setTimeout(() => {
if (oneShot) {
oneShot = false;
crossfade(base, { dur: 0.25 });
}
}, Math.min(len, EMOTE_MAX_MS) + 250);
return true;
},
playWave() {
if (oneShot) return;
this.playEmote('wave');
},
// Scale playback rate of every action (global mixer multiplier) so a walk
// cycle can be sped up/slowed to match actual travel speed and keep feet
// planted instead of skating. 1 = authored cadence. Survives crossfades.
setSpeed(scale) {
mixer.timeScale = scale > 0 ? scale : 1;
},
update(dt) {
mixer.update(dt);
},
dispose() {
clearTimeout(this._emoteGuard);
mixer.stopAllAction();
mixer.uncacheRoot(root);
},
};
}
// ── Shared retargeted-clip controller (rig: 'shared') ────────────────────────
async function buildSharedController(model, clips, { manifestUrl, waveMs, emotes }) {
const manager = new AnimationManager();
manager.attach(model);
// attach() decides synchronously whether this rig is a retargetable humanoid
// (skinned mesh + enough canonically-named bones). If not, the library can't
// drive it — bail before fetching the manifest so loadWalkAvatar can recover
// to baked clips or the default rig instead of leaving a silent bind pose.
if (!manager.supportsCanonicalClips()) {
manager.dispose();
const err = new Error(
'walk: rig is not a retargetable humanoid (no skinned skeleton) — cannot drive shared clips',
);
err.code = RIG_UNSUPPORTED;
throw err;
}
const resolved = {};
let resolvedEmotes = {};
const clipMs = new Map();
try {
const manifest = await fetch(manifestUrl, { cache: 'force-cache' }).then((r) => {
if (!r.ok) throw new Error(`HTTP ${r.status} fetching animation manifest`);
return r.json();
}).then((defs) => resolveClipUrls(defs, manifestUrl));
const available = new Set(manifest.map((d) => d.name));
for (const d of manifest) clipMs.set(d.name, (Number(d.duration) || 0) * 1000);
// Resolve each requested clip to one that actually exists; unknown names fall
// back to idle so the controller never asks the manager for a missing clip.
for (const [state, name] of Object.entries({ ...DEFAULT_SHARED_CLIPS, ...clips })) {
resolved[state] = available.has(name) ? name : null;
}
resolved.idle = resolved.idle || (available.has('idle') ? 'idle' : null);
if (!resolved.idle) throw new Error('animation manifest missing an idle clip');
for (const k of Object.keys(resolved)) if (!resolved[k]) resolved[k] = resolved.idle;
// Emotes never fall back to idle: a missing clip drops the emote instead,
// so every rendered emote button visibly performs.
resolvedEmotes = resolveEmotes(available, { ...DEFAULT_EMOTES, ...(emotes || {}) });
const wanted = new Set([...Object.values(resolved), ...Object.values(resolvedEmotes)]);
manager.setAnimationDefs(manifest.filter((d) => wanted.has(d.name)));
await manager.loadAll();
} catch (err) {
// The GLB loaded but the animation manifest/clips didn't — release the
// AnimationManager's mixer/retarget state instead of leaking it.
manager.dispose();
throw err;
}
let base = 'idle';
let waveTimer = null;
const fade = (name, dur) => Promise.resolve(manager.crossfadeTo(name, dur)).catch(() => {});
const clipFor = (state) => resolved[state] || resolved.idle;
fade(resolved.idle, 0);
return {
setState(next) {
if (next === base) return;
base = next;
if (!waveTimer) fade(clipFor(next), next === 'jump' ? 0.12 : 0.3);
},
// Names this rig can actually perform; render only these as buttons.
emotes() {
return Object.keys(resolvedEmotes);
},
// Play an emote once (loops get one capped pass), then settle back to the
// base state. A tap mid-emote switches to the new one. False = unsupported.
playEmote(name) {
const clip = resolvedEmotes[name];
if (!clip || clip === resolved.idle) return false;
clearTimeout(waveTimer);
fade(clip, 0.25);
const len = clipMs.get(clip) || waveMs;
waveTimer = setTimeout(() => {
waveTimer = null;
fade(clipFor(base), 0.3);
}, Math.min(len, EMOTE_MAX_MS) + 150);
return true;
},
playWave() {
if (waveTimer) return;
if (this.playEmote('wave')) return;
// No wave emote resolved (host stripped it): fall back to the state
// map's wave clip so nav-waves keep working exactly as before.
const w = resolved.wave;
if (!w || w === resolved.idle) return;
fade(w, 0.25);
waveTimer = setTimeout(() => {
waveTimer = null;
fade(clipFor(base), 0.3);
}, waveMs);
},
// Scale playback rate of the active clip (global mixer multiplier) so a walk
// cycle can be sped up/slowed to match actual travel speed and keep feet
// planted instead of skating. 1 = authored cadence. Survives crossfades.
setSpeed(scale) {
if (manager.mixer) manager.mixer.timeScale = scale > 0 ? scale : 1;
},
update(dt) {
manager.update(dt);
},
dispose() {
clearTimeout(waveTimer);
manager.dispose();
},
};
}