Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// mocap-studio.js — record, save, and replay face-mocap clips on a three.ws avatar.
//
// Pipeline:
// 1. Load an avatar (handle-resolved, or the user's primary public avatar).
// 2. Initialize FaceMocap (MediaPipe FaceLandmarker @ ~30Hz, GPU delegate).
// 3. Calibrate to a neutral pose (subtracts resting-face baseline).
// 4. Record / stop — uses FaceMocap's built-in clip buffer.
// 5. Replay any saved clip via FaceMocap.playback().
// 6. Save to /api/mocap/clips with metadata.
// 7. List + replay anyone's public clips, plus your own private ones.
import { Viewer } from './viewer.js';
import { FaceMocap } from './face-mocap.js';
import { IdleAnimation } from './idle-animation.js';
import { log } from './shared/log.js';
// Saving and deleting a clip are cookie-session mutations, and the endpoints
// gate those on a single-use X-CSRF-Token (api/_lib/csrf.js). A bare fetch()
// never carries one, so every save and delete from this page answered 403
// csrf_missing in production. apiFetch mints and attaches the token per call.
import { apiFetch, noteSession } from './api.js';
const $ = (sel) => document.querySelector(sel);
const stage = $('#stage');
const pip = $('#webcam-pip');
const recPill = $('#rec-pill');
const recTime = $('#rec-time');
const timer = $('#timer');
const startCamBtn = $('#start-cam');
const calibrateBtn = $('#calibrate');
const recordBtn = $('#record');
const replayBtn = $('#replay');
const discardBtn = $('#discard');
const saveBtn = $('#save-clip');
const downloadBtn = $('#download-clip');
const nameInput = $('#clip-name');
const descInput = $('#clip-desc');
const tagsInput = $('#clip-tags');
const visibilityInput = $('#clip-visibility');
const handleInput = $('#avatar-handle');
const loadAvatarBtn = $('#load-avatar');
const avatarInfo = $('#avatar-info');
const clipListEl = $('#clip-list');
const authWarning = $('#auth-warning');
const state = {
viewer: null,
avatar: null, // { id, modelUrl, name, handle }
mocap: null,
idle: null,
lastClip: null,
recording: false,
tStart: 0,
rafId: 0,
playback: null,
signedIn: false,
username: null, // signed-in user's handle (lowercased) — used to gate avatar_id on save
};
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
// ── Bring up the Three.js scene ────────────────────────────────────────────
const viewer = new Viewer(stage, { kiosk: true });
viewer.renderer?.setClearAlpha(0);
if (viewer.scene) viewer.scene.background = null;
state.viewer = viewer;
// Reasonable defaults
handleInput.value = '';
nameInput.value = '';
// ── Auth state ────────────────────────────────────────────────────────────
fetchMe().then((user) => {
state.signedIn = !!user;
state.username = user?.username ? String(user.username).toLowerCase() : null;
authWarning.classList.toggle('visible', !user);
});
// ── Boot: try to load user's primary public avatar ────────────────────────
bootAvatar().catch((err) => {
avatarInfo.textContent = err?.message || 'No avatar loaded — enter a handle above.';
});
refreshClipList();
// ── Button wiring ─────────────────────────────────────────────────────────
loadAvatarBtn.addEventListener('click', () => loadByHandle());
handleInput.addEventListener('keydown', (e) => {
if (e.key === 'Enter') {
e.preventDefault();
loadByHandle();
}
});
startCamBtn.addEventListener('click', () => startMocap().catch(showErr));
calibrateBtn.addEventListener('click', () => {
if (!state.mocap) return;
const ok = state.mocap.calibrate();
calibrateBtn.textContent = ok ? 'Recalibrate' : 'Calibrate (no face yet)';
setTimeout(() => (calibrateBtn.textContent = 'Calibrate neutral'), 1400);
});
recordBtn.addEventListener('click', () => {
if (state.recording) stopRecording();
else startRecording();
});
replayBtn.addEventListener('click', () => {
if (state.lastClip) replay(state.lastClip);
});
discardBtn.addEventListener('click', () => {
state.lastClip = null;
updateButtons();
timer.textContent = '00.0s';
});
saveBtn.addEventListener('click', () => saveClip().catch(showErr));
downloadBtn.addEventListener('click', () => downloadClip());
// ── Functions ──────────────────────────────────────────────────────────────
async function fetchMe() {
try {
const r = await fetch('/api/auth/me', { credentials: 'include' });
if (!r.ok) {
noteSession(false);
return null;
}
const data = await r.json().catch(() => null);
noteSession(!!data?.user);
return data?.user || null;
} catch {
return null;
}
}
// True only when the loaded avatar is one the signed-in user owns — i.e. it was
// resolved from their own handle and carries a real UUID. The save endpoint
// rejects an avatar_id the caller doesn't own (404) and the non-UUID 'default'
// sentinel (400 uuid-validation), so attaching the id in any other case would
// make "Save clip" fail. When false we simply omit avatar_id: the clip still
// saves, just without an avatar association.
function ownsLoadedAvatar() {
const a = state.avatar;
if (!a || !a.id || !UUID_RE.test(a.id)) return false;
if (!state.username || !a.handle) return false;
return String(a.handle).toLowerCase() === state.username;
}
async function bootAvatar() {
const params = new URLSearchParams(location.search);
const handle = params.get('handle') || params.get('h');
if (handle) {
handleInput.value = handle.replace(/^@/, '');
try {
await loadByHandle();
if (state.avatar) return;
} catch {}
return loadDefaultAvatar();
}
// Try the signed-in user's username — auto-load their public avatar.
const user = await fetchMe();
if (user?.username) {
handleInput.value = user.username;
try {
await loadByHandle();
if (state.avatar) return;
} catch {}
}
// Nothing to resolve (signed out, or no public avatar): drop in the
// platform's default avatar so the camera + mocap pipeline is usable now.
return loadDefaultAvatar();
}
// Bring a freshly loaded model into the scene and (re)wire idle + mocap to its
// skeleton. Shared by handle-resolved loads and the default-avatar fallback.
async function applyLoadedAvatar(modelUrl, meta) {
await viewer.load(modelUrl, '', new Map());
state.avatar = meta;
// If this is a real saved avatar (UUID id), surface its agent-wallet panel.
// Non-UUID stand-ins (e.g. the default mannequin) self-hide in the component.
const actions = $('#mc-actions');
if (actions && meta?.id && UUID_RE.test(meta.id)) {
actions.style.display = 'block';
actions.setAttribute('avatar-id', meta.id);
} else if (actions) {
actions.style.display = 'none';
actions.removeAttribute('avatar-id');
}
if (state.idle) state.idle.dispose();
if (!viewer._afterAnimateHooks) viewer._afterAnimateHooks = [];
if (state._idleHook) {
const idx = viewer._afterAnimateHooks.indexOf(state._idleHook);
if (idx !== -1) viewer._afterAnimateHooks.splice(idx, 1);
}
state.idle = new IdleAnimation({ getRoot: () => viewer.content, seed: meta.id });
state._idleHook = (dt) => state.idle?.update(dt);
viewer._afterAnimateHooks.push(state._idleHook);
if (state.mocap) reattachMocap();
else startCamBtn.disabled = false;
}
async function loadDefaultAvatar() {
avatarInfo.textContent = 'Loading default avatar…';
try {
await applyLoadedAvatar('/avatars/default.glb', {
id: 'default',
modelUrl: '/avatars/default.glb',
name: 'Default avatar',
handle: null,
});
avatarInfo.textContent = 'Default avatar · enter a handle above to load your own';
} catch (err) {
avatarInfo.textContent = err?.message || 'Failed to load the default avatar.';
}
}
async function loadByHandle() {
const raw = (handleInput.value || '').trim().replace(/^@/, '').toLowerCase();
if (!/^[a-z0-9_-]{3,30}$/.test(raw)) {
avatarInfo.textContent = `Invalid handle: ${raw || '(empty)'}`;
return;
}
avatarInfo.textContent = `Loading @${raw}…`;
try {
const r = await fetch(`/api/users/${encodeURIComponent(raw)}/avatar`);
if (!r.ok) throw new Error(`@${raw} has no public avatar`);
const data = await r.json();
const avatar = data.avatar;
await applyLoadedAvatar(avatar.model_url, {
id: avatar.id,
modelUrl: avatar.model_url,
name: data.user.display_name || data.user.username,
handle: data.user.username,
});
avatarInfo.textContent = `${state.avatar.name} · @${state.avatar.handle}`;
} catch (err) {
state.avatar = null;
avatarInfo.textContent = err?.message || 'Failed to load.';
}
}
async function startMocap() {
if (state.mocap) return;
if (!viewer.content) throw new Error('Load an avatar first.');
startCamBtn.disabled = true;
startCamBtn.textContent = 'Starting…';
const mocap = new FaceMocap();
await mocap.init();
const video = await mocap.startWebcam();
pip.srcObject = video.srcObject;
pip.classList.add('live');
// Idle pauses saccade + blink while we drive the head.
state.idle?.setChannels({ saccade: false, blink: false });
reattachMocapRoot(mocap);
mocap.start();
state.mocap = mocap;
// Hook into viewer's per-frame loop for FaceMocap.update().
viewer._afterAnimateHooks.push(() => state.mocap?.update());
startCamBtn.textContent = 'Camera live';
startCamBtn.disabled = true;
calibrateBtn.disabled = false;
recordBtn.disabled = false;
}
function reattachMocap() {
if (!state.mocap || !viewer.content) return;
reattachMocapRoot(state.mocap);
}
function reattachMocapRoot(mocap) {
let head = null;
let neck = null;
viewer.content.traverse((node) => {
if (!node.isBone) return;
const canon = node.name
.replace(/^mixamorig:?/i, '')
.replace(/^[A-Za-z0-9]+[_:]/, '')
.toLowerCase();
if (!head && canon === 'head') head = node;
else if (!neck && canon === 'neck') neck = node;
});
mocap.attach(viewer.content, head || neck);
}
function startRecording() {
if (!state.mocap) return;
if (state.playback) {
state.playback.stop();
state.playback = null;
}
state.mocap.startRecording();
state.recording = true;
state.tStart = performance.now();
recPill.classList.add('on');
recordBtn.textContent = 'Stop';
recordBtn.classList.add('danger');
timer.classList.add('recording');
loopTimer();
}
function stopRecording() {
if (!state.mocap || !state.recording) return;
state.recording = false;
const clip = state.mocap.stopRecording();
state.lastClip = clip;
recPill.classList.remove('on');
timer.classList.remove('recording');
recordBtn.textContent = 'Record';
if (state.rafId) cancelAnimationFrame(state.rafId);
timer.textContent = formatTime(clip.duration || 0);
updateButtons();
}
function loopTimer() {
if (!state.recording) return;
const dt = (performance.now() - state.tStart) / 1000;
timer.textContent = formatTime(dt);
recTime.textContent = `${dt.toFixed(1)}s`;
state.rafId = requestAnimationFrame(loopTimer);
}
function replay(clip) {
if (!state.mocap) return;
if (state.playback) state.playback.stop();
state.playback = state.mocap.playback(clip, {
onEnd: () => {
state.playback = null;
},
});
}
function updateButtons() {
const has = !!state.lastClip;
saveBtn.disabled = !has || !state.signedIn;
downloadBtn.disabled = !has;
replayBtn.disabled = !has || !state.mocap;
discardBtn.disabled = !has;
}
async function saveClip() {
if (!state.lastClip) return;
const name = (nameInput.value || '').trim();
if (!name) {
alert('Give the clip a name first.');
nameInput.focus();
return;
}
const tags = (tagsInput.value || '')
.split(',')
.map((s) => s.trim().toLowerCase())
.filter(Boolean)
.slice(0, 20);
const payload = {
name,
description: (descInput.value || '').trim() || undefined,
tags,
visibility: visibilityInput.value,
avatar_id: ownsLoadedAvatar() ? state.avatar.id : undefined,
clip: state.lastClip,
};
saveBtn.disabled = true;
saveBtn.textContent = 'Saving…';
try {
const r = await apiFetch('/api/mocap/clips', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(payload),
});
if (!r.ok) {
const body = await r.json().catch(() => null);
throw new Error(
body?.error_description || body?.message || `save failed (${r.status})`,
);
}
saveBtn.textContent = 'Saved';
nameInput.value = '';
descInput.value = '';
tagsInput.value = '';
setTimeout(() => {
saveBtn.textContent = 'Save clip';
updateButtons();
}, 1400);
refreshClipList();
} catch (err) {
saveBtn.disabled = false;
saveBtn.textContent = 'Save clip';
alert(err?.message || 'Save failed.');
}
}
function downloadClip() {
if (!state.lastClip) return;
const blob = new Blob([JSON.stringify(state.lastClip, null, 2)], {
type: 'application/json',
});
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = `three-ws-mocap-${Date.now()}.json`;
document.body.appendChild(a);
a.click();
a.remove();
setTimeout(() => URL.revokeObjectURL(url), 5000);
}
async function refreshClipList() {
clipListEl.innerHTML = '<div class="empty-state">Loading…</div>';
try {
const r = await fetch('/api/mocap/clips?include_public=true&limit=30', {
credentials: 'include',
});
if (!r.ok) throw new Error(`list failed (${r.status})`);
const { items } = await r.json();
if (!items || items.length === 0) {
clipListEl.innerHTML = '<div class="empty-state">No clips saved yet. Record one above.</div>';
return;
}
clipListEl.innerHTML = '';
for (const c of items) {
const el = document.createElement('div');
el.className = 'clip';
el.innerHTML = `
<div class="meta">
<div class="n">${escapeHtml(c.name)} <span class="v-pill ${escapeAttr(c.visibility)}">${escapeHtml(c.visibility)}</span></div>
<div class="d">${escapeHtml(c.kind)} · ${formatTime((c.duration_ms || 0) / 1000)} · ${c.frame_count} frames${c.owner === 'self' ? '' : ' · public'}</div>
</div>
<div class="clip-actions">
<button class="clip-btn" data-act="play" data-id="${escapeAttr(c.id)}">Replay</button>
${c.owner === 'self' ? `<button class="clip-btn" data-act="delete" data-id="${escapeAttr(c.id)}">Delete</button>` : ''}
</div>
`;
clipListEl.appendChild(el);
}
clipListEl.querySelectorAll('button[data-act]').forEach((btn) => {
btn.addEventListener('click', () => {
const id = btn.dataset.id;
if (btn.dataset.act === 'play') playSavedClip(id);
else if (btn.dataset.act === 'delete') deleteSavedClip(id);
});
});
} catch (err) {
clipListEl.innerHTML = `<div class="empty-state">${escapeHtml(err?.message || 'Could not load clips.')}</div>`;
}
}
async function playSavedClip(id) {
if (!state.mocap) {
alert('Start the camera first so the playback driver is attached.');
return;
}
try {
const r = await fetch(`/api/mocap/clips/${encodeURIComponent(id)}`, { credentials: 'include' });
if (!r.ok) throw new Error(`fetch failed (${r.status})`);
const { clip } = await r.json();
replay({ format: clip.format, duration: clip.duration, frames: clip.frames });
} catch (err) {
alert(err?.message || 'Could not load clip.');
}
}
async function deleteSavedClip(id) {
if (!confirm('Delete this clip?')) return;
try {
const r = await apiFetch(`/api/mocap/clips/${encodeURIComponent(id)}`, {
method: 'DELETE',
});
if (!r.ok) {
const body = await r.json().catch(() => null);
throw new Error(body?.error_description || `delete failed (${r.status})`);
}
refreshClipList();
} catch (err) {
alert(err?.message || 'Could not delete.');
}
}
function formatTime(seconds) {
const s = Math.max(0, Number(seconds) || 0);
return `${s.toFixed(1)}s`;
}
function showErr(err) {
log.error('[mocap-studio]', err);
alert(err?.message || String(err));
startCamBtn.disabled = false;
startCamBtn.textContent = 'Start camera';
}
function escapeHtml(s) {
if (s == null) return '';
return String(s)
.replace(/&/g, '&')
.replace(/</g, '<')
.replace(/>/g, '>')
.replace(/"/g, '"')
.replace(/'/g, ''');
}
function escapeAttr(s) { return escapeHtml(s); }