Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// avatar-embed.js — runtime for /pages/avatar-embed.html
//
// Renders a portable, identity-light avatar in an iframe and exposes a
// v1.avatar.* postMessage bridge to the host page. Designed to be dropped
// into any third-party site via the <script src="…/embed.js" data-avatar=…>
// snippet, and to coexist on a page with the agent:* bridge.
//
// ── Wire format ──────────────────────────────────────────────────────────
//
// Host → iframe (v1.avatar.*):
// { type: 'v1.avatar.hello' }
// Handshake. Iframe replies with v1.avatar.ready (now or when init
// finishes).
// { type: 'v1.avatar.speak', text?, audioUrl? }
// Speak `text` (browser TTS) and/or play `audioUrl` — both routes
// drive audio-based lipsync onto the avatar's mouth morphs.
// { type: 'v1.avatar.emote', name, weight }
// Apply weight (0..1) to any ARKit-52 canonical morph by name.
// { type: 'v1.avatar.morphs', weights: { name: weight, ... } }
// Bulk apply many morph weights in one message. Names are normalized
// through MORPH_ALIASES, so `mouthOpen` etc. still work.
// { type: 'v1.avatar.lookAt', yaw, pitch }
// Rotate head bone (radians) around its rest pose.
// { type: 'v1.avatar.mocap', enabled }
// Toggle webcam mocap. The user is *also* prompted via the pill UI
// but a host can drive it programmatically here.
// { type: 'v1.avatar.idle', enabled }
// Toggle idle-life loop. Idle is on by default — turn off if you're
// driving the avatar from external animation.
// { type: 'v1.avatar.bg', mode }
// Switch background between 'transparent' | 'dark' | 'light'.
// { type: 'v1.avatar.stop' }
// Stop all motion (cancel speech, stop mocap, settle morphs to 0).
// { type: 'v1.avatar.ping', id }
// Liveness probe.
//
// Iframe → host:
// { type: 'v1.avatar.ready', version, capabilities, name, conformance }
// Sent once init succeeds + in response to v1.avatar.hello.
// `conformance` reports how many ARKit-52 morphs the loaded GLB
// implements — hosts can warn users when the body is incomplete.
// { type: 'v1.avatar.speak:start', source: 'tts'|'audio' }
// { type: 'v1.avatar.speak:end' }
// { type: 'v1.avatar.frame', blendshapes: {...}, headPose: {yaw,pitch,roll} }
// Per-frame mocap output, when mocap is active. Hosts can record
// this for replay (the recording format is identical to face-mocap's
// getRecording()).
// { type: 'v1.avatar.error', message }
// { type: 'v1.avatar.pong', id }
//
// URL params (read on load):
// ?id=<avatar_id> direct R2 storage key id
// ?handle=<username> resolved via /api/users/:handle/avatar
// ?model=<url> arbitrary GLB URL (advanced; same-origin trust)
// ?bg=transparent|dark|light default: transparent
// ?idle=on|off default: on
// ?mocap=off|webcam default: off
// ?lod=0|1|2 default: 0
// ?textureSize=128..2048 default: 2048
// ?morphs=arkit52|all default: all
// ?draco=1 opt-in mesh compression
import { Viewer } from './viewer.js';
import { IdleAnimation } from './idle-animation.js';
import { LipsyncDriver, tapAudioElement } from './voice/lipsync-driver.js';
import { AvatarMouthTarget } from './voice/avatar-morph-target.js';
import { A2FPlayer } from './voice/a2f-player.js';
import {
resolveMorphTargets,
setCanonicalMorph,
MORPH_ALIASES,
conformanceReport,
} from './runtime/arkit52.js';
import { log } from './shared/log.js';
import { isSafeQueryModelUrl } from './shared/safe-model-url.js';
import { viewerErrorText } from './shared/viewer-error-text.js';
const BRIDGE_VERSION = '1.0';
const CAPABILITIES = [
'speak',
'emote',
'morphs',
'lookAt',
'mocap',
'idle',
'bg',
'overlay',
'hotkey',
'state',
'mic',
'gesture',
'animation',
];
const HEAD_BONE_NAMES = ['head', 'neck'];
// ── Default hotkey → ARKit-52 morph map ───────────────────────────────────
// Modeled after Veadotube/VTube Studio's quick-action panel. Keys 1-9 map to
// named emotes that any host can override via the v1.avatar.hotkeys message.
// Each entry is a list of { name, weight, hold } records — names go through
// MORPH_ALIASES so combined morphs like `mouthSmile` resolve to L+R.
const DEFAULT_HOTKEYS = Object.freeze({
'1': { label: 'smile', hold: 1500, morphs: { mouthSmileLeft: 0.85, mouthSmileRight: 0.85, cheekSquintLeft: 0.4, cheekSquintRight: 0.4 } },
'2': { label: 'wink', hold: 280, morphs: { eyeBlinkLeft: 1, mouthSmileLeft: 0.5, mouthSmileRight: 0.3 } },
'3': { label: 'surprised', hold: 1400, morphs: { jawOpen: 0.55, eyeWideLeft: 0.8, eyeWideRight: 0.8, browInnerUp: 0.7 } },
'4': { label: 'sad', hold: 1800, morphs: { mouthFrownLeft: 0.7, mouthFrownRight: 0.7, browInnerUp: 0.55 } },
'5': { label: 'angry', hold: 1600, morphs: { browDownLeft: 0.8, browDownRight: 0.8, mouthPressLeft: 0.6, mouthPressRight: 0.6, noseSneerLeft: 0.4, noseSneerRight: 0.4 } },
'6': { label: 'disgust', hold: 1600, morphs: { noseSneerLeft: 0.8, noseSneerRight: 0.8, mouthUpperUpLeft: 0.55, mouthUpperUpRight: 0.55, eyeSquintLeft: 0.45, eyeSquintRight: 0.45 } },
'7': { label: 'thinking', hold: 1800, morphs: { eyeLookUpLeft: 0.6, eyeLookUpRight: 0.6, mouthLeft: 0.35, browOuterUpLeft: 0.45 } },
'8': { label: 'kiss', hold: 1200, morphs: { mouthPucker: 0.95, eyeBlinkLeft: 0.6, eyeBlinkRight: 0.6 } },
'9': { label: 'tongue', hold: 1400, morphs: { tongueOut: 0.9, jawOpen: 0.45, mouthSmileLeft: 0.4, mouthSmileRight: 0.4 } },
'0': { label: 'neutral', hold: 80, morphs: {} },
});
main().catch((err) => {
// The visitor gets plain language; the raw rejection stays in the console so
// a developer debugging an embed still sees which URL and status failed.
log.warn('[avatar-embed] load failed', err);
showError(viewerErrorText(err));
});
async function main() {
const params = new URL(location.href).searchParams;
applyBackground(params.get('bg') || 'transparent');
// Overlay mode — chrome-free canvas for OBS Browser Source. Hides the
// name plate, mocap pill, and error positioning; treats the body as a
// pure render surface. Press space to re-show chrome (VSeeFace "※" pattern).
const overlayMode = params.get('overlay') === '1' || params.get('chrome') === '0';
if (overlayMode) document.body.classList.add('overlay-mode');
// ── Resolve which avatar to render ─────────────────────────────────────
const resolved = await resolveAvatar(params);
if (!resolved) {
showError('No avatar id, handle, or model URL provided.');
return;
}
const hideChrome = params.get('hide-chrome') === '1';
const namePlate = !overlayMode && !hideChrome && params.get('name') !== '0';
if (namePlate && resolved.name) {
document.getElementById('name-plate').textContent = resolved.name;
}
// ── Bring up the Three.js scene ─────────────────────────────────────────
const stage = document.getElementById('stage');
const viewer = new Viewer(stage, { kiosk: true });
const bg = params.get('bg') || 'transparent';
if (bg === 'transparent') {
viewer.renderer?.setClearAlpha(0);
if (viewer.scene) viewer.scene.background = null;
}
if (params.get('gaze') === 'off') viewer.state.followMode = 'off';
await viewer.load(resolved.modelUrl, '', new Map());
const root = viewer.content;
if (!root) throw new Error('avatar failed to load');
// ── Animation clips (manifest-based) ──────────────────────────────────
const animMgr = viewer.animationManager;
let animDefs = [];
try {
const mRes = await fetch('/animations/manifest.json');
if (mRes.ok) {
animDefs = await mRes.json();
animMgr.setAnimationDefs(animDefs);
animMgr.loadAll().catch(() => {});
}
} catch (err) {
log.warn('[avatar-embed] animation manifest load failed', err?.message);
}
// ?animation= / ?anim= — supports manifest clip names, full-library (R2)
// clip names, and API clip ids/slugs.
const startAnim = params.get('animation') || params.get('anim') || params.get('pose');
if (startAnim) {
const isApiId = /^[0-9a-f-]{36}$/i.test(startAnim) || startAnim.startsWith('user:');
if (isApiId) {
viewer.playAnimationById(startAnim.startsWith('user:') ? startAnim.slice(5) : startAnim)
.catch(() => animMgr.crossfadeTo('idle', 0.3).catch(() => {}));
} else if (animDefs.some((d) => d.name === startAnim)) {
animMgr.crossfadeTo(startAnim, 0.3).catch(() => {});
} else {
// Not in the curated manifest — resolve from the full motion library
// and register just the requested def (its url points at the R2 CDN).
// The ?name= lookup keeps that one embed request small: the whole
// manifest is over a megabyte and only one entry of it is wanted.
(async () => {
try {
const res = await fetch(
`/api/animations/library?name=${encodeURIComponent(startAnim)}`,
);
if (res.ok) {
const data = await res.json();
const def = (data.clips || []).find((d) => d.name === startAnim);
if (def) animMgr.appendAnimationDefs([def]);
}
} catch (err) {
log.warn('[avatar-embed] library lookup failed', err?.message);
}
animMgr.crossfadeTo(startAnim, 0.3).catch(() => {});
})();
}
}
const pickerEnabled = params.get('animPicker') !== '0' && !overlayMode && !hideChrome;
if (pickerEnabled && animDefs.length > 0) {
buildAnimPicker(animDefs, animMgr);
}
// ── Mouth lipsync (audio-driven) ───────────────────────────────────────
const mouthTarget = new AvatarMouthTarget();
mouthTarget.attach(root);
// Lipsync driver is created lazily per-utterance — needs a live analyser.
let activeLipsync = null;
let audioCtx = null;
const ensureAudio = () => {
if (!audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)();
if (audioCtx.state === 'suspended') audioCtx.resume();
return audioCtx;
};
// ── Audio2Face-3D (server-side NVIDIA facial animation) ────────────────────
// Progressive enhancement over the amplitude lipsync above: when the platform
// has the A2F lane configured (NVIDIA_API_KEY → /api/a2f), the spoken audio is
// sent to Audio2Face-3D and the avatar's face is driven by the returned ARKit
// blendshape track instead of the audio envelope — real lip sync to the actual
// voice. Fully self-disabling: any failure (no key, 404, unsupported rig)
// leaves the amplitude path untouched, so this never blocks or freezes speech.
// Opt out per-embed with ?a2f=off.
const a2fEnabled = (params.get('a2f') || 'auto') !== 'off';
const a2fPlayer = new A2FPlayer();
a2fPlayer.attach(root);
let a2fAudio = null; // the <audio> element currently driven by an A2F track
// One-time capability probe so we never do the (non-trivial) decode→resample
// work on deployments where the A2F lane is unconfigured — there it stays pure
// amplitude lipsync with zero added cost.
let a2fProbe = null;
// The GET probe only reports whether the lane is env-configured; a POST can
// still fail hard (invalid model/function id → 502, provider outage → 5xx).
// Latch that so we stop re-POSTing a lane that won't recover this session and
// stay on amplitude lipsync instead.
let a2fLaneBroken = false;
const a2fAvailable = () => {
if (!a2fProbe) {
a2fProbe = fetch('/api/a2f', { method: 'GET' })
.then((r) => (r.ok ? r.json() : null))
.then((c) => !!(c && c.configured))
.catch(() => false);
}
return a2fProbe;
};
// Per-frame: sample the track at the playing audio's time. Registered with the
// viewer's hook array (created below by the idle loop if absent).
if (!viewer._afterAnimateHooks) viewer._afterAnimateHooks = [];
viewer._afterAnimateHooks.push(() => {
if (a2fAudio && !a2fAudio.paused && !a2fAudio.ended) a2fPlayer.update(a2fAudio.currentTime);
});
// ── Idle-life loop ─────────────────────────────────────────────────────
const idle = new IdleAnimation({
getRoot: () => viewer.content,
seed: resolved.id || resolved.handle || resolved.modelUrl,
});
let idleEnabled = params.get('idle') !== 'off';
if (!idleEnabled) idle.setChannels({ breathing: false, saccade: false, blink: false, weightShift: false });
// Wire idle into the viewer's per-frame hooks. Viewer exposes an
// `_afterAnimateHooks` array used by the cloud renderer; we follow the
// same pattern. The viewer's animate() loop pumps these each frame with
// the elapsed-seconds delta.
if (!viewer._afterAnimateHooks) viewer._afterAnimateHooks = [];
viewer._afterAnimateHooks.push((dt) => {
if (idleEnabled) idle.update(dt);
});
// ── Face mocap (webcam, opt-in) ────────────────────────────────────────
let mocap = null;
let mocapVideo = null;
let mocapBindingsReady = false;
const findHeadBone = () => {
let head = null;
let neck = null;
root.traverse((node) => {
if (!node.isBone) return;
const canon = node.name
.replace(/^mixamorig:?/i, '')
.replace(/^[A-Za-z0-9]+[_:]/, '')
.toLowerCase();
if (!head && canon === 'head') head = node;
else if (!neck && canon === 'neck') neck = node;
});
return head || neck;
};
const startMocap = async () => {
if (mocap) return;
const { FaceMocap } = await import('./face-mocap.js');
mocap = new FaceMocap();
await mocap.init();
const head = findHeadBone();
mocap.attach(root, head);
mocap.onFaceDetected((has) => {
updatePill(true, has);
});
mocapVideo = await mocap.startWebcam();
mocapVideo.style.cssText =
'position:absolute;right:10px;bottom:10px;width:120px;height:90px;' +
'object-fit:cover;border-radius:8px;border:1px solid rgba(255,255,255,0.15);' +
'transform:scaleX(-1);z-index:4;background:#000;';
document.body.appendChild(mocapVideo);
mocap.start();
mocapBindingsReady = true;
// Drive per-frame via viewer hook.
viewer._afterAnimateHooks.push(() => {
if (!mocap?._running) return;
mocap.update();
const last = mocap.getLastResult?.();
if (!last?.faceBlendshapes?.length) return;
const shapes = {};
for (const { categoryName, score } of last.faceBlendshapes[0].categories) {
shapes[categoryName] = score;
}
const matData = last.facialTransformationMatrixes?.[0]?.data || null;
postToParent({
type: 'v1.avatar.frame',
blendshapes: shapes,
headPose: matData ? decodeYawPitchRoll(matData) : null,
});
});
// Pause idle saccade/blink while mocap drives the head + eyes.
idle.setChannels({ saccade: false, blink: false });
};
const stopMocap = () => {
if (!mocap) return;
mocap.stop();
// Restore idle.
if (idleEnabled) idle.setChannels({ saccade: true, blink: true });
// Remove the picture-in-picture <video>.
mocapVideo?.remove();
mocapVideo = null;
mocap = null;
mocapBindingsReady = false;
updatePill(false, false);
};
const pill = document.getElementById('mocap-pill');
const updatePill = (on, faceLocked) => {
pill.classList.toggle('on', !!on);
pill.querySelector('.label').textContent = on
? faceLocked
? 'mocap live'
: 'mocap on (no face)'
: 'webcam mocap';
};
const togglePill = async () => {
if (mocap) stopMocap();
else await startMocap();
};
pill.addEventListener('click', togglePill);
pill.addEventListener('keydown', (ev) => {
if (ev.key === 'Enter' || ev.key === ' ') {
ev.preventDefault();
togglePill();
}
});
// Show pill if a mocap-capable surface was requested.
if ((params.get('mocap') || 'off') !== 'off') {
if (!hideChrome) pill.style.display = 'inline-flex';
// Auto-start when explicitly opted in. Browsers block getUserMedia
// from auto-running without a user gesture in most cases — the pill
// is the click target if so.
try {
await startMocap();
} catch (err) {
log.warn('[avatar-embed] mocap auto-start failed', err);
}
} else if (!hideChrome) {
pill.style.display = 'inline-flex';
}
// ── Speak (TTS + audio URL) ────────────────────────────────────────────
const speakText = async (text) => {
if (!text || !('speechSynthesis' in window)) return;
await new Promise((resolve) => {
const utter = new SpeechSynthesisUtterance(text);
utter.lang = 'en-US';
let analyser = null;
let source = null;
utter.onstart = () => {
postToParent({ type: 'v1.avatar.speak:start', source: 'tts' });
// Browser TTS audio is not exposed as a MediaStream, so we can't
// drive RMS lipsync from it. Use the text-driven phoneme heuristic
// fallback so the mouth still moves in sync with speech.
activeLipsync = startTextLipsync(text, root, mouthTarget);
};
utter.onend = () => {
activeLipsync?.stop();
activeLipsync = null;
postToParent({ type: 'v1.avatar.speak:end' });
resolve();
};
utter.onerror = () => {
activeLipsync?.stop();
activeLipsync = null;
resolve();
};
window.speechSynthesis.speak(utter);
});
};
// Decode any audio URL (mp3/wav/opus) to 16 kHz mono 16-bit PCM, base64'd —
// the format Audio2Face-3D expects. Done in-browser via the Web Audio decoder
// the page already uses for playback, so the embed's A2F path is codec-agnostic
// and we never round-trip the bytes through a server transcode.
const fetchA2FTrack = async (url) => {
const resp = await fetch(url);
if (!resp.ok) return null;
const encoded = await resp.arrayBuffer();
const decodeCtx = ensureAudio();
const decoded = await decodeCtx.decodeAudioData(encoded.slice(0));
// Downmix to mono.
const chs = decoded.numberOfChannels;
const mono = new Float32Array(decoded.length);
for (let c = 0; c < chs; c++) {
const data = decoded.getChannelData(c);
for (let i = 0; i < data.length; i++) mono[i] += data[i] / chs;
}
// Resample to 16 kHz and convert to s16 little-endian.
const ratio = 16000 / decoded.sampleRate;
const outLen = Math.max(1, Math.floor(mono.length * ratio));
const pcm = new Int16Array(outLen);
for (let i = 0; i < outLen; i++) {
const src = i / ratio;
const i0 = Math.floor(src);
const i1 = Math.min(i0 + 1, mono.length - 1);
const s = mono[i0] + (mono[i1] - mono[i0]) * (src - i0);
pcm[i] = Math.max(-32768, Math.min(32767, Math.round(s * 32767)));
}
const bytes = new Uint8Array(pcm.buffer);
let bin = '';
for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]);
const r = await fetch('/api/a2f', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ audio: btoa(bin), format: 'pcm', sampleRate: 16000 }),
});
if (!r.ok) {
if (r.status >= 500) a2fLaneBroken = true; // provider down/misconfigured — stop retrying
return null;
}
const data = await r.json();
return data?.animation?.frames?.length ? data.animation : null;
};
const playAudio = async (url) => {
const ctx = ensureAudio();
const audio = new Audio();
audio.crossOrigin = 'anonymous';
audio.src = url;
// Kick off the A2F request in parallel with playback. Amplitude lipsync
// starts immediately (zero latency); if a real blendshape track comes back
// before the clip ends and the rig can show it, we upgrade seamlessly.
let a2fUpgraded = false;
const a2fPromise = (a2fEnabled && !a2fLaneBroken)
? a2fAvailable()
.then((ok) => (ok ? fetchA2FTrack(url) : null))
.then((track) => {
if (!track || audio.ended) return;
a2fPlayer.setTrack(track);
if (!a2fPlayer.hasCoverage()) return;
// Hand the face over from amplitude → A2F track.
activeLipsync?.stop();
activeLipsync = null;
a2fAudio = audio;
a2fUpgraded = true;
postToParent({ type: 'v1.avatar.speak:a2f', frames: track.frameCount, fps: track.fps });
})
.catch(() => {})
: Promise.resolve();
await audio.play();
// Only run amplitude lipsync until/unless A2F takes over.
if (!a2fUpgraded) {
const { analyser } = tapAudioElement(audio, ctx);
activeLipsync = new LipsyncDriver({ analyser, target: mouthTarget });
activeLipsync.start();
}
postToParent({ type: 'v1.avatar.speak:start', source: 'audio' });
await new Promise((resolve) => {
audio.onended = resolve;
audio.onerror = resolve;
});
await a2fPromise;
activeLipsync?.stop();
activeLipsync = null;
if (a2fAudio === audio) { a2fAudio = null; a2fPlayer.reset(); }
postToParent({ type: 'v1.avatar.speak:end' });
};
// ── Morph control ──────────────────────────────────────────────────────
const morphResolved = resolveMorphTargets(root);
const setMorph = (name, weight) => {
const canonical = MORPH_ALIASES[name] ?? name;
setCanonicalMorph(morphResolved, canonical, weight);
};
// ── Head look-at ───────────────────────────────────────────────────────
const headBone = findHeadBone();
const headRest = headBone
? { x: headBone.rotation.x, y: headBone.rotation.y, z: headBone.rotation.z }
: null;
const lookAt = (yaw, pitch) => {
if (!headBone) return;
const maxYaw = 0.6, maxPitch = 0.45;
const y = Math.max(-maxYaw, Math.min(maxYaw, Number(yaw) || 0));
const p = Math.max(-maxPitch, Math.min(maxPitch, Number(pitch) || 0));
headBone.rotation.y = headRest.y + y;
headBone.rotation.x = headRest.x + p;
};
// ── Emote hotkey panel ─────────────────────────────────────────────────
// Numeric 1-9 + 0 trigger named emotes (Veadotube quick-action pattern).
// Each hotkey blends a set of canonical ARKit-52 morphs, holds for the
// configured duration, then releases. Hosts can override the map via the
// v1.avatar.hotkeys message.
const hotkeys = JSON.parse(JSON.stringify(DEFAULT_HOTKEYS));
const activeReleases = new Map(); // key → timeout id, so re-triggering a key resets the hold
function triggerHotkey(key, options = {}) {
const entry = hotkeys[String(key)];
if (!entry) return false;
// Apply morphs.
for (const [name, weight] of Object.entries(entry.morphs || {})) {
setMorph(name, Number(weight) || 0);
}
// Reset any other key's pending release (Stream Deck users mash keys).
for (const t of activeReleases.values()) clearTimeout(t);
activeReleases.clear();
// Schedule release at end of hold, unless hold is 0 (latching mode).
const hold = Number.isFinite(options.hold) ? options.hold : entry.hold;
const slot = document.querySelector(`#emote-panel .slot[data-key="${CSS.escape(String(key))}"]`);
if (slot) {
document.querySelectorAll('#emote-panel .slot.active').forEach((el) => el.classList.remove('active'));
slot.classList.add('active');
}
if (hold && hold > 0) {
const id = setTimeout(() => {
for (const name of Object.keys(entry.morphs || {})) setMorph(name, 0);
if (slot) slot.classList.remove('active');
activeReleases.delete(key);
}, hold);
activeReleases.set(key, id);
}
postToParent({ type: 'v1.avatar.hotkey:fired', key: String(key), label: entry.label });
broadcast({ type: 'v1.avatar.hotkey:fired', key: String(key), label: entry.label });
return true;
}
function renderEmotePanel() {
const panel = document.getElementById('emote-panel');
if (!panel) return;
panel.innerHTML = '';
const keys = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0'];
for (const k of keys) {
const entry = hotkeys[k];
if (!entry) continue;
const slot = document.createElement('div');
slot.className = 'slot';
slot.dataset.key = k;
slot.tabIndex = 0;
slot.setAttribute('role', 'button');
slot.setAttribute('aria-label', entry.label || `Emote ${k}`);
slot.innerHTML = `<span class="k">${k}</span><span class="l">${escapeHtmlSafe(entry.label || '')}</span>`;
slot.addEventListener('click', () => triggerHotkey(k));
slot.addEventListener('keydown', (ev) => {
if (ev.key === 'Enter' || ev.key === ' ') {
ev.preventDefault();
triggerHotkey(k);
}
});
panel.appendChild(slot);
}
}
if (!hideChrome) renderEmotePanel();
// Hotkey listener — captures 1-9 + 0 globally, plus space to toggle the
// panel (VSeeFace ※ pattern), plus h to toggle chrome.
window.addEventListener('keydown', (ev) => {
// Skip if focus is in a form control — the host page may have its own
// inputs (this matters for the in-page control panel below).
if (ev.target && /^(INPUT|TEXTAREA|SELECT)$/.test(ev.target.tagName)) return;
if (ev.key >= '0' && ev.key <= '9') {
if (triggerHotkey(ev.key)) ev.preventDefault();
return;
}
if (ev.key === ' ' || ev.code === 'Space') {
document.body.classList.toggle('panel-open');
ev.preventDefault();
return;
}
if (ev.key === 'h' || ev.key === 'H') {
document.body.classList.toggle('overlay-mode');
return;
}
if (ev.key === 's' || ev.key === 'S') {
document.body.classList.toggle('show-state');
return;
}
});
// In non-overlay mode the panel is visible by default. In overlay mode
// the user opts in via space, matching the OBS Browser Source workflow.
if (!overlayMode && !hideChrome) document.body.classList.add('panel-open');
// ── Veadotube-style state machine ──────────────────────────────────────
// State = {expression, talking}. Expression is one of the hotkey labels
// (neutral by default). Talking flips automatically when mic RMS exceeds
// the configured threshold (PNGTuber Plus dual-threshold pattern).
const state = {
expression: 'neutral',
talking: false,
micFloor: 0.04,
micCeiling: 0.18,
};
const statePill = document.getElementById('state-pill');
const renderState = () => {
if (statePill) statePill.textContent = `${state.expression}${state.talking ? ' · talking' : ''}`;
};
renderState();
function setState(patch) {
Object.assign(state, patch);
renderState();
postToParent({ type: 'v1.avatar.state', state: { ...state } });
broadcast({ type: 'v1.avatar.state', state: { ...state } });
}
// ── Mic-driven talking threshold (PNGTuber pattern) ───────────────────
// Opt-in via ?mic=1. Floor and ceiling expressed in 0..1 (max byte 255).
// Floor is the noise gate (below = not talking). Ceiling is full open.
let micCleanup = null;
async function enableMic() {
if (micCleanup) return;
const ctx = ensureAudio();
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const src = ctx.createMediaStreamSource(stream);
const analyser = ctx.createAnalyser();
analyser.fftSize = 1024;
analyser.smoothingTimeConstant = 0.6;
src.connect(analyser);
const buf = new Uint8Array(analyser.frequencyBinCount);
let raf = 0;
let stopped = false;
const tick = () => {
if (stopped) return;
analyser.getByteFrequencyData(buf);
let sum = 0;
for (let i = 0; i < buf.length; i++) sum += buf[i];
const avg = sum / buf.length / 255;
const talking = avg > state.micFloor;
// Drive the mouth proportional to the loud band [floor..ceiling].
if (mouthTarget) {
const t = Math.max(0, Math.min(1, (avg - state.micFloor) / Math.max(0.0001, state.micCeiling - state.micFloor)));
try {
mouthTarget.setMouthShape({ open: t * 0.85, wide: 0, round: 0 });
} catch {}
}
if (talking !== state.talking) setState({ talking });
raf = requestAnimationFrame(tick);
};
raf = requestAnimationFrame(tick);
micCleanup = () => {
stopped = true;
if (raf) cancelAnimationFrame(raf);
for (const t of stream.getTracks()) t.stop();
micCleanup = null;
if (state.talking) setState({ talking: false });
if (mouthTarget) {
try { mouthTarget.setMouthShape({ open: 0, wide: 0, round: 0 }); } catch {}
}
};
}
function disableMic() {
if (micCleanup) micCleanup();
}
if (params.get('mic') === '1') {
enableMic().catch((err) =>
log.warn('[avatar-embed] mic auto-start failed', err?.message),
);
}
if (params.get('state') === '1') document.body.classList.add('show-state');
// ── BroadcastChannel — same-origin control surface ─────────────────────
// Lets a sibling tab/window (overlay-control.html) drive this embed
// without depending on iframe parent semantics. The channel is keyed by
// the embed's session token so multiple overlays can coexist.
// Handle is preferred over id so the control panel — which only knows the
// handle the user typed — lands on the same channel as the overlay window.
// Falls back to id when the avatar was mounted by uuid, then a stable
// 'default' last resort.
const channelKey = params.get('channel') || `three-ws-overlay:${resolved.handle || resolved.id || 'default'}`;
let bc = null;
try {
if (typeof BroadcastChannel !== 'undefined') {
bc = new BroadcastChannel(channelKey);
bc.onmessage = (ev) => handleControl(ev.data, { source: 'broadcast' });
// Announce presence so an already-open control panel can light up.
bc.postMessage({ type: 'v1.avatar.online', channel: channelKey, name: resolved.name, handle: resolved.handle, id: resolved.id });
}
} catch (err) {
log.warn('[avatar-embed] BroadcastChannel unavailable', err?.message);
}
function broadcast(msg) {
if (!bc) return;
try { bc.postMessage(msg); } catch {}
}
// ── postMessage bridge ─────────────────────────────────────────────────
let parentOrigin = (() => {
try {
if (window.parent === window) return location.origin;
if (document.referrer) return new URL(document.referrer).origin;
} catch {}
return null;
})();
let parentOriginLocked = false;
let bridgeReady = false;
const pendingHellos = [];
function postToParent(msg) {
if (!parent || parent === window) return;
if (!parentOrigin) return;
parent.postMessage(msg, parentOrigin);
}
function postReady() {
const conformance = conformanceReport(root);
const readyMsg = {
type: 'v1.avatar.ready',
version: BRIDGE_VERSION,
capabilities: CAPABILITIES,
name: resolved.name || null,
handle: resolved.handle || null,
id: resolved.id || null,
conformance: {
implemented: conformance.implemented.length,
total: conformance.implemented.length + conformance.missing.length,
coverage: conformance.coverage,
},
hotkeys: Object.fromEntries(
Object.entries(hotkeys).map(([k, v]) => [k, { label: v.label, hold: v.hold }])
),
};
postToParent(readyMsg);
// RPM-compatible aliases — emit the same payload under Ready Player
// Me's event names so integrators can swap to three.ws with one URL
// change. See docs.readyplayer.me/.../avatar-creator/events.
postToParent({ ...readyMsg, type: 'v1.frame.ready' });
postToParent({
eventName: 'v2.avatar.exported',
source: 'threews',
data: {
url: resolved.modelUrl,
avatarId: resolved.id,
handle: resolved.handle,
name: resolved.name,
},
});
broadcast(readyMsg);
}
// Unified control handler — same logic for parent postMessage and the
// BroadcastChannel sibling-tab pathway. The only difference is origin
// locking: postMessage messages are origin-checked above; broadcasts are
// already same-origin per the spec.
function handleControl(msg, ctx = {}) {
if (!msg || typeof msg !== 'object') return;
if (typeof msg.type !== 'string') return;
// Accept v1.avatar.* messages plus the RPM-compatible v1.frame.hello
// handshake from integrators expecting the RPM event vocabulary.
const accepted = msg.type.startsWith('v1.avatar.') || msg.type === 'v1.frame.hello';
if (!accepted) return;
switch (msg.type) {
case 'v1.avatar.hello':
case 'v1.frame.hello':
if (bridgeReady) postReady();
else pendingHellos.push(true);
return;
case 'v1.avatar.ping':
postToParent({ type: 'v1.avatar.pong', id: msg.id });
if (ctx.source === 'broadcast') broadcast({ type: 'v1.avatar.pong', id: msg.id });
return;
case 'v1.avatar.speak': {
const tasks = [];
if (msg.text) tasks.push(speakText(String(msg.text)));
if (msg.audioUrl) tasks.push(playAudio(String(msg.audioUrl)));
Promise.all(tasks).catch((err) =>
postToParent({ type: 'v1.avatar.error', message: err?.message || 'speak failed' }),
);
return;
}
case 'v1.avatar.emote':
if (msg.name) setMorph(String(msg.name), Number(msg.weight) || 0);
return;
case 'v1.avatar.morphs':
if (msg.weights && typeof msg.weights === 'object') {
for (const [k, v] of Object.entries(msg.weights)) {
setMorph(String(k), Number(v) || 0);
}
}
return;
case 'v1.avatar.lookAt':
lookAt(msg.yaw, msg.pitch);
return;
case 'v1.avatar.mocap':
if (msg.enabled) startMocap().catch((e) => postToParent({ type: 'v1.avatar.error', message: e?.message || 'mocap failed' }));
else stopMocap();
return;
case 'v1.avatar.idle':
idleEnabled = !!msg.enabled;
idle.setChannels({
breathing: idleEnabled,
saccade: idleEnabled && !mocap,
blink: idleEnabled && !mocap,
weightShift: idleEnabled,
});
return;
case 'v1.avatar.bg':
applyBackground(msg.mode || 'transparent');
if ((msg.mode || 'transparent') === 'transparent') {
viewer.renderer?.setClearAlpha(0);
if (viewer.scene) viewer.scene.background = null;
}
return;
case 'v1.avatar.stop':
window.speechSynthesis?.cancel();
activeLipsync?.stop();
activeLipsync = null;
stopMocap();
return;
// ── Animation clip playback ───────────────────────────────
case 'v1.avatar.gesture':
case 'v1.avatar.animation': {
const clipName = msg.name || msg.clip;
if (!clipName) return;
animMgr.crossfadeTo(String(clipName), 0.35).catch(() => {});
postToParent({ type: 'v1.avatar.gesture:playing', name: String(clipName) });
broadcast({ type: 'v1.avatar.gesture:playing', name: String(clipName) });
return;
}
// ── OBS / streaming overlay surface ──────────────────────────
case 'v1.avatar.hotkey':
if (msg.key != null) triggerHotkey(msg.key, msg);
return;
case 'v1.avatar.hotkeys':
// Replace or merge the hotkey map. Pass `{ replace: true }`
// for a clean wipe; otherwise merges so the host can override
// just one slot.
if (msg.map && typeof msg.map === 'object') {
if (msg.replace) {
for (const k of Object.keys(hotkeys)) delete hotkeys[k];
}
for (const [k, entry] of Object.entries(msg.map)) {
hotkeys[String(k)] = {
label: String(entry?.label || ''),
hold: Number.isFinite(entry?.hold) ? Number(entry.hold) : 1200,
morphs: entry?.morphs && typeof entry.morphs === 'object' ? entry.morphs : {},
};
}
renderEmotePanel();
}
return;
case 'v1.avatar.state':
// Force a state update from outside (e.g. chat command says
// "switch to angry expression for the next clip").
if (msg.state && typeof msg.state === 'object') setState(msg.state);
return;
case 'v1.avatar.mic':
if (msg.enabled) {
if (Number.isFinite(msg.floor)) state.micFloor = clamp01s(msg.floor);
if (Number.isFinite(msg.ceiling)) state.micCeiling = clamp01s(msg.ceiling);
enableMic().catch((e) =>
postToParent({ type: 'v1.avatar.error', message: e?.message || 'mic failed' }),
);
} else {
disableMic();
}
return;
case 'v1.avatar.overlay':
// Toggle chrome-free overlay mode at runtime.
if (msg.enabled === true) document.body.classList.add('overlay-mode');
else if (msg.enabled === false) document.body.classList.remove('overlay-mode');
return;
}
}
function handleMessage(ev) {
if (ev.source !== window.parent) return;
const msg = ev.data;
if (!msg || typeof msg !== 'object') return;
if (typeof msg.type !== 'string') return;
// Accept both v1.avatar.* and the RPM-compatible v1.frame.hello.
if (!msg.type.startsWith('v1.avatar.') && msg.type !== 'v1.frame.hello') return;
if (!parentOriginLocked) {
parentOrigin = ev.origin;
parentOriginLocked = true;
} else if (ev.origin !== parentOrigin) {
return;
}
handleControl(msg, { source: 'window' });
}
window.addEventListener('message', handleMessage);
// ── Auto-fit reporting (resize host iframe to content) ─────────────────
let pendingPost = false;
const reportSize = () => {
if (pendingPost) return;
pendingPost = true;
requestAnimationFrame(() => {
pendingPost = false;
const h = Math.ceil(document.documentElement.scrollHeight || stage.clientHeight || 0);
if (h > 0) postToParent({ type: 'v1.avatar.resize', height: h });
});
};
if (typeof ResizeObserver !== 'undefined') {
new ResizeObserver(reportSize).observe(document.documentElement);
}
reportSize();
bridgeReady = true;
postReady();
while (pendingHellos.length) {
pendingHellos.pop();
postReady();
}
}
// ── Resolve which avatar to render from URL params ──────────────────────────
async function resolveAvatar(params) {
// Direct model URL — must be same-origin OR a trusted three.ws asset host.
// Untrusted origins are rejected so the embed can't be turned into an open
// relay that renders an attacker-hosted GLB under the three.ws origin.
const model = params.get('model');
if (model) {
if (!isSafeQueryModelUrl(model)) throw new Error('untrusted model URL');
return { modelUrl: model, name: null, id: null, handle: null };
}
const id = params.get('id');
if (id) {
const r = await fetch(`/api/avatars/${encodeURIComponent(id)}`, { credentials: 'include' });
if (!r.ok) throw new Error(`avatar ${id} not found`);
const { avatar } = await r.json();
if (!avatar?.id) throw new Error(`avatar ${id} not found`);
return {
modelUrl: storedAvatarUrl(avatar, params),
name: avatar.name || null,
id: avatar.id,
handle: null,
};
}
// /embed/avatar/:handle — path-based resolution; URL is rewritten to
// /avatar-embed.html?handle=… by the dev middleware and vercel rewrite.
let handle = params.get('handle');
if (!handle) {
const parts = location.pathname.split('/').filter(Boolean);
const idx = parts.findIndex((p) => p === 'avatar');
if (idx !== -1 && parts[idx + 1] && parts[idx + 1] !== 'embed') {
handle = parts[idx + 1];
}
}
if (handle) {
handle = handle.replace(/^@/, '');
const q = new URLSearchParams();
for (const k of ['lod', 'textureSize', 'morphs', 'draco', 'baked']) {
const v = params.get(k);
if (v != null) q.set(k, v);
}
const r = await fetch(`/api/users/${encodeURIComponent(handle)}/avatar?${q.toString()}`);
if (!r.ok) {
const body = await r.json().catch(() => null);
throw new Error(body?.message || `@${handle} avatar not available`);
}
const { user, avatar } = await r.json();
return {
modelUrl: avatar.model_url,
name: user.display_name || user.username,
id: avatar.id,
handle: user.username,
};
}
return null;
}
// A stored avatar always renders through the same-origin GLB proxy
// (api/avatars/[id]/[action].js), never through the `url` the API hands back.
// That `url` is a presigned r2.cloudflarestorage.com read for a private avatar
// and the public r2.dev domain for a shared one: the first sends no
// access-control-allow-origin at all, and the second only allows the three.ws
// origin, so GLTFLoader is CORS-blocked in every other context this frame runs
// in (a dev server, a preview host, an embedding page on another domain). The
// proxy is same-origin, carries the session cookie through the permission gate,
// and falls back to the public bucket when the signed read is refused.
function storedAvatarUrl(avatar, params) {
const proxy = `/api/avatars/${encodeURIComponent(avatar.id)}/glb`;
if (!wantsOptimization(params)) return proxy;
// /api/avatar/optimize is unauthenticated and resolves an id only for a
// public or unlisted avatar (resolving a private one by id was an IDOR), so
// a private avatar stays on the unoptimized proxy rather than 404ing.
const shared = avatar.visibility === 'public' || avatar.visibility === 'unlisted';
if (!shared) return proxy;
return optimizedUrl(avatar.id, params);
}
function wantsOptimization(params) {
return !!(
params.get('lod') ||
params.get('textureSize') ||
params.get('morphs') ||
params.get('draco') === '1'
);
}
function optimizedUrl(avatarId, params) {
const lod = params.get('lod');
const textureSize = params.get('textureSize');
const morphs = params.get('morphs');
const draco = params.get('draco');
const u = new URL('/api/avatar/optimize', location.origin);
u.searchParams.set('id', avatarId);
if (lod) u.searchParams.set('lod', lod);
if (textureSize) u.searchParams.set('textureSize', textureSize);
if (morphs) u.searchParams.set('morphs', morphs);
if (draco === '1') u.searchParams.set('draco', '1');
return u.toString();
}
// ── Background ──────────────────────────────────────────────────────────────
function applyBackground(mode) {
document.body.classList.remove('bg-dark', 'bg-light');
if (mode === 'dark') document.body.classList.add('bg-dark');
else if (mode === 'light') document.body.classList.add('bg-light');
}
// ── Phoneme-heuristic lipsync (no audio analyser available) ─────────────────
function startTextLipsync(text, root, mouthTarget) {
// Drive the mouth from a coarse phoneme heuristic synced to total
// utterance duration. This is the fallback path when browser TTS doesn't
// expose a MediaStream we can analyse. Updates the AvatarMouthTarget
// shape (open/wide/round) rather than going through canonical morphs so
// it lives alongside the audio-driven LipsyncDriver path.
if (!text) return { stop() {} };
// Approximate 110ms/syllable; collapse runs of vowels to syllable beats.
const syllables = String(text).toLowerCase().match(/[aeiouy]+/g) || [];
const durMs = Math.max(600, syllables.length * 110);
const start = performance.now();
let raf = 0;
let stopped = false;
const tick = () => {
if (stopped) return;
const t = performance.now() - start;
if (t >= durMs) {
try {
mouthTarget.setMouthShape({ open: 0, wide: 0, round: 0 });
} catch {}
stopped = true;
return;
}
// Position within current syllable [0,1).
const sylIdx = Math.min(syllables.length - 1, Math.floor((t / durMs) * syllables.length));
const local = (t / durMs) * syllables.length - sylIdx;
// Triangular envelope: open at midpoint, closed at the edges.
const env = local < 0.5 ? local * 2 : (1 - local) * 2;
const v = syllables[sylIdx] || '';
const shape =
/[oö]/.test(v) ? { open: env * 0.55, wide: 0, round: env * 0.9 } :
/[u]/.test(v) ? { open: env * 0.35, wide: 0, round: env * 0.95 } :
/[i]/.test(v) ? { open: env * 0.35, wide: env * 0.7, round: 0 } :
/[e]/.test(v) ? { open: env * 0.45, wide: env * 0.55, round: 0 } :
/* a / fallback */ { open: env * 0.75, wide: env * 0.2, round: 0 };
try {
mouthTarget.setMouthShape(shape);
} catch {}
raf = requestAnimationFrame(tick);
};
raf = requestAnimationFrame(tick);
return {
stop() {
stopped = true;
if (raf) cancelAnimationFrame(raf);
try {
mouthTarget.setMouthShape({ open: 0, wide: 0, round: 0 });
} catch {}
},
};
}
// ── Head pose extraction (YXZ Tait-Bryan) ──────────────────────────────────
function decodeYawPitchRoll(m) {
const r02 = m[2], r12 = m[6], r22 = m[10], r10 = m[4], r11 = m[5];
const pitch = Math.asin(Math.max(-1, Math.min(1, -r12)));
let yaw, roll;
if (Math.abs(r12) < 0.9999) {
yaw = Math.atan2(r02, r22);
roll = Math.atan2(r10, r11);
} else {
yaw = Math.atan2(-m[8], m[0]);
roll = 0;
}
return { yaw, pitch, roll };
}
// ── Animation picker UI ─────────────────────────────────────────────────────
function buildAnimPicker(defs, animMgr) {
const toggle = document.getElementById('anim-toggle');
const panel = document.getElementById('anim-picker');
if (!toggle || !panel) return;
toggle.style.display = 'inline-flex';
function render() {
panel.innerHTML = '';
const header = document.createElement('div');
header.className = 'anim-picker-header';
const title = document.createElement('span');
title.className = 'anim-picker-title';
title.textContent = 'Animations';
const stopBtn = document.createElement('button');
stopBtn.className = 'anim-stop-btn';
stopBtn.textContent = '⏹ Stop';
stopBtn.addEventListener('click', () => {
animMgr.stopAll();
render();
});
header.appendChild(title);
header.appendChild(stopBtn);
panel.appendChild(header);
const grid = document.createElement('div');
grid.className = 'anim-chip-grid';
for (const def of defs) {
const chip = document.createElement('button');
chip.className = 'anim-chip' + (animMgr.currentName === def.name ? ' active' : '');
chip.title = def.label || def.name;
const icon = document.createElement('span');
icon.className = 'anim-chip-icon';
icon.textContent = def.icon || '▶';
const label = document.createElement('span');
label.className = 'anim-chip-label';
label.textContent = def.label || def.name;
chip.appendChild(icon);
chip.appendChild(label);
chip.addEventListener('click', () => {
animMgr.crossfadeTo(def.name, 0.35).then(render).catch(() => {});
});
grid.appendChild(chip);
}
panel.appendChild(grid);
}
animMgr.onChange = () => render();
render();
toggle.addEventListener('click', () => {
const open = document.body.classList.toggle('anim-picker-open');
toggle.setAttribute('aria-expanded', String(open));
});
}
// ── Error ───────────────────────────────────────────────────────────────────
function showError(msg) {
const el = document.getElementById('error');
if (!el) return;
el.textContent = msg;
el.style.display = 'flex';
}
// ── Misc utilities ──────────────────────────────────────────────────────────
function escapeHtmlSafe(s) {
if (s == null) return '';
return String(s)
.replace(/&/g, '&')
.replace(/</g, '<')
.replace(/>/g, '>')
.replace(/"/g, '"')
.replace(/'/g, ''');
}
function clamp01s(n) {
const x = Number(n);
if (!Number.isFinite(x)) return 0;
if (x < 0) return 0;
if (x > 1) return 1;
return x;
}