Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title data-i18n="demos_lipsync_mic.meta_title">Audio-driven lipsync · three.ws</title>
<meta name="description" content="Real-time mic → viseme analyser → avatar morph targets. The exact pipeline that drives ElevenLabs-streamed TTS in three.ws chat, sourced from your microphone." data-i18n-attr="content:demos_lipsync_mic.meta_description"/>
<link rel="canonical" href="https://three.ws/lipsync/mic" />
<meta property="og:title" content="Audio-driven lipsync · three.ws" data-i18n-attr="content:demos_lipsync_mic.meta_og_title"/>
<meta property="og:description" content="Speak into your mic and watch the avatar's mouth track your voice in real time." data-i18n-attr="content:demos_lipsync_mic.meta_og_description"/>
<meta property="og:url" content="https://three.ws/lipsync/mic" />
<meta name="twitter:card" content="summary_large_image" />
<link rel="stylesheet" href="/style.css" />
<link rel="stylesheet" href="/nav.css" />
<style>
:root { color-scheme: dark; }
body { background: #060606; color: #eee; font: 15px/1.5 -apple-system, system-ui, sans-serif; margin: 0; }
.stage { max-width: 1000px; margin: 0 auto; padding: 48px 24px 80px; }
.crumb a { color: #8a8d98; text-decoration: none; }
.crumb a:hover { color: #3dc1ff; }
h1 { font-size: 28px; font-weight: 600; letter-spacing: -0.015em; margin: 14px 0 8px; }
.sub { color: #8a8d98; max-width: 64ch; margin: 0 0 30px; }
.row { display: grid; grid-template-columns: 1fr 320px; gap: 24px; }
@media (max-width: 800px) { .row { grid-template-columns: 1fr; } }
.viewer-host {
background: #101010; border: 1.5px solid #202020; border-radius: 14px;
min-height: 460px; position: relative; overflow: hidden;
}
.viewer-host canvas { width: 100%; height: 100%; display: block; }
.panel {
background: #101010; border: 1.5px solid #202020; border-radius: 14px;
padding: 20px; display: flex; flex-direction: column; gap: 12px;
}
button {
background: #3dc1ff; color: #0a0a0a; border: 0; border-radius: 9px;
padding: 10px 14px; font-weight: 600; cursor: pointer; font: inherit;
}
button.secondary { background: rgba(255,255,255,0.06); color: #eee; }
button:disabled { opacity: 0.4; cursor: not-allowed; }
.meter {
display: flex; gap: 4px; height: 16px; align-items: end;
}
.meter .bar {
flex: 1; height: 0%; background: linear-gradient(180deg, #3dc1ff, #5a7a1a);
border-radius: 2px; transition: height var(--duration-instant) var(--ease-standard);
}
.viseme-row { display: flex; justify-content: space-between; font-size: 12px; color: #8a8d98; }
.viseme-row code { color: #3dc1ff; font-weight: 600; }
.log {
background: #080808; border: 1px solid #202020; border-radius: 9px;
padding: 12px; font-family: ui-monospace, monospace; font-size: 12px;
color: #8a8d98; line-height: 1.6;
}
.mic-status {
border-radius: 9px; padding: 12px 14px; font-size: 13px; line-height: 1.55;
display: flex; flex-direction: column; gap: 8px;
}
.mic-status[hidden] { display: none; }
.mic-status.error {
background: rgba(255, 86, 86, 0.08);
border: 1px solid rgba(255, 86, 86, 0.3);
color: #ffb4b4;
}
.mic-status.info {
background: rgba(61, 193, 255, 0.07);
border: 1px solid rgba(61, 193, 255, 0.25);
color: #aee3ff;
}
.mic-status strong { color: #fff; font-weight: 600; }
.mic-status .retry {
align-self: flex-start; cursor: pointer; font: inherit; font-weight: 600;
color: #fff; background: rgba(255, 255, 255, 0.08);
border: 1px solid rgba(255, 255, 255, 0.18); border-radius: 7px;
padding: 6px 12px; transition: background var(--duration-fast) var(--ease-standard), transform var(--duration-fast) var(--ease-standard);
}
.mic-status .retry:hover { background: rgba(255, 255, 255, 0.16); }
.mic-status .retry:active { transform: translateY(1px); }
.mic-status .retry:focus-visible { outline: 2px solid #3dc1ff; outline-offset: 2px; }
</style>
<meta property="og:type" content="website">
<meta property="og:site_name" content="three.ws">
<meta property="og:image" content="https://three.ws/api/page-og?s=labs&t=Lipsync+%28Mic%29&d=Feed+live+mic+audio+through+the+viseme+analyser+and+watch+the+avatar%27s+mouth+track+your+voice+in+real+time.&p=%2Flipsync%2Fmic">
<meta property="og:image:width" content="1200">
<meta property="og:image:height" content="630">
<meta name="twitter:title" content="Lipsync (Mic) · three.ws" data-i18n-attr="content:demos_lipsync_mic.meta_twitter_title">
<meta name="twitter:description" content="Feed live mic audio through the viseme analyser and watch the avatar's mouth track your voice in real time." data-i18n-attr="content:demos_lipsync_mic.meta_twitter_description">
<meta name="twitter:image" content="https://three.ws/api/page-og?s=labs&t=Lipsync+%28Mic%29&d=Feed+live+mic+audio+through+the+viseme+analyser+and+watch+the+avatar%27s+mouth+track+your+voice+in+real+time.&p=%2Flipsync%2Fmic">
<script type="application/ld+json">{"@context":"https://schema.org","@type":"WebPage","name":"Lipsync (Mic)","description":"Feed live mic audio through the viseme analyser and watch the avatar's mouth track your voice in real time.","url":"https://three.ws/lipsync/mic","isPartOf":{"@type":"WebSite","name":"three.ws","url":"https://three.ws"}}</script>
<meta name="robots" content="index, follow, max-image-preview:large, max-snippet:-1">
<meta property="og:locale" content="en_US">
<meta property="og:image:alt" content="Lipsync (Mic) — three.ws">
<meta name="twitter:site" content="@trythreews">
</head>
<body>
<header><div id="nav-container"></div></header>
<main class="stage">
<p class="crumb" data-i18n-html="demos_lipsync_mic.three_ws_lipsync_mic"><a href="/">three.ws</a> · <a href="/lipsync">lipsync</a> · mic</p>
<h1 data-i18n="demos_lipsync_mic.audio_driven_lipsync">Audio-driven lipsync</h1>
<p class="sub" data-i18n-html="demos_lipsync_mic.real_time_test_of_the_three_ws">
Real-time test of the three.ws viseme analyser
(<code>src/lip-sync-analyser.js</code>). Hit <strong>Start</strong>,
allow mic access, and speak — the analyser hands per-viseme weights
to the avatar's morph targets every frame. This is the exact pipeline
that drives ElevenLabs-streamed TTS in the live chat, just sourced
from your mic so you can verify the mouth shapes against your speech.
</p>
<div class="row">
<div class="viewer-host" id="viewer-host"></div>
<div class="panel">
<button id="start" data-i18n="demos_lipsync_mic.start_mic">Start mic</button>
<button id="stop" class="secondary" disabled data-i18n="demos_lipsync_mic.stop">Stop</button>
<div id="mic-status" class="mic-status" role="status" aria-live="polite" hidden></div>
<div class="meter" id="meter">
<div class="bar"></div><div class="bar"></div><div class="bar"></div>
<div class="bar"></div><div class="bar"></div><div class="bar"></div>
<div class="bar"></div><div class="bar"></div><div class="bar"></div>
</div>
<div id="visemes" style="display:flex;flex-direction:column;gap:4px;margin-top:6px"></div>
<div class="log">
The avatar's <code>viseme_aa / E / O / SS / PP …</code> morph
targets are driven by frequency-band energy from the mic. Your
audio never leaves the browser — analysis is all
<code>AnalyserNode</code> + <code>requestAnimationFrame</code>.
</div>
</div>
</div>
</main>
<script src="/nav.js" defer></script>
<script type="module">
import * as THREE from 'three';
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
import { LipSyncAnalyser } from '/src/lip-sync-analyser.js';
const $ = (id) => document.getElementById(id);
const host = $('viewer-host');
// ── three.js scene ─────────────────────────────────────────────
const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
renderer.setPixelRatio(window.devicePixelRatio);
renderer.setSize(host.clientWidth, host.clientHeight);
renderer.outputColorSpace = THREE.SRGBColorSpace;
host.appendChild(renderer.domElement);
const scene = new THREE.Scene();
const camera = new THREE.PerspectiveCamera(28, host.clientWidth / host.clientHeight, 0.01, 100);
camera.position.set(0, 1.55, 0.85);
const controls = new OrbitControls(camera, renderer.domElement);
controls.target.set(0, 1.55, 0);
controls.enableDamping = true;
scene.add(new THREE.HemisphereLight(0xffffff, 0x404060, 1.1));
const dir = new THREE.DirectionalLight(0xffffff, 1.4);
dir.position.set(2, 4, 3);
scene.add(dir);
window.addEventListener('resize', () => {
const w = host.clientWidth, h = host.clientHeight;
renderer.setSize(w, h);
camera.aspect = w / h;
camera.updateProjectionMatrix();
});
// ── Load the default avatar ────────────────────────────────────
let visemeMorphs = new Map(); // viseme_name -> [{ mesh, index }]
const VISEMES = [
'viseme_aa', 'viseme_O', 'viseme_E', 'viseme_I', 'viseme_nn',
'viseme_SS', 'viseme_FF', 'viseme_CH', 'viseme_PP',
];
const loader = new GLTFLoader();
function onAvatarLoaded(gltf) {
scene.add(gltf.scene);
gltf.scene.traverse((node) => {
if (!node.isMesh || !node.morphTargetDictionary || !node.morphTargetInfluences) return;
for (const v of VISEMES) {
const idx = node.morphTargetDictionary[v];
if (idx === undefined) continue;
if (!visemeMorphs.has(v)) visemeMorphs.set(v, []);
visemeMorphs.get(v).push({ mesh: node, index: idx });
}
});
const row = $('visemes');
for (const v of VISEMES) {
const found = visemeMorphs.has(v);
const el = document.createElement('div');
el.className = 'viseme-row';
el.innerHTML = `<span>${v}</span><code data-v="${v}">${found ? '0.00' : 'n/a'}</code>`;
row.appendChild(el);
}
}
// Server-baked avatars (the /api/avatars/<id>/glb lane) and Forge output emit
// EXT_meshopt_compression; GLTFLoader throws "setMeshoptDecoder must be called
// before loading compressed files" unless the decoder is registered before the
// load starts. Wire ONLY the meshopt decoder — this page loads no Draco/KTX2
// GLBs — the same lazy way walk-sdk/src/internal/meshopt.js does. Uncompressed
// rigs ignore it harmlessly. Not top-level await: the default vite build target
// includes Safari 14, which can't parse TLA.
import('three/addons/libs/meshopt_decoder.module.js')
.then(({ MeshoptDecoder }) => {
loader.setMeshoptDecoder(MeshoptDecoder);
loader.load('/avatars/default.glb', onAvatarLoaded);
})
.catch((err) => console.error('[lipsync] avatar load failed', err));
// ── Animation loop ─────────────────────────────────────────────
let analyser = null;
let stream = null;
const bars = document.querySelectorAll('#meter .bar');
function tick() {
requestAnimationFrame(tick);
controls.update();
if (analyser) {
const visemes = analyser.sample();
if (visemes) {
let i = 0;
for (const [name, targets] of visemeMorphs.entries()) {
const w = visemes[name] ?? 0;
for (const { mesh, index } of targets) {
mesh.morphTargetInfluences[index] = w;
}
const code = document.querySelector(`code[data-v="${name}"]`);
if (code) code.textContent = w.toFixed(2);
if (bars[i]) bars[i].style.height = `${Math.min(100, w * 100)}%`;
i++;
}
}
}
renderer.render(scene, camera);
}
tick();
// ── Mic status (designed permission / device states) ───────────
const micStatus = $('mic-status');
function clearMicStatus() {
micStatus.hidden = true;
micStatus.className = 'mic-status';
micStatus.innerHTML = '';
}
function showMicStatus(kind, title, body, retry) {
micStatus.className = `mic-status ${kind}`;
micStatus.innerHTML = `<span><strong>${title}</strong> ${body}</span>`;
if (retry) {
const btn = document.createElement('button');
btn.type = 'button';
btn.className = 'retry';
btn.textContent = 'Try again';
btn.addEventListener('click', () => $('start').click());
micStatus.appendChild(btn);
}
micStatus.hidden = false;
}
// ── Mic wiring ─────────────────────────────────────────────────
$('start').addEventListener('click', async () => {
clearMicStatus();
// No mic API at all (insecure origin / unsupported browser).
if (!navigator.mediaDevices?.getUserMedia) {
showMicStatus(
'error',
'Microphone not available.',
'Your browser blocks mic capture here — this demo needs a secure (https) context and a browser with Web Audio support.',
false,
);
return;
}
$('start').disabled = true;
try {
stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const ctx = new (window.AudioContext || window.webkitAudioContext)();
const src = ctx.createMediaStreamSource(stream);
const an = ctx.createAnalyser();
an.fftSize = 256;
an.smoothingTimeConstant = 0.7;
src.connect(an);
// We deliberately do NOT connect an → ctx.destination — keeps mic from
// echoing back through the speakers.
analyser = new LipSyncAnalyser();
analyser.connect(an);
$('stop').disabled = false;
} catch (err) {
$('start').disabled = false;
// Map the DOMException to a designed, actionable state.
switch (err?.name) {
case 'NotAllowedError':
case 'SecurityError':
showMicStatus(
'error',
'Microphone blocked.',
'Access was denied. Click the mic icon in your browser’s address bar, allow access for three.ws, then try again.',
true,
);
break;
case 'NotFoundError':
case 'OverconstrainedError':
showMicStatus(
'error',
'No microphone found.',
'Connect a microphone or check your system input settings, then try again.',
true,
);
break;
case 'NotReadableError':
showMicStatus(
'error',
'Microphone is busy.',
'Another app is using your mic. Close it (e.g. a call or recorder) and try again.',
true,
);
break;
default:
showMicStatus(
'error',
'Couldn’t start the microphone.',
err?.message || 'An unexpected error occurred. Try again.',
true,
);
}
}
});
$('stop').addEventListener('click', () => {
clearMicStatus();
if (analyser) { analyser.disconnect(); analyser = null; }
if (stream) { stream.getTracks().forEach((t) => t.stop()); stream = null; }
$('start').disabled = false;
$('stop').disabled = true;
// Reset morph values to zero.
for (const targets of visemeMorphs.values()) {
for (const { mesh, index } of targets) mesh.morphTargetInfluences[index] = 0;
}
});
</script>
<script src="/brand.js" defer></script>
<div id="footer-container"></div>
<script src="/footer.js"></script>
<!-- Runtime i18n: detects locale, swaps annotated copy, mounts <lang-switcher>. -->
<script type="module" src="/i18n.js"></script>
</body>
</html>