Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

17.7 KB
Raw Blame
<!doctype html>
<html lang="en">
	<head>
		<meta charset="utf-8" />
		<meta name="viewport" content="width=device-width, initial-scale=1" />
		<title data-i18n="demos_lipsync_tts.meta_title">TTS-driven lipsync · three.ws</title>
		<meta name="description" content="Type text, stream it through OpenAI TTS, and watch a 3D avatar's mouth animate in real time via wawa-lipsync viseme detection. No server-side audio processing."  data-i18n-attr="content:demos_lipsync_tts.meta_description"/>
		<link rel="canonical" href="https://three.ws/lipsync" />
		<meta property="og:title" content="TTS-driven lipsync · three.ws"  data-i18n-attr="content:demos_lipsync_tts.meta_og_title"/>
		<meta property="og:description" content="Type any text → OpenAI TTS → in-browser viseme detection → 3D avatar lipsync. Zero server-side audio processing."  data-i18n-attr="content:demos_lipsync_tts.meta_og_description"/>
		<meta property="og:url" content="https://three.ws/lipsync" />
		<meta name="twitter:card" content="summary_large_image" />
		<link rel="stylesheet" href="/style.css" />
		<link rel="stylesheet" href="/nav.css" />
		<style>
			:root { color-scheme: dark; }
			body { background: #060606; color: #eee; font: 15px/1.5 -apple-system, system-ui, sans-serif; margin: 0; }
			.stage { max-width: 1080px; margin: 0 auto; padding: 48px 24px 80px; }
			.crumb a { color: #8a8d98; text-decoration: none; }
			.crumb a:hover { color: #3dc1ff; }
			h1 { font-size: 28px; font-weight: 600; letter-spacing: -0.015em; margin: 14px 0 8px; }
			.sub { color: #8a8d98; max-width: 64ch; margin: 0 0 26px; }
			.row { display: grid; grid-template-columns: 1fr 320px; gap: 22px; }
			@media (max-width: 880px) { .row { grid-template-columns: 1fr; } }
			.viewer-host { background: #101010; border: 1.5px solid #202020; border-radius: 14px; min-height: 480px; position: relative; overflow: hidden; }
			.viewer-host canvas { display: block; width: 100%; height: 100%; }
			.viewer-label { position: absolute; top: 12px; left: 12px; background: rgba(0,0,0,0.55); padding: 4px 10px; border-radius: 999px; font-size: 11.5px; color: #3dc1ff; font-weight: 600; }
			.panel { background: #101010; border: 1.5px solid #202020; border-radius: 14px; padding: 18px; display: flex; flex-direction: column; gap: 14px; }
			.panel textarea { background: #080808; border: 1px solid #202020; color: #eee; padding: 11px 12px; border-radius: 9px; font: inherit; min-height: 92px; resize: vertical; }
			.panel select { background: #080808; border: 1px solid #202020; color: #eee; padding: 8px 10px; border-radius: 9px; font: inherit; }
			.panel label { font-size: 11px; color: #8a8d98; text-transform: uppercase; letter-spacing: 0.08em; }
			.panel button { background: #3dc1ff; color: #0a0a0a; border: 0; border-radius: 9px; padding: 11px 14px; font-weight: 600; cursor: pointer; font: inherit; }
			.panel button.secondary { background: rgba(255,255,255,0.06); color: #eee; }
			.panel button:disabled { opacity: 0.4; cursor: not-allowed; }
			.viseme-row { display: flex; justify-content: space-between; font-size: 12px; color: #8a8d98; font-family: ui-monospace, monospace; }
			.viseme-row code { color: #3dc1ff; font-weight: 600; }
			.bar { display: inline-block; height: 6px; vertical-align: middle; border-radius: 3px; background: rgba(61, 193, 255,0.18); margin-left: 8px; }
			.bar > span { display: block; height: 100%; background: #3dc1ff; border-radius: 3px; transition: width var(--duration-instant) var(--ease-standard); }
			.log { background: #080808; border: 1px solid #202020; border-radius: 9px; padding: 12px; font-family: ui-monospace, monospace; font-size: 11.5px; color: #8a8d98; line-height: 1.55; min-height: 60px; max-height: 140px; overflow-y: auto; }
			.log .ok { color: #3dc1ff; }
			.log .err { color: #ff8a8a; }
			.log .info { color: #9ad4ff; }
			.tts-error {
				background: rgba(255, 86, 86, 0.08); border: 1px solid rgba(255, 86, 86, 0.3);
				color: #ffb4b4; border-radius: 9px; padding: 11px 13px; font-size: 13px;
				line-height: 1.5; display: flex; align-items: center; gap: 10px; justify-content: space-between;
			}
			.tts-error[hidden] { display: none; }
			.tts-error span { flex: 1; }
			.tts-error strong { color: #fff; font-weight: 600; }
			.tts-error .retry {
				flex: 0 0 auto; cursor: pointer; font: inherit; font-weight: 600; color: #0a0a0a;
				background: #ff8a8a; border: 0; border-radius: 7px; padding: 7px 13px;
				transition: filter var(--duration-fast) var(--ease-standard), transform var(--duration-fast) var(--ease-standard);
			}
			.tts-error .retry:hover { filter: brightness(1.08); }
			.tts-error .retry:active { transform: translateY(1px); }
			.tts-error .retry:focus-visible { outline: 2px solid #fff; outline-offset: 2px; }
		</style>
	
	<meta property="og:type" content="website">
	<meta property="og:site_name" content="three.ws">
	<meta property="og:image" content="https://three.ws/api/page-og?s=labs&amp;t=Lipsync+%28TTS%29&amp;d=Type+text%2C+stream+it+through+the+avatar%27s+voice%2C+and+the+mouth+animates+in+real+time+via+wawa-lipsync+viseme+detection.&amp;p=%2Flipsync">
	<meta property="og:image:width" content="1200">
	<meta property="og:image:height" content="630">
	<meta name="twitter:title" content="Lipsync (TTS) · three.ws" data-i18n-attr="content:demos_lipsync_tts.meta_twitter_title">
	<meta name="twitter:description" content="Type text, stream it through OpenAI TTS, and the avatar's mouth animates in real time via wawa-lipsync viseme detection." data-i18n-attr="content:demos_lipsync_tts.meta_twitter_description">
	<meta name="twitter:image" content="https://three.ws/api/page-og?s=labs&amp;t=Lipsync+%28TTS%29&amp;d=Type+text%2C+stream+it+through+the+avatar%27s+voice%2C+and+the+mouth+animates+in+real+time+via+wawa-lipsync+viseme+detection.&amp;p=%2Flipsync">
	<script type="application/ld+json">{"@context":"https://schema.org","@type":"WebPage","name":"Lipsync (TTS)","description":"Type text, stream it through OpenAI TTS, and the avatar's mouth animates in real time via wawa-lipsync viseme detection.","url":"https://three.ws/lipsync","isPartOf":{"@type":"WebSite","name":"three.ws","url":"https://three.ws"}}</script>

	<meta name="robots" content="index, follow, max-image-preview:large, max-snippet:-1">
	<meta property="og:locale" content="en_US">
	<meta property="og:image:alt" content="Lipsync (TTS) — three.ws">
	<meta name="twitter:site" content="@trythreews">
</head>
	<body>
		<header><div id="nav-container"></div></header>

		<main class="stage">
			<p class="crumb" data-i18n-html="demos_lipsync_tts.three_ws_lipsync_mic_mode"><a href="/">three.ws</a> · lipsync · <a href="/lipsync/mic">mic mode</a></p>
			<h1 data-i18n="demos_lipsync_tts.tts_driven_lipsync">TTS-driven lipsync</h1>
			<p class="sub" data-i18n-html="demos_lipsync_tts.type_any_text_stream_it_through_openai">
				Type any text, stream it through OpenAI TTS, and the avatar's mouth animates in
				real time. Visemes are extracted in-browser via <code>wawa-lipsync</code>
				analysing the audio's frequency bands — no server-side audio processing, no
				per-word timing data, no plugins.
			</p>

			<div class="row">
				<div class="viewer-host" id="viewer-host">
					<div class="viewer-label">live · TTS → viseme → morph</div>
				</div>
				<div class="panel">
					<div>
						<label for="text" data-i18n="demos_lipsync_tts.text_to_speak">Text to speak</label>
						<textarea id="text">Welcome to three dot ws. This avatar is talking through real-time viseme detection, powered entirely in the browser.</textarea>
					</div>

					<div style="display:flex;gap:8px">
						<div style="flex:1">
							<label for="voice" data-i18n="demos_lipsync_tts.voice">Voice</label>
							<select id="voice" style="width:100%">
								<option value="nova" selected data-i18n="demos_lipsync_tts.nova">Nova</option>
								<option value="alloy" data-i18n="demos_lipsync_tts.alloy">Alloy</option>
								<option value="echo" data-i18n="demos_lipsync_tts.echo">Echo</option>
								<option value="fable" data-i18n="demos_lipsync_tts.fable">Fable</option>
								<option value="onyx" data-i18n="demos_lipsync_tts.onyx">Onyx</option>
								<option value="shimmer" data-i18n="demos_lipsync_tts.shimmer">Shimmer</option>
								<option value="ash" data-i18n="demos_lipsync_tts.ash">Ash</option>
								<option value="coral" data-i18n="demos_lipsync_tts.coral">Coral</option>
								<option value="sage" data-i18n="demos_lipsync_tts.sage">Sage</option>
							</select>
						</div>
						<div style="flex:1">
							<label for="speed" data-i18n="demos_lipsync_tts.speed">Speed</label>
							<select id="speed" style="width:100%">
								<option value="0.8">0.8×</option>
								<option value="1.0" selected>1.0×</option>
								<option value="1.2">1.2×</option>
								<option value="1.5">1.5×</option>
							</select>
						</div>
					</div>

					<div style="display:flex;gap:8px">
						<button id="speak" style="flex:1" data-i18n="demos_lipsync_tts.speak">Speak</button>
						<button id="stop" class="secondary" disabled data-i18n="demos_lipsync_tts.stop">Stop</button>
					</div>

					<div id="tts-error" class="tts-error" role="alert" aria-live="assertive" hidden></div>

					<div>
						<label data-i18n="demos_lipsync_tts.visemes_live">Visemes (live)</label>
						<div id="visemes" style="display:flex;flex-direction:column;gap:5px;margin-top:6px"></div>
					</div>

					<div class="log" id="log"></div>
				</div>
			</div>
		</main>

		<script src="/nav.js" defer></script>
		<script type="module">
			import * as THREE from 'three';
			import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
			import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
			import { Lipsync } from 'wawa-lipsync';

			const $ = (id) => document.getElementById(id);
			const host = $('viewer-host');
			const logEl = $('log');

			function log(cls, msg) {
				const div = document.createElement('div');
				div.className = cls;
				div.textContent = msg;
				logEl.appendChild(div);
				logEl.scrollTop = logEl.scrollHeight;
			}

			const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
			renderer.setPixelRatio(window.devicePixelRatio);
			renderer.setSize(host.clientWidth, host.clientHeight);
			renderer.outputColorSpace = THREE.SRGBColorSpace;
			host.appendChild(renderer.domElement);

			const scene = new THREE.Scene();
			const camera = new THREE.PerspectiveCamera(28, host.clientWidth / host.clientHeight, 0.01, 100);
			camera.position.set(0, 1.55, 0.85);
			const controls = new OrbitControls(camera, renderer.domElement);
			controls.target.set(0, 1.55, 0);
			controls.enableDamping = true;
			controls.minDistance = 0.4;
			controls.maxDistance = 3.0;

			scene.add(new THREE.HemisphereLight(0xffffff, 0x404060, 1.1));
			const dir = new THREE.DirectionalLight(0xffffff, 1.4);
			dir.position.set(2, 4, 3);
			scene.add(dir);

			window.addEventListener('resize', () => {
				const w = host.clientWidth, h = host.clientHeight;
				renderer.setSize(w, h);
				camera.aspect = w / h;
				camera.updateProjectionMatrix();
			});

			// wawa-lipsync produces visemes: sil, aa, PP, FF, TH, DD, kk, CH, SS, nn, RR, ou
			// Our default avatar's morph targets follow the Oculus naming:
			// viseme_aa, viseme_O, viseme_E, viseme_I, viseme_nn, viseme_SS, viseme_FF, viseme_CH, viseme_PP
			const VISEME_MAP = {
				aa: 'viseme_aa',
				PP: 'viseme_PP',
				FF: 'viseme_FF',
				TH: 'viseme_E',
				DD: 'viseme_E',
				kk: 'viseme_E',
				CH: 'viseme_CH',
				SS: 'viseme_SS',
				nn: 'viseme_nn',
				RR: 'viseme_O',
				ou: 'viseme_O',
				sil: null,
			};
			const MORPH_NAMES = [...new Set(Object.values(VISEME_MAP).filter(Boolean))];

			const morphRefs = new Map();
			const targetWeights = new Map();
			for (const m of MORPH_NAMES) targetWeights.set(m, 0);

			const visemesEl = $('visemes');
			const bars = new Map();
			for (const code of Object.keys(VISEME_MAP)) {
				if (code === 'sil') continue;
				const row = document.createElement('div');
				row.className = 'viseme-row';
				row.innerHTML = `<span>${code} → ${VISEME_MAP[code]}</span><span class="bar" style="width:120px"><span data-v="${code}" style="width:0%"></span></span>`;
				visemesEl.appendChild(row);
				bars.set(code, row.querySelector(`[data-v="${code}"]`));
			}

			let avatarReady = false;
			new GLTFLoader().load('/avatars/default.glb', (gltf) => {
				scene.add(gltf.scene);
				gltf.scene.traverse((node) => {
					if (!node.isMesh || !node.morphTargetDictionary || !node.morphTargetInfluences) return;
					for (const m of MORPH_NAMES) {
						const idx = node.morphTargetDictionary[m];
						if (idx === undefined) continue;
						if (!morphRefs.has(m)) morphRefs.set(m, []);
						morphRefs.get(m).push({ mesh: node, index: idx });
					}
				});
				avatarReady = true;
				log('ok', `Avatar loaded · ${morphRefs.size}/${MORPH_NAMES.length} viseme morphs wired`);
			}, undefined, (err) => log('err', `Avatar load failed: ${err.message}`));

			const lipsync = new Lipsync();
			let audioEl = null;
			let isPlaying = false;
			let currentURL = null;

			function tick() {
				requestAnimationFrame(tick);
				controls.update();

				if (isPlaying && audioEl) {
					lipsync.processAudio();
					const v = lipsync.viseme || 'sil';
					for (const m of MORPH_NAMES) targetWeights.set(m, 0);
					const morph = VISEME_MAP[v];
					if (morph) targetWeights.set(morph, 1.0);
					for (const code of bars.keys()) {
						bars.get(code).style.width = (code === v ? 100 : 0) + '%';
					}
				} else {
					for (const m of MORPH_NAMES) targetWeights.set(m, 0);
					for (const bar of bars.values()) bar.style.width = '0%';
				}

				if (avatarReady) {
					for (const [m, w] of targetWeights.entries()) {
						const refs = morphRefs.get(m);
						if (!refs) continue;
						for (const { mesh, index } of refs) {
							const prev = mesh.morphTargetInfluences[index];
							mesh.morphTargetInfluences[index] = prev + (w - prev) * 0.5;
						}
					}
				}

				renderer.render(scene, camera);
			}
			tick();

			const speakBtn = $('speak');
			const stopBtn = $('stop');
			const ttsErr = $('tts-error');

			function clearTtsError() { ttsErr.hidden = true; ttsErr.innerHTML = ''; }
			// Persistent, actionable error banner with one-click retry — survives
			// after the log scrolls and tells the user how to recover.
			function showTtsError(title, detail) {
				ttsErr.innerHTML = `<span><strong>${title}</strong> ${detail}</span>`;
				const btn = document.createElement('button');
				btn.type = 'button';
				btn.className = 'retry';
				btn.textContent = 'Retry';
				btn.addEventListener('click', () => speak());
				ttsErr.appendChild(btn);
				ttsErr.hidden = false;
			}

			async function speak() {
				const text = $('text').value.trim();
				if (!text) { log('err', 'Enter text first.'); return; }

				clearTtsError();
				speakBtn.disabled = true;
				stopBtn.disabled = false;
				log('info', `Requesting TTS · voice=${$('voice').value} speed=${$('speed').value}`);

				let res;
				try {
					res = await fetch('/api/tts/speak', {
						method: 'POST',
						headers: { 'content-type': 'application/json' },
						body: JSON.stringify({
							text,
							voice: $('voice').value,
							speed: parseFloat($('speed').value),
							format: 'mp3',
						}),
					});
				} catch (err) {
					log('err', `TTS request failed: ${err.message}`);
					showTtsError(
						'Couldn’t reach the TTS service.',
						'Check your connection and try again.',
					);
					speakBtn.disabled = false;
					stopBtn.disabled = true;
					return;
				}

				if (!res.ok) {
					const txt = await res.text().catch(() => '');
					log('err', `TTS error · HTTP ${res.status} · ${txt.slice(0, 200)}`);
					const detail =
						res.status === 429
							? 'Rate limit reached — wait a moment and retry.'
							: res.status >= 500
								? 'The speech service hit an error. Try again in a moment.'
								: res.status === 400
									? 'The request was rejected — try shorter or different text.'
									: `HTTP ${res.status}.`;
					showTtsError('Speech generation failed.', detail);
					speakBtn.disabled = false;
					stopBtn.disabled = true;
					return;
				}

				const blob = await res.blob();
				if (currentURL) URL.revokeObjectURL(currentURL);
				currentURL = URL.createObjectURL(blob);

				if (audioEl) {
					audioEl.pause();
					audioEl.src = '';
				}
				audioEl = new Audio();
				audioEl.crossOrigin = 'anonymous';
				audioEl.src = currentURL;

				audioEl.addEventListener('play', () => {
					isPlaying = true;
					log('ok', `Playing · ${Math.round(blob.size / 1024)} KB · audio/mpeg`);
				});
				audioEl.addEventListener('ended', () => {
					isPlaying = false;
					speakBtn.disabled = false;
					stopBtn.disabled = true;
				});
				audioEl.addEventListener('error', () => {
					isPlaying = false;
					speakBtn.disabled = false;
					stopBtn.disabled = true;
					log('err', 'Audio playback error.');
					showTtsError('Playback failed.', 'The generated audio couldn’t be played. Try again.');
				});

				lipsync.connectAudio(audioEl);

				try {
					await audioEl.play();
				} catch (err) {
					log('err', `Playback blocked: ${err.message}`);
					showTtsError(
						'Playback was blocked.',
						'Your browser needs a click to play audio — press Retry to try again.',
					);
					speakBtn.disabled = false;
					stopBtn.disabled = true;
				}
			}

			function stop() {
				if (audioEl) {
					audioEl.pause();
					audioEl.currentTime = 0;
				}
				isPlaying = false;
				speakBtn.disabled = false;
				stopBtn.disabled = true;
				log('info', 'Stopped.');
			}

			speakBtn.addEventListener('click', speak);
			stopBtn.addEventListener('click', stop);
		</script>
		<script src="/brand.js" defer></script>
		<div id="footer-container"></div>
		<script src="/footer.js"></script>
	<!-- Runtime i18n: detects locale, swaps annotated copy, mounts <lang-switcher>. -->
	<script type="module" src="/i18n.js"></script>
</body>
</html>