Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(avatar): add the LLM-composable gesture tool, active-listening nods, and real multi-language viseme timing
nirholas572e2a132b79
diff --git a/api/tts/eleven.js b/api/tts/eleven.jsindex f52751fac..4013d62b5 100644--- a/api/tts/eleven.js+++ b/api/tts/eleven.js@@ -28,8 +28,15 @@* stability?, similarity_boost?, style?, // clamped to 0..1* use_speaker_boost? // boolean* }+ * withTimestamps?: boolean // see below* }- * Response: audio/mpeg (chunked)+ * Response: audio/mpeg (chunked), or when `withTimestamps` is true:+ * application/json { audio_base64, alignment, normalized_alignment }, character-level+ * timing from ElevenLabs' `/with-timestamps` endpoint, in whatever language/script the+ * text was written in. The client (src/runtime/speech.js) builds a real per-character+ * viseme sequence from `alignment` instead of estimating visemes from audio energy.+ * Same billing and cache lanes as the audio path; the only difference is upstream URL,+ * accept header, and what gets cached (the JSON body instead of raw MP3 bytes).** Auth: a session or bearer token, except on the agent_byok lane — an agent whose* owner bound their own key speaks to anonymous visitors too (rate limited per IP),@@ -53,13 +60,15 @@ import {normalizeVoiceSettings,} from '../_lib/elevenlabs.js';-// Fire-and-forget R2 cache write — a miss on failure is acceptable.-function cacheAudio(key, buffer) {+// Fire-and-forget R2 cache write. A miss on failure is acceptable. Caches+// exactly the bytes served to the client, so a cache hit is byte-identical+// whether that's raw MP3 or the with-timestamps JSON envelope.+function cacheAudio(key, buffer, contentType) {if (!buffer.length) return;putObject({key,body: buffer,- contentType: 'audio/mpeg',+ contentType,metadata: { 'created-at': new Date().toISOString() },}).catch((e) => console.warn('[tts/eleven] R2 cache write failed:', e.message));}@@ -73,6 +82,7 @@ export default wrap(async (req, res) => {const text = String(body.text || '').trim();const modelId = String(body.modelId || DEFAULT_TTS_MODEL).trim();const agentId = String(body.agentId || '').trim();+ const withTimestamps = body.withTimestamps === true;if (!voiceId) return error(res, 400, 'validation_error', 'voiceId is required');if (!text) return error(res, 400, 'validation_error', 'text is required');@@ -145,15 +155,16 @@ export default wrap(async (req, res) => {// Checked BEFORE any metering: a cached clip costs nothing upstream, so it// never consumes the free budget and never charges credits.const cacheHash = await sha256(- `${voiceId}\x00${text}\x00${modelId}\x00${settings.stability}\x00${settings.similarity_boost}\x00${settings.style}\x00${settings.use_speaker_boost}`,+ `${voiceId}\x00${text}\x00${modelId}\x00${settings.stability}\x00${settings.similarity_boost}\x00${settings.style}\x00${settings.use_speaker_boost}\x00${withTimestamps ? 'ts' : 'audio'}`,);- const cacheKey = `tts/cache/${cacheHash}.mp3`;+ const responseContentType = withTimestamps ? 'application/json' : 'audio/mpeg';+ const cacheKey = `tts/cache/${cacheHash}.${withTimestamps ? 'json' : 'mp3'}`;const cached = await headObject(cacheKey).catch(() => null);if (cached) {try {const buf = await getObjectBuffer(cacheKey);- res.setHeader('content-type', 'audio/mpeg');+ res.setHeader('content-type', responseContentType);res.setHeader('content-length', String(buf.length));res.setHeader('x-tts-cache', 'hit');res.setHeader('x-tts-billing', 'cached');@@ -220,29 +231,31 @@ export default wrap(async (req, res) => {};// ── Synthesize via ElevenLabs ─────────────────────────────────────────────+ // The with-timestamps endpoint returns one JSON object (audio + alignment),+ // not a streamable MP3, so it has no /stream segment and takes a json accept.+ const upstreamPath = withTimestamps+ ? `/text-to-speech/${encodeURIComponent(voiceId)}/with-timestamps`+ : `/text-to-speech/${encodeURIComponent(voiceId)}/stream`;let elResp;try {- elResp = await fetch(- `${ELEVEN_BASE}/text-to-speech/${encodeURIComponent(voiceId)}/stream`,- {- method: 'POST',- headers: {- 'content-type': 'application/json',- accept: 'audio/mpeg',- 'xi-api-key': apiKey,- },- body: JSON.stringify({- text,- model_id: modelId,- voice_settings: settings,- }),- // Bound a hung connection (ElevenLabs accepting but never responding)- // under Vercel's 30s kill, so a stall returns a clean 502 + char refund- // instead of a 504 with characters charged but no audio delivered. Set- // well above a real clip's synth+stream time so it never cuts valid audio.- signal: AbortSignal.timeout(28000),+ elResp = await fetch(`${ELEVEN_BASE}${upstreamPath}`, {+ method: 'POST',+ headers: {+ 'content-type': 'application/json',+ accept: withTimestamps ? 'application/json' : 'audio/mpeg',+ 'xi-api-key': apiKey,},- );+ body: JSON.stringify({+ text,+ model_id: modelId,+ voice_settings: settings,+ }),+ // Bound a hung connection (ElevenLabs accepting but never responding)+ // under Vercel's 30s kill, so a stall returns a clean 502 + char refund+ // instead of a 504 with characters charged but no audio delivered. Set+ // well above a real clip's synth+stream time so it never cuts valid audio.+ signal: AbortSignal.timeout(28000),+ });} catch (fetchErr) {console.error('[tts/eleven] ElevenLabs fetch failed', fetchErr);await refundMetering();@@ -269,21 +282,35 @@ export default wrap(async (req, res) => {);}- // Stream the clip to the client for low TTFB while teeing the chunks into a- // buffer, so the complete audio can be cached once it has fully arrived.- res.setHeader('content-type', 'audio/mpeg');+ res.setHeader('content-type', responseContentType);res.setHeader('x-tts-cache', 'miss');res.setHeader('x-tts-billing', billing);if (creditCharge?.chargedUsd > 0)res.setHeader('x-tts-charged-usd', creditCharge.chargedUsd.toFixed(6));res.setHeader('cache-control', 'private, max-age=86400');+ if (withTimestamps) {+ // One JSON object, not a stream: buffer it whole, forward it, cache it.+ let jsonBuffer;+ try {+ jsonBuffer = Buffer.from(await elResp.arrayBuffer());+ } catch (readErr) {+ console.error('[tts/eleven] with-timestamps read failed', readErr);+ await refundMetering();+ return error(res, 502, 'upstream_error', 'ElevenLabs with-timestamps response unreadable');+ }+ res.setHeader('content-length', String(jsonBuffer.length));+ res.end(jsonBuffer);+ cacheAudio(cacheKey, jsonBuffer, responseContentType);+ return;+ }+if (!elResp.body) {// No readable stream (unexpected) — fall back to a single buffered write.const audioBuffer = Buffer.from(await elResp.arrayBuffer());res.setHeader('content-length', String(audioBuffer.length));res.end(audioBuffer);- cacheAudio(cacheKey, audioBuffer);+ cacheAudio(cacheKey, audioBuffer, responseContentType);return;}@@ -323,5 +350,5 @@ export default wrap(async (req, res) => {if (!res.writableEnded && !res.destroyed) res.end();// Cache only a complete clip — a truncated one would poison the deterministic// cache key and be served as a permanent "hit" for this utterance.- if (completed) cacheAudio(cacheKey, Buffer.concat(chunks));+ if (completed) cacheAudio(cacheKey, Buffer.concat(chunks), responseContentType);});diff --git a/public/agent-3d-companion-harness.html b/public/agent-3d-companion-harness.htmlnew file mode 100644index 000000000..b6a13a05c--- /dev/null+++ b/public/agent-3d-companion-harness.html@@ -0,0 +1,22 @@+<!doctype html>+<html>+<head>+<meta charset="utf-8">+<title>agent-3d companion presence test harness</title>+<!--+ Playwright-only fixture for tests/e2e/agent-3d-companion-presence.spec.js.+ Imports /src/element.js directly so the spec exercises today's source instead+ of the published https://three.ws/agent-3d/latest/agent-3d.js bundle that+ every real page loads. Mirrors public/agent-3d-ar-harness.html's convention.+ Loads a real production avatar asset (public/avatars/default.glb) already+ checked into the repo, so mood/gesture/viseme morph targets and the+ humanoid skeleton the baked clip library retargets onto are real, not+ synthetic geometry.+-->+<style>html,body{margin:0;height:100%}agent-3d{display:block;width:480px;height:480px}</style>+</head>+<body>+<agent-3d id="a3d" eager src="/avatars/default.glb"></agent-3d>+<script type="module" src="/src/element.js"></script>+</body>+</html>diff --git a/src/agent-protocol.js b/src/agent-protocol.jsindex 0be23d860..8ee14d42d 100644--- a/src/agent-protocol.js+++ b/src/agent-protocol.js@@ -30,6 +30,8 @@ export const ACTION_TYPES = {PRESENCE: 'presence', // agent came online / went idlePROTOCOL_ERROR: 'protocol-error', // emitted by the rate limiter on cascade detectionINTERRUPTED: 'interrupted', // user spoke while agent was mid-TTS+ LISTEN_START: 'listen-start', // user started speaking (VAD/STT signal)+ LISTEN_END: 'listen-end', // user stopped speakingPERMISSIONS_REDEEM_START: 'permissions.redeem.start',PERMISSIONS_REDEEM_SUCCESS: 'permissions.redeem.success',PERMISSIONS_REDEEM_ERROR: 'permissions.redeem.error',diff --git a/src/element.js b/src/element.jsindex 2858e7be7..50e0f48cd 100644--- a/src/element.js+++ b/src/element.js@@ -2194,6 +2194,7 @@ class Agent3DElement extends HTMLElement {'voice:speech-end','voice:transcript','voice:listen-start',+ 'voice:listen-end','voice:stt-status','memory:write',]) {@@ -2226,6 +2227,16 @@ class Agent3DElement extends HTMLElement {}if (this._chatEl) this._renderMessage(detail);}+ if (ev === 'voice:listen-start') {+ // The mic opened for this turn (generic STT loop in runtime/index.js).+ // Route through the protocol bus so AgentAvatar's _onListenStart+ // drives the same attentive gaze + backchannel nod cadence the+ // LiveKit realtime voice path already triggers from its own VAD.+ protocol.emit({ type: ACTION_TYPES.LISTEN_START, payload: {} });+ }+ if (ev === 'voice:listen-end') {+ protocol.emit({ type: ACTION_TYPES.LISTEN_END, payload: {} });+ }if (ev === 'brain:stream') {if (this._thoughtBubbleEl && this.getAttribute('avatar-chat') !== 'off') {this._streamToBubble(e.detail?.chunk ?? '');diff --git a/src/runtime/index.js b/src/runtime/index.jsindex 17ecddc8c..707140828 100644--- a/src/runtime/index.js+++ b/src/runtime/index.js@@ -303,27 +303,33 @@ export class Runtime extends EventTarget {async listen({ onInterim, onFinal, onStatus } = {}) {if (!this.stt) throw new Error('STT not configured');this.dispatchEvent(new CustomEvent('voice:listen-start', {}));- const text = await this.stt.listen({- // Recognizers that do real work before text exists (an on-device model- // downloading, an utterance decoding) report it, so the UI can say so.- onStatus: (status) => {- this.dispatchEvent(new CustomEvent('voice:stt-status', { detail: status }));- onStatus?.(status);- },- onInterim: (t) => {- this.dispatchEvent(- new CustomEvent('voice:transcript', { detail: { text: t, final: false } }),- );- onInterim?.(t);- },- onFinal: (t) => {- this.dispatchEvent(- new CustomEvent('voice:transcript', { detail: { text: t, final: true } }),- );- onFinal?.(t);- },- });- return text;+ try {+ const text = await this.stt.listen({+ // Recognizers that do real work before text exists (an on-device model+ // downloading, an utterance decoding) report it, so the UI can say so.+ onStatus: (status) => {+ this.dispatchEvent(new CustomEvent('voice:stt-status', { detail: status }));+ onStatus?.(status);+ },+ onInterim: (t) => {+ this.dispatchEvent(+ new CustomEvent('voice:transcript', { detail: { text: t, final: false } }),+ );+ onInterim?.(t);+ },+ onFinal: (t) => {+ this.dispatchEvent(+ new CustomEvent('voice:transcript', { detail: { text: t, final: true } }),+ );+ onFinal?.(t);+ },+ });+ return text;+ } finally {+ // Always fires, success, error or cancellation, so a caller driving the+ // avatar's attentive posture off listen-start never gets stuck in it.+ this.dispatchEvent(new CustomEvent('voice:listen-end', {}));+ }}clearConversation() {diff --git a/src/runtime/lipsync.js b/src/runtime/lipsync.jsindex 25951082a..a012d4022 100644--- a/src/runtime/lipsync.js+++ b/src/runtime/lipsync.js@@ -178,6 +178,55 @@ export function playVisemeSequence(root, sequence, elapsedMs = () => 0) {};}+// Characters that carry no viseme of their own: true silence in the sequence,+// not just "unmapped". Mirrors the word/punctuation gaps _tokenize() inserts.+const SILENT_CHARS = new Set([' ', '\t', '\n', '.', ',', '!', '?', ';', ':', '"', "'", '-', ' ']);++// Any letter CHAR_TO_VISEME doesn't cover (most non-Latin scripts: CJK, Cyrillic,+// Arabic, Devanagari…) still gets a real per-character timestamp from ElevenLabs;+// it just can't pick a specific phoneme shape from a single codepoint. A generic+// open-mouth viseme keeps the timing real rather than falling silent on every+// language the heuristic alphabet-based lookup wasn't written for.+const DEFAULT_LETTER_VISEME = 'viseme_aa';++/**+ * Build a timed viseme sequence from ElevenLabs' character-level alignment+ * (`/with-timestamps`): `{ characters, character_start_times_seconds,+ * character_end_times_seconds }`. Real per-character timing in whatever+ * language the text was written in, unlike {@link startLipsync}'s English-letter+ * duration estimate. Used by the ElevenLabs TTS lane (runtime/speech.js) with+ * the energy-based {@link LipSyncAnalyser} path as its fallback.+ *+ * @param {{characters?: string[], character_start_times_seconds?: number[], character_end_times_seconds?: number[]}} alignment+ * @returns {Array<{viseme:string,startMs:number,endMs:number}>}+ */+export function alignmentToSequence(alignment) {+ const chars = alignment?.characters;+ const starts = alignment?.character_start_times_seconds;+ const ends = alignment?.character_end_times_seconds;+ if (!Array.isArray(chars) || !Array.isArray(starts) || !Array.isArray(ends)) return [];++ const sequence = [];+ for (let i = 0; i < chars.length; i++) {+ const raw = chars[i];+ if (!raw) continue;+ const ch = raw.toLowerCase();+ if (SILENT_CHARS.has(ch)) continue;++ const viseme = CHAR_TO_VISEME[ch] || (/\S/.test(ch) ? DEFAULT_LETTER_VISEME : null);+ if (!viseme) continue;++ const startMs = (starts[i] ?? 0) * 1000;+ const endRaw = ends[i] ?? starts[i + 1] ?? starts[i] ?? 0;+ // A zero-length entry (two characters sharing one timestamp) would never+ // activate in activeVisemeAt()'s half-open interval check, so floor it to+ // a visible minimum instead of letting the viseme silently vanish.+ const endMs = Math.max(startMs + 20, endRaw * 1000);+ sequence.push({ viseme, startMs, endMs });+ }+ return sequence;+}+/*** Drive jaw/mouth morph targets in sync with speech text via a phoneme heuristic.* Supports the ARKit viseme_* morph targets that three.ws avatars ship with (anddiff --git a/src/runtime/livekit-voice.js b/src/runtime/livekit-voice.jsindex 693fa14f5..7085b3b54 100644--- a/src/runtime/livekit-voice.js+++ b/src/runtime/livekit-voice.js@@ -3,6 +3,7 @@// data-channel messages with transcript payloads.import { Room, RoomEvent, Track } from 'livekit-client';+import { ACTION_TYPES } from '../agent-protocol.js';export class LiveKitVoice {/**@@ -66,12 +67,12 @@ export class LiveKitVoice {// User speech boundaries (VAD from agent server)if (msg.type === 'speech_started') {this._userSpeaking = true;- this._protocol.emit({ type: 'listen-start', payload: {} });+ this._protocol.emit({ type: ACTION_TYPES.LISTEN_START, payload: {} });}if (msg.type === 'speech_ended') {this._userSpeaking = false;this._protocol.emit({- type: 'listen-end',+ type: ACTION_TYPES.LISTEN_END,payload: { text: msg.transcript || '' },});}@@ -83,10 +84,10 @@ export class LiveKitVoice {const active = speakers.some((p) => p.identity === localId);if (active && !this._userSpeaking) {this._userSpeaking = true;- this._protocol.emit({ type: 'listen-start', payload: {} });+ this._protocol.emit({ type: ACTION_TYPES.LISTEN_START, payload: {} });} else if (!active && this._userSpeaking) {this._userSpeaking = false;- this._protocol.emit({ type: 'listen-end', payload: { text: '' } });+ this._protocol.emit({ type: ACTION_TYPES.LISTEN_END, payload: { text: '' } });}});diff --git a/src/runtime/speech.js b/src/runtime/speech.jsindex 4ffd23c4b..3ba6190d8 100644--- a/src/runtime/speech.js+++ b/src/runtime/speech.js@@ -5,8 +5,9 @@ export { NeuralTTS } from './neural-tts.js';export { WhisperSTT } from './whisper-stt.js';import { NeuralTTS } from './neural-tts.js';import { WhisperSTT, onDeviceSupported } from './whisper-stt.js';-import { startLipsync } from './lipsync.js';+import { startLipsync, alignmentToSequence, playVisemeSequence } from './lipsync.js';import { ACTION_TYPES } from '../agent-protocol.js';+import { log } from '../shared/log.js';export class BrowserTTS {constructor({ voiceId = 'default', rate = 1, pitch = 1, lang = 'en-US' } = {}) {@@ -235,10 +236,140 @@ export class ElevenLabsTTS {this._positionalAudio = pa;}+ /**+ * Speak `text` aloud. Tries real per-character viseme timing first+ * ({@link _speakWithTimestamps}, ElevenLabs' `/with-timestamps`) whenever+ * there's an avatar `scene` to animate, since that's the only reason to pay+ * for the slower non-streaming round trip. Any failure there (network,+ * malformed response, playback error before a frame plays) falls back to+ * the original low-latency streaming path with energy-based lipsync+ * ({@link _speakStreaming}), so a transient ElevenLabs hiccup never silences+ * the agent.+ */async speak(text, { onStart, onEnd, scene } = {}) {this.cancel();if (!text) return;+ if (scene) {+ try {+ return await this._speakWithTimestamps(text, { onStart, onEnd, scene });+ } catch (err) {+ if (err?.name === 'AbortError') return;+ log.warn(+ '[ElevenLabsTTS] with-timestamps failed, falling back to energy-based lipsync:',+ err?.message || err,+ );+ this.cancel();+ }+ }+ return this._speakStreaming(text, { onStart, onEnd, scene });+ }++ /**+ * Real multi-language viseme timing: ElevenLabs' `/with-timestamps` endpoint+ * returns the full clip as `audio_base64` plus character-level `alignment`+ * (start/end seconds per character, in whatever script the text used). No+ * MediaSource streaming: the response is one JSON object, not chunked MP3,+ * so first-audio latency is higher than {@link _speakStreaming}; the real+ * timing is the point.+ */+ async _speakWithTimestamps(text, { onStart, onEnd, scene }) {+ this._abort = new AbortController();+ this._speaking = true;++ const url =+ this.proxyURL ||+ `https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(this.voiceId)}/with-timestamps`;+ const headers = { 'content-type': 'application/json', accept: 'application/json' };+ if (!this.proxyURL && this.apiKey) headers['xi-api-key'] = this.apiKey;++ const rateNorm = Math.min(1.5, Math.max(0.5, this.rate));+ const styleVal =+ this.style != null+ ? Math.min(1, Math.max(0, Number(this.style)))+ : Math.min(1, Math.max(0, (rateNorm - 0.5) / 1.0));++ const bodyObj = {+ text,+ model_id: this.modelId,+ voice_settings: {+ stability: this.stability,+ similarity_boost: this.similarityBoost,+ style: styleVal,+ use_speaker_boost: this.useSpeakerBoost,+ },+ voiceId: this.voiceId,+ modelId: this.modelId,+ ...(this.agentId ? { agentId: this.agentId } : {}),+ };+ // Only the proxy understands this flag; the direct ElevenLabs API reads+ // the /with-timestamps URL path instead.+ if (this.proxyURL) bodyObj.withTimestamps = true;++ const resp = await fetch(url, {+ method: 'POST',+ headers,+ body: JSON.stringify(bodyObj),+ signal: this._abort.signal,+ });+ if (!resp.ok) throw new Error(`ElevenLabs with-timestamps HTTP ${resp.status}`);+ const data = await resp.json();+ if (!data?.audio_base64) throw new Error('ElevenLabs with-timestamps returned no audio');++ const binary = atob(data.audio_base64);+ const bytes = new Uint8Array(binary.length);+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);+ const blob = new Blob([bytes], { type: 'audio/mpeg' });++ const sequence = alignmentToSequence(data.alignment || data.normalized_alignment);++ const audioURL = URL.createObjectURL(blob);+ this._mediaSourceURL = audioURL;+ const audio = new Audio(audioURL);+ this._audio = audio;+ // No analyser hookup here: a stale analyser from an earlier energy-path+ // speak() on this same instance would otherwise leak into AgentAvatar's+ // connectLipSync() and fight this sequence's direct morph writes for the+ // same viseme targets every frame.+ this._analyserNode = null;+ this._connectPositionalAudio(audio);++ return new Promise((resolve, reject) => {+ this._onEnd = () => {+ this._speaking = false;+ this._lipsync?.stop();+ this._lipsync = null;+ this._cleanupAudio();+ this.onEnd?.();+ onEnd?.();+ this._onEnd = null;+ resolve();+ };+ audio.addEventListener(+ 'playing',+ () => {+ if (sequence.length) {+ this._lipsync = playVisemeSequence(scene, sequence, () => audio.currentTime * 1000);+ }+ this.onStart?.();+ onStart?.();+ },+ { once: true },+ );+ audio.addEventListener('ended', () => this._onEnd?.(), { once: true });+ audio.addEventListener(+ 'error',+ () => {+ if (this._onEnd) this._onEnd();+ },+ { once: true },+ );+ audio.play().catch(reject);+ });+ }++ /** Original streaming path: low first-audio latency, energy-based lipsync via LipSyncAnalyser. */+ async _speakStreaming(text, { onStart, onEnd, scene }) {this._abort = new AbortController();this._speaking = true;@@ -363,6 +494,12 @@ export class ElevenLabsTTS {try {if (!this._audioCtx || this._audioCtx.state === 'closed') {this._audioCtx = new AC();+ }+ // Recreated independently of the context: a prior _speakWithTimestamps()+ // call on this same instance nulls _analyserNode (see that method) while+ // leaving _audioCtx open, so this must not assume "context exists" implies+ // "analyser exists".+ if (!this._analyserNode) {this._analyserNode = this._audioCtx.createAnalyser();this._analyserNode.fftSize = 256;this._analyserNode.smoothingTimeConstant = 0.7;diff --git a/src/runtime/tools.js b/src/runtime/tools.jsindex 48ca72a4d..54f8927c5 100644--- a/src/runtime/tools.js+++ b/src/runtime/tools.js@@ -2,6 +2,13 @@// See specs/AGENT_MANIFEST.md § Built-in tools.import { log } from '../shared/log.js';+import { SLOTS } from './animation-slots.js';++// Gestures are the subset of the slot vocabulary (animation-slots.js) an agent+// can request by name as a deliberate body-language choice. 'idle' is the rest+// state, not a performable gesture, so it is excluded from the tool's enum.+const GESTURE_NAMES = SLOTS.filter((s) => s !== 'idle');+export const BUILTIN_TOOLS = [{name: 'wave',@@ -41,6 +48,23 @@ export const BUILTIN_TOOLS = [required: ['name'],},},+ {+ name: 'gesture',+ description:+ 'Perform a body-language gesture. Use this to react physically while speaking or ' ++ 'listening: nod to agree or acknowledge, shake the head to disagree, wave to greet or ' ++ 'say goodbye, shrug when unsure, bow to thank or apologize, point to direct attention, ' ++ 'celebrate good news, show concern for bad news, think while working something out, ' ++ 'inspect something closely, present a result, or signal patience, curiosity, or fidgety ' ++ 'restlessness. The gesture plays once and the body returns to its resting pose.',+ input_schema: {+ type: 'object',+ properties: {+ name: { type: 'string', enum: GESTURE_NAMES, description: 'Which gesture to perform.' },+ },+ required: ['name'],+ },+ },{name: 'setExpression',description: 'Set facial expression via morph target preset.',@@ -155,6 +179,17 @@ export const BUILTIN_HANDLERS = {return { ok: true, name, resolved };},+ async gesture(args, ctx) {+ const { name } = args;+ const resolved = ctx.viewer.resolveAnimationSlot?.(name) ?? name;+ const played = ctx.viewer.playClipByName?.(resolved, { loop: false, fade_ms: 200 });+ if (!played) {+ log.warn(`[gesture] slot "${name}" → "${resolved}" not found in library`);+ return { ok: false, error: `No clip for gesture "${name}"` };+ }+ return { ok: true, name, resolved };+ },+async setExpression(args, ctx) {const { preset, intensity = 1 } = args;ctx.viewer.setExpression?.(preset, intensity);diff --git a/tests/agent-avatar-active-listening.test.js b/tests/agent-avatar-active-listening.test.jsnew file mode 100644index 000000000..a4a1634cd--- /dev/null+++ b/tests/agent-avatar-active-listening.test.js@@ -0,0 +1,127 @@+/**+ * AgentAvatar, active-listening idle reactions (Stage 2c of `_tickEmotion()`).+ *+ * `_onListenStart`/`_onListenEnd` are the handlers the protocol bus invokes+ * when a real VAD/STT signal fires (never a fabricated loading timer); see+ * `_sub(ACTION_TYPES.LISTEN_START, ...)` in `attach()`. Called directly here,+ * mirroring the direct-`_tickEmotion()` approach in+ * tests/agent-avatar-mood-gesture.test.js, so no protocol bus or DOM is needed.+ */++import { describe, it, expect, beforeEach } from 'vitest';++let AgentAvatar;++function makeRoot() {+ return {+ isObject3D: true,+ children: [],+ traverse(fn) {+ fn(this);+ },+ };+}++function makeFakeAnimationManager(loadedNames = []) {+ const loaded = new Set(loadedNames);+ const played = [];+ return {+ currentName: null,+ isLoaded: (name) => loaded.has(name),+ getAnimationDefs: () => [],+ play: (name) => {+ played.push(name);+ return Promise.resolve(true);+ },+ crossfadeTo: (name) => {+ played.push(name);+ },+ played,+ };+}++function makeFakeViewer(am) {+ return {+ content: makeRoot(),+ animationManager: am,+ state: {},+ };+}++function makeAvatar(am) {+ const viewer = makeFakeViewer(am);+ const protocol = {}; // _onListenStart/_onListenEnd are called directly, bus unused+ const identity = { id: 'test' };+ return new AgentAvatar(viewer, protocol, identity);+}++beforeEach(async () => {+ if (!AgentAvatar) {+ const mod = await import('../src/agent-avatar.js');+ AgentAvatar = mod.AgentAvatar;+ }+});++describe('AgentAvatar, active-listening wiring', () => {+ it('_onListenStart sets the attentive-gaze flag the head-transform priority chain reads', () => {+ const avatar = makeAvatar(makeFakeAnimationManager(['nod']));+ expect(avatar._userSpeaking).toBe(false);+ avatar._onListenStart();+ expect(avatar._userSpeaking).toBe(true);+ });++ it('_onListenEnd clears the attentive-gaze flag', () => {+ const avatar = makeAvatar(makeFakeAnimationManager(['nod']));+ avatar._onListenStart();+ avatar._onListenEnd();+ expect(avatar._userSpeaking).toBe(false);+ });++ it('nods on a fixed cadence while the user speaks, driven only by setUserSpeaking (no timer of its own)', () => {+ const am = makeFakeAnimationManager(['nod']);+ const avatar = makeAvatar(am);++ avatar._onListenStart();+ // LISTENING_NOD_INTERVAL_S is 2.8s: tick just under it, nothing yet.+ avatar._tickEmotion(2.7);+ expect(am.played).not.toContain('nod');++ // Cross the threshold: one nod fires.+ avatar._tickEmotion(0.2);+ expect(am.played.filter((n) => n === 'nod')).toHaveLength(1);+ });++ it('stops nodding immediately once the user stops speaking', () => {+ const am = makeFakeAnimationManager(['nod']);+ const avatar = makeAvatar(am);++ avatar._onListenStart();+ avatar._tickEmotion(2.9); // one nod fires+ expect(am.played.filter((n) => n === 'nod')).toHaveLength(1);++ avatar._onListenEnd();+ am.played.length = 0;+ avatar._tickEmotion(10); // would have nodded 3x more if still "listening"+ expect(am.played).not.toContain('nod');+ });++ it('never nods when the user never speaks (idle avatar, no real signal)', () => {+ const am = makeFakeAnimationManager(['nod']);+ const avatar = makeAvatar(am);++ for (let i = 0; i < 20; i++) avatar._tickEmotion(1);+ expect(am.played).not.toContain('nod');+ });++ it('defers the backchannel nod while a higher-priority one-shot gesture is already playing', () => {+ const am = makeFakeAnimationManager(['nod', 'wave']);+ const avatar = makeAvatar(am);++ avatar._onListenStart();+ avatar._playSlot('wave', 5.0); // long one-shot in flight, sets _isPlayingOneShot+ am.played.length = 0;++ avatar._tickEmotion(3.0); // would cross the nod threshold if unblocked+ expect(am.played).not.toContain('nod');+ });+});diff --git a/tests/alignment-to-sequence.test.js b/tests/alignment-to-sequence.test.jsnew file mode 100644index 000000000..6580f6b03--- /dev/null+++ b/tests/alignment-to-sequence.test.js@@ -0,0 +1,71 @@+// Tests for alignmentToSequence(): turns ElevenLabs' /with-timestamps+// character-level alignment into the {viseme,startMs,endMs} shape+// playVisemeSequence() expects. Pure function, no AudioContext/DOM needed.++import { describe, it, expect } from 'vitest';+import { alignmentToSequence, activeVisemeAt } from '../src/runtime/lipsync.js';++function alignment(chars, startSec, durationSec = 0.08) {+ const starts = [];+ const ends = [];+ let t = startSec;+ for (let i = 0; i < chars.length; i++) {+ starts.push(t);+ ends.push(t + durationSec);+ t += durationSec;+ }+ return { characters: chars.split(''), character_start_times_seconds: starts, character_end_times_seconds: ends };+}++describe('alignmentToSequence', () => {+ it('maps known Latin letters to their viseme with real millisecond timing', () => {+ const seq = alignmentToSequence(alignment('ba', 0, 0.1));+ expect(seq).toEqual([+ { viseme: 'viseme_PP', startMs: 0, endMs: 100 },+ { viseme: 'viseme_aa', startMs: 100, endMs: 200 },+ ]);+ });++ it('skips whitespace and punctuation as true silence, not a viseme', () => {+ const seq = alignmentToSequence(alignment('a, b.', 0, 0.1));+ const visemes = seq.map((s) => s.viseme);+ expect(visemes).toEqual(['viseme_aa', 'viseme_PP']);+ });++ it('falls back to a generic open-mouth viseme for non-Latin scripts', () => {+ // CJK/Cyrillic/Arabic characters have no entry in CHAR_TO_VISEME, but still+ // carry a real per-character timestamp from ElevenLabs, so the timing must+ // not be dropped just because the exact mouth shape is unknown.+ const seq = alignmentToSequence(alignment('日本', 0, 0.12));+ expect(seq).toEqual([+ { viseme: 'viseme_aa', startMs: 0, endMs: 120 },+ { viseme: 'viseme_aa', startMs: 120, endMs: 240 },+ ]);+ });++ it('floors zero-length entries to a visible minimum duration', () => {+ // Two characters sharing one timestamp (end == start) would never satisfy+ // activeVisemeAt()'s half-open interval check if left at 0 width.+ const alignmentData = {+ characters: ['a', 'b'],+ character_start_times_seconds: [0, 0.05],+ character_end_times_seconds: [0.05, 0.05],+ };+ const seq = alignmentToSequence(alignmentData);+ expect(seq[1].endMs).toBeGreaterThan(seq[1].startMs);+ expect(seq[1].endMs - seq[1].startMs).toBeGreaterThanOrEqual(20);+ });++ it('produces a sequence activeVisemeAt() can actually scan', () => {+ const seq = alignmentToSequence(alignment('hi', 0, 0.1));+ expect(activeVisemeAt(seq, 50)).toBe(seq[0].viseme);+ expect(activeVisemeAt(seq, 150)).toBe(seq[1].viseme);+ expect(activeVisemeAt(seq, 1000)).toBeNull();+ });++ it('returns an empty array when alignment data is missing or malformed', () => {+ expect(alignmentToSequence(undefined)).toEqual([]);+ expect(alignmentToSequence({})).toEqual([]);+ expect(alignmentToSequence({ characters: ['a'] })).toEqual([]);+ });+});diff --git a/tests/e2e/agent-3d-companion-presence.spec.js b/tests/e2e/agent-3d-companion-presence.spec.jsnew file mode 100644index 000000000..ef0ce8f1c--- /dev/null+++ b/tests/e2e/agent-3d-companion-presence.spec.js@@ -0,0 +1,132 @@+import { test, expect } from '@playwright/test';++// Work order 082: companion presence (mood, emoji-driven expression, the+// LLM-composable gesture tool, and active-listening idle reactions). Drives+// a real Chromium against the dev server's /src/element.js (not the+// published CDN bundle) using public/agent-3d-companion-harness.html, which+// loads a real production avatar GLB (public/avatars/default.glb) already+// checked into the repo. No GLB bytes are route-intercepted and no app+// logic is mocked: every assertion reads the real mesh's+// `morphTargetInfluences`, the real `AnimationManager`'s loaded clip, or the+// real `AgentAvatar` emotion state that drives them.++function watchErrors(page) {+ const errors = [];+ page.on('pageerror', (err) => errors.push(String(err?.message || err)));+ return errors;+}++async function ready(page) {+ await page.evaluate(() => document.getElementById('a3d')._waitForReady());+ // AgentAvatar resolves the GLB's morph targets synchronously at+ // construction time, but construction itself is queued behind model+ // load; wait for a non-empty resolution before reading morph weights.+ await expect+ .poll(() => page.evaluate(() => document.getElementById('a3d')._avatar?._morphResolved?.size ?? 0), {+ timeout: 60_000,+ })+ .toBeGreaterThan(0);+}++function mouthSmileWeight(page) {+ return page.evaluate(() => {+ const avatar = document.getElementById('a3d')._avatar;+ const slots = avatar._morphResolved.get('mouthSmileLeft') || avatar._morphResolved.get('mouthSmile');+ if (!slots?.length) return null;+ return slots[0].mesh.morphTargetInfluences[slots[0].index];+ });+}++function mouthFrownWeight(page) {+ return page.evaluate(() => {+ const avatar = document.getElementById('a3d')._avatar;+ const slots = avatar._morphResolved.get('mouthFrownLeft') || avatar._morphResolved.get('mouthFrown');+ if (!slots?.length) return null;+ return slots[0].mesh.morphTargetInfluences[slots[0].index];+ });+}++test.describe('agent-3d companion presence (mood, emoji, gesture, active listening)', () => {+ test('setMood drives a real smile onto the loaded avatar mesh', async ({ page }) => {+ const errors = watchErrors(page);+ await page.goto('/agent-3d-companion-harness.html', { waitUntil: 'domcontentloaded' });+ await ready(page);++ // The idle rig carries its own small baseline smile, so assert a real+ // increase from whatever baseline this GLB renders at rather than an+ // absolute near-zero floor.+ const baseline = await mouthSmileWeight(page);+ await page.evaluate(() => document.getElementById('a3d').setMood(0.9, 0.6));+ // The mood layer lerps in over ~1s (k = dt*1.2); give it real wall-clock time.+ await expect.poll(() => mouthSmileWeight(page), { timeout: 5_000 }).toBeGreaterThan(baseline + 0.2);++ expect(errors, `page errors: ${errors.join(' | ')}`).toEqual([]);+ });++ test('an emoji-only reply with no English sentiment words still smiles the avatar', async ({ page }) => {+ const errors = watchErrors(page);+ await page.goto('/agent-3d-companion-harness.html', { waitUntil: 'domcontentloaded' });+ await ready(page);++ const baseline = await mouthSmileWeight(page);+ await page.evaluate(() =>+ document.getElementById('a3d')._avatar._onSpeak({ payload: { text: 'here you go 🎉' } }),+ );+ await expect.poll(() => mouthSmileWeight(page), { timeout: 5_000 }).toBeGreaterThan(baseline + 0.1);++ expect(errors, `page errors: ${errors.join(' | ')}`).toEqual([]);+ });++ test('a sad emoji frowns the avatar instead', async ({ page }) => {+ const errors = watchErrors(page);+ await page.goto('/agent-3d-companion-harness.html', { waitUntil: 'domcontentloaded' });+ await ready(page);++ const baseline = await mouthFrownWeight(page);+ await page.evaluate(() =>+ document.getElementById('a3d')._avatar._onSpeak({ payload: { text: 'that did not work out 😢' } }),+ );+ await expect.poll(() => mouthFrownWeight(page), { timeout: 5_000 }).toBeGreaterThan(baseline + 0.1);++ expect(errors, `page errors: ${errors.join(' | ')}`).toEqual([]);+ });++ test('the gesture tool plays a real retargeted baked clip by slot name', async ({ page }) => {+ const errors = watchErrors(page);+ await page.goto('/agent-3d-companion-harness.html', { waitUntil: 'domcontentloaded' });+ await ready(page);++ const resolved = await page.evaluate(() => document.getElementById('a3d')._scene.resolveAnimationSlot('wave'));+ expect(resolved).toBe('wave');++ await page.evaluate(() => document.getElementById('a3d')._scene.playClipByName('wave', { loop: false }));+ await expect+ .poll(() => page.evaluate(() => document.getElementById('a3d')._viewer?.animationManager?.currentName), {+ timeout: 15_000,+ })+ .toBe('wave');++ expect(errors, `page errors: ${errors.join(' | ')}`).toEqual([]);+ });++ test('the avatar nods on a fixed cadence while the user speaks, and stops when they stop', async ({ page }) => {+ const errors = watchErrors(page);+ await page.goto('/agent-3d-companion-harness.html', { waitUntil: 'domcontentloaded' });+ await ready(page);++ await page.evaluate(() => document.getElementById('a3d')._avatar._onListenStart());+ // LISTENING_NOD_INTERVAL_S is 2.8s, plus the AnimationManager's own+ // lazy clip load over the dev server on first use; give both real time.+ await expect+ .poll(+ () => page.evaluate(() => document.getElementById('a3d')._viewer?.animationManager?.currentName),+ { timeout: 20_000 },+ )+ .toBe('nod');++ await page.evaluate(() => document.getElementById('a3d')._avatar._onListenEnd());+ expect(await page.evaluate(() => document.getElementById('a3d')._avatar._userSpeaking)).toBe(false);++ expect(errors, `page errors: ${errors.join(' | ')}`).toEqual([]);+ });+});diff --git a/tests/src/runtime.test.js b/tests/src/runtime.test.jsindex f2c3fbc0a..38324d96e 100644--- a/tests/src/runtime.test.js+++ b/tests/src/runtime.test.js@@ -1,7 +1,7 @@import { describe, it, expect, beforeEach, vi } from 'vitest';import { Runtime } from '../../src/runtime/index.js';import { NullProvider } from '../../src/runtime/providers.js';-import { BUILTIN_HANDLERS } from '../../src/runtime/tools.js';+import { BUILTIN_HANDLERS, BUILTIN_TOOLS } from '../../src/runtime/tools.js';// Disable voice in all Runtime instances — avoids BrowserTTS/BrowserSTT in testsconst NO_VOICE = { tts: { provider: 'none' }, stt: { provider: 'none' } };@@ -358,6 +358,41 @@ describe('Runtime — busy guard', () => {});});+// ── Runtime.listen() ──────────────────────────────────────────────────────────++describe('Runtime, listen() voice:listen-start/listen-end symmetry', () => {+ it('fires listen-start then listen-end around a successful transcript', async () => {+ const rt = makeRuntime(mockProvider([textResp('ok')]));+ rt.stt = { listen: vi.fn().mockResolvedValue('hello') };+ const events = [];+ rt.addEventListener('voice:listen-start', () => events.push('start'));+ rt.addEventListener('voice:listen-end', () => events.push('end'));++ const text = await rt.listen();+ expect(text).toBe('hello');+ expect(events).toEqual(['start', 'end']);+ });++ it('still fires listen-end when the STT call rejects (try/finally, not just the happy path)', async () => {+ const rt = makeRuntime(mockProvider([textResp('ok')]));+ rt.stt = { listen: vi.fn().mockRejectedValue(new Error('mic denied')) };+ const events = [];+ rt.addEventListener('voice:listen-start', () => events.push('start'));+ rt.addEventListener('voice:listen-end', () => events.push('end'));++ await expect(rt.listen()).rejects.toThrow('mic denied');+ expect(events).toEqual(['start', 'end']);+ });++ it('throws without dispatching anything when no STT is configured', async () => {+ const rt = makeRuntime(mockProvider([textResp('ok')]));+ rt.stt = null;+ const events = collect(rt, 'voice:listen-start');+ await expect(rt.listen()).rejects.toThrow('STT not configured');+ expect(events).toEqual([]);+ });+});+// ── BUILTIN_HANDLERS ──────────────────────────────────────────────────────────describe('BUILTIN_HANDLERS.speak', () => {@@ -420,3 +455,40 @@ describe('BUILTIN_HANDLERS.setExpression', () => {expect(ctx.viewer.setExpression).toHaveBeenCalledWith('sad', 1);});});++describe('gesture tool, LLM-composable body language', () => {+ it('enumerates every animation slot except the rest state "idle"', () => {+ const tool = BUILTIN_TOOLS.find((t) => t.name === 'gesture');+ const names = tool.input_schema.properties.name.enum;+ expect(names).toContain('nod');+ expect(names).toContain('wave');+ expect(names).not.toContain('idle');+ });++ it('resolves the slot name through resolveAnimationSlot before playing it', async () => {+ const ctx = {+ viewer: {+ resolveAnimationSlot: vi.fn().mockReturnValue('Armature|Nod'),+ playClipByName: vi.fn().mockReturnValue(true),+ },+ };+ const result = await BUILTIN_HANDLERS.gesture({ name: 'nod' }, ctx);+ expect(ctx.viewer.resolveAnimationSlot).toHaveBeenCalledWith('nod');+ expect(ctx.viewer.playClipByName).toHaveBeenCalledWith('Armature|Nod', {+ loop: false,+ fade_ms: 200,+ });+ expect(result).toEqual({ ok: true, name: 'nod', resolved: 'Armature|Nod' });+ });++ it('reports failure instead of throwing when the resolved clip is missing from the library', async () => {+ const ctx = {+ viewer: {+ resolveAnimationSlot: vi.fn().mockReturnValue('nod'),+ playClipByName: vi.fn().mockReturnValue(false),+ },+ };+ const result = await BUILTIN_HANDLERS.gesture({ name: 'nod' }, ctx);+ expect(result).toEqual({ ok: false, error: 'No clip for gesture "nod"' });+ });+});