Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(avatar): add the LLM-composable gesture tool, active-listening nods, and real multi-language viseme timing

nirholas572e2a132b79
diff --git a/api/tts/eleven.js b/api/tts/eleven.js
index f52751fac..4013d62b5 100644
--- a/api/tts/eleven.js
+++ b/api/tts/eleven.js
@@ -28,8 +28,15 @@
* stability?, similarity_boost?, style?, // clamped to 0..1
* use_speaker_boost? // boolean
* }
+ * withTimestamps?: boolean // see below
* }
- * Response: audio/mpeg (chunked)
+ * Response: audio/mpeg (chunked), or when `withTimestamps` is true:
+ * application/json { audio_base64, alignment, normalized_alignment }, character-level
+ * timing from ElevenLabs' `/with-timestamps` endpoint, in whatever language/script the
+ * text was written in. The client (src/runtime/speech.js) builds a real per-character
+ * viseme sequence from `alignment` instead of estimating visemes from audio energy.
+ * Same billing and cache lanes as the audio path; the only difference is upstream URL,
+ * accept header, and what gets cached (the JSON body instead of raw MP3 bytes).
*
* Auth: a session or bearer token, except on the agent_byok lane — an agent whose
* owner bound their own key speaks to anonymous visitors too (rate limited per IP),
@@ -53,13 +60,15 @@ import {
normalizeVoiceSettings,
} from '../_lib/elevenlabs.js';
-// Fire-and-forget R2 cache write — a miss on failure is acceptable.
-function cacheAudio(key, buffer) {
+// Fire-and-forget R2 cache write. A miss on failure is acceptable. Caches
+// exactly the bytes served to the client, so a cache hit is byte-identical
+// whether that's raw MP3 or the with-timestamps JSON envelope.
+function cacheAudio(key, buffer, contentType) {
if (!buffer.length) return;
putObject({
key,
body: buffer,
- contentType: 'audio/mpeg',
+ contentType,
metadata: { 'created-at': new Date().toISOString() },
}).catch((e) => console.warn('[tts/eleven] R2 cache write failed:', e.message));
}
@@ -73,6 +82,7 @@ export default wrap(async (req, res) => {
const text = String(body.text || '').trim();
const modelId = String(body.modelId || DEFAULT_TTS_MODEL).trim();
const agentId = String(body.agentId || '').trim();
+ const withTimestamps = body.withTimestamps === true;
if (!voiceId) return error(res, 400, 'validation_error', 'voiceId is required');
if (!text) return error(res, 400, 'validation_error', 'text is required');
@@ -145,15 +155,16 @@ export default wrap(async (req, res) => {
// Checked BEFORE any metering: a cached clip costs nothing upstream, so it
// never consumes the free budget and never charges credits.
const cacheHash = await sha256(
- `${voiceId}\x00${text}\x00${modelId}\x00${settings.stability}\x00${settings.similarity_boost}\x00${settings.style}\x00${settings.use_speaker_boost}`,
+ `${voiceId}\x00${text}\x00${modelId}\x00${settings.stability}\x00${settings.similarity_boost}\x00${settings.style}\x00${settings.use_speaker_boost}\x00${withTimestamps ? 'ts' : 'audio'}`,
);
- const cacheKey = `tts/cache/${cacheHash}.mp3`;
+ const responseContentType = withTimestamps ? 'application/json' : 'audio/mpeg';
+ const cacheKey = `tts/cache/${cacheHash}.${withTimestamps ? 'json' : 'mp3'}`;
const cached = await headObject(cacheKey).catch(() => null);
if (cached) {
try {
const buf = await getObjectBuffer(cacheKey);
- res.setHeader('content-type', 'audio/mpeg');
+ res.setHeader('content-type', responseContentType);
res.setHeader('content-length', String(buf.length));
res.setHeader('x-tts-cache', 'hit');
res.setHeader('x-tts-billing', 'cached');
@@ -220,29 +231,31 @@ export default wrap(async (req, res) => {
};
// ── Synthesize via ElevenLabs ─────────────────────────────────────────────
+ // The with-timestamps endpoint returns one JSON object (audio + alignment),
+ // not a streamable MP3, so it has no /stream segment and takes a json accept.
+ const upstreamPath = withTimestamps
+ ? `/text-to-speech/${encodeURIComponent(voiceId)}/with-timestamps`
+ : `/text-to-speech/${encodeURIComponent(voiceId)}/stream`;
let elResp;
try {
- elResp = await fetch(
- `${ELEVEN_BASE}/text-to-speech/${encodeURIComponent(voiceId)}/stream`,
- {
- method: 'POST',
- headers: {
- 'content-type': 'application/json',
- accept: 'audio/mpeg',
- 'xi-api-key': apiKey,
- },
- body: JSON.stringify({
- text,
- model_id: modelId,
- voice_settings: settings,
- }),
- // Bound a hung connection (ElevenLabs accepting but never responding)
- // under Vercel's 30s kill, so a stall returns a clean 502 + char refund
- // instead of a 504 with characters charged but no audio delivered. Set
- // well above a real clip's synth+stream time so it never cuts valid audio.
- signal: AbortSignal.timeout(28000),
+ elResp = await fetch(`${ELEVEN_BASE}${upstreamPath}`, {
+ method: 'POST',
+ headers: {
+ 'content-type': 'application/json',
+ accept: withTimestamps ? 'application/json' : 'audio/mpeg',
+ 'xi-api-key': apiKey,
},
- );
+ body: JSON.stringify({
+ text,
+ model_id: modelId,
+ voice_settings: settings,
+ }),
+ // Bound a hung connection (ElevenLabs accepting but never responding)
+ // under Vercel's 30s kill, so a stall returns a clean 502 + char refund
+ // instead of a 504 with characters charged but no audio delivered. Set
+ // well above a real clip's synth+stream time so it never cuts valid audio.
+ signal: AbortSignal.timeout(28000),
+ });
} catch (fetchErr) {
console.error('[tts/eleven] ElevenLabs fetch failed', fetchErr);
await refundMetering();
@@ -269,21 +282,35 @@ export default wrap(async (req, res) => {
);
}
- // Stream the clip to the client for low TTFB while teeing the chunks into a
- // buffer, so the complete audio can be cached once it has fully arrived.
- res.setHeader('content-type', 'audio/mpeg');
+ res.setHeader('content-type', responseContentType);
res.setHeader('x-tts-cache', 'miss');
res.setHeader('x-tts-billing', billing);
if (creditCharge?.chargedUsd > 0)
res.setHeader('x-tts-charged-usd', creditCharge.chargedUsd.toFixed(6));
res.setHeader('cache-control', 'private, max-age=86400');
+ if (withTimestamps) {
+ // One JSON object, not a stream: buffer it whole, forward it, cache it.
+ let jsonBuffer;
+ try {
+ jsonBuffer = Buffer.from(await elResp.arrayBuffer());
+ } catch (readErr) {
+ console.error('[tts/eleven] with-timestamps read failed', readErr);
+ await refundMetering();
+ return error(res, 502, 'upstream_error', 'ElevenLabs with-timestamps response unreadable');
+ }
+ res.setHeader('content-length', String(jsonBuffer.length));
+ res.end(jsonBuffer);
+ cacheAudio(cacheKey, jsonBuffer, responseContentType);
+ return;
+ }
+
if (!elResp.body) {
// No readable stream (unexpected) — fall back to a single buffered write.
const audioBuffer = Buffer.from(await elResp.arrayBuffer());
res.setHeader('content-length', String(audioBuffer.length));
res.end(audioBuffer);
- cacheAudio(cacheKey, audioBuffer);
+ cacheAudio(cacheKey, audioBuffer, responseContentType);
return;
}
@@ -323,5 +350,5 @@ export default wrap(async (req, res) => {
if (!res.writableEnded && !res.destroyed) res.end();
// Cache only a complete clip — a truncated one would poison the deterministic
// cache key and be served as a permanent "hit" for this utterance.
- if (completed) cacheAudio(cacheKey, Buffer.concat(chunks));
+ if (completed) cacheAudio(cacheKey, Buffer.concat(chunks), responseContentType);
});
diff --git a/public/agent-3d-companion-harness.html b/public/agent-3d-companion-harness.html
new file mode 100644
index 000000000..b6a13a05c
--- /dev/null
+++ b/public/agent-3d-companion-harness.html
@@ -0,0 +1,22 @@
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>agent-3d companion presence test harness</title>
+<!--
+ Playwright-only fixture for tests/e2e/agent-3d-companion-presence.spec.js.
+ Imports /src/element.js directly so the spec exercises today's source instead
+ of the published https://three.ws/agent-3d/latest/agent-3d.js bundle that
+ every real page loads. Mirrors public/agent-3d-ar-harness.html's convention.
+ Loads a real production avatar asset (public/avatars/default.glb) already
+ checked into the repo, so mood/gesture/viseme morph targets and the
+ humanoid skeleton the baked clip library retargets onto are real, not
+ synthetic geometry.
+-->
+<style>html,body{margin:0;height:100%}agent-3d{display:block;width:480px;height:480px}</style>
+</head>
+<body>
+<agent-3d id="a3d" eager src="/avatars/default.glb"></agent-3d>
+<script type="module" src="/src/element.js"></script>
+</body>
+</html>
diff --git a/src/agent-protocol.js b/src/agent-protocol.js
index 0be23d860..8ee14d42d 100644
--- a/src/agent-protocol.js
+++ b/src/agent-protocol.js
@@ -30,6 +30,8 @@ export const ACTION_TYPES = {
PRESENCE: 'presence', // agent came online / went idle
PROTOCOL_ERROR: 'protocol-error', // emitted by the rate limiter on cascade detection
INTERRUPTED: 'interrupted', // user spoke while agent was mid-TTS
+ LISTEN_START: 'listen-start', // user started speaking (VAD/STT signal)
+ LISTEN_END: 'listen-end', // user stopped speaking
PERMISSIONS_REDEEM_START: 'permissions.redeem.start',
PERMISSIONS_REDEEM_SUCCESS: 'permissions.redeem.success',
PERMISSIONS_REDEEM_ERROR: 'permissions.redeem.error',
diff --git a/src/element.js b/src/element.js
index 2858e7be7..50e0f48cd 100644
--- a/src/element.js
+++ b/src/element.js
@@ -2194,6 +2194,7 @@ class Agent3DElement extends HTMLElement {
'voice:speech-end',
'voice:transcript',
'voice:listen-start',
+ 'voice:listen-end',
'voice:stt-status',
'memory:write',
]) {
@@ -2226,6 +2227,16 @@ class Agent3DElement extends HTMLElement {
}
if (this._chatEl) this._renderMessage(detail);
}
+ if (ev === 'voice:listen-start') {
+ // The mic opened for this turn (generic STT loop in runtime/index.js).
+ // Route through the protocol bus so AgentAvatar's _onListenStart
+ // drives the same attentive gaze + backchannel nod cadence the
+ // LiveKit realtime voice path already triggers from its own VAD.
+ protocol.emit({ type: ACTION_TYPES.LISTEN_START, payload: {} });
+ }
+ if (ev === 'voice:listen-end') {
+ protocol.emit({ type: ACTION_TYPES.LISTEN_END, payload: {} });
+ }
if (ev === 'brain:stream') {
if (this._thoughtBubbleEl && this.getAttribute('avatar-chat') !== 'off') {
this._streamToBubble(e.detail?.chunk ?? '');
diff --git a/src/runtime/index.js b/src/runtime/index.js
index 17ecddc8c..707140828 100644
--- a/src/runtime/index.js
+++ b/src/runtime/index.js
@@ -303,27 +303,33 @@ export class Runtime extends EventTarget {
async listen({ onInterim, onFinal, onStatus } = {}) {
if (!this.stt) throw new Error('STT not configured');
this.dispatchEvent(new CustomEvent('voice:listen-start', {}));
- const text = await this.stt.listen({
- // Recognizers that do real work before text exists (an on-device model
- // downloading, an utterance decoding) report it, so the UI can say so.
- onStatus: (status) => {
- this.dispatchEvent(new CustomEvent('voice:stt-status', { detail: status }));
- onStatus?.(status);
- },
- onInterim: (t) => {
- this.dispatchEvent(
- new CustomEvent('voice:transcript', { detail: { text: t, final: false } }),
- );
- onInterim?.(t);
- },
- onFinal: (t) => {
- this.dispatchEvent(
- new CustomEvent('voice:transcript', { detail: { text: t, final: true } }),
- );
- onFinal?.(t);
- },
- });
- return text;
+ try {
+ const text = await this.stt.listen({
+ // Recognizers that do real work before text exists (an on-device model
+ // downloading, an utterance decoding) report it, so the UI can say so.
+ onStatus: (status) => {
+ this.dispatchEvent(new CustomEvent('voice:stt-status', { detail: status }));
+ onStatus?.(status);
+ },
+ onInterim: (t) => {
+ this.dispatchEvent(
+ new CustomEvent('voice:transcript', { detail: { text: t, final: false } }),
+ );
+ onInterim?.(t);
+ },
+ onFinal: (t) => {
+ this.dispatchEvent(
+ new CustomEvent('voice:transcript', { detail: { text: t, final: true } }),
+ );
+ onFinal?.(t);
+ },
+ });
+ return text;
+ } finally {
+ // Always fires, success, error or cancellation, so a caller driving the
+ // avatar's attentive posture off listen-start never gets stuck in it.
+ this.dispatchEvent(new CustomEvent('voice:listen-end', {}));
+ }
}
clearConversation() {
diff --git a/src/runtime/lipsync.js b/src/runtime/lipsync.js
index 25951082a..a012d4022 100644
--- a/src/runtime/lipsync.js
+++ b/src/runtime/lipsync.js
@@ -178,6 +178,55 @@ export function playVisemeSequence(root, sequence, elapsedMs = () => 0) {
};
}
+// Characters that carry no viseme of their own: true silence in the sequence,
+// not just "unmapped". Mirrors the word/punctuation gaps _tokenize() inserts.
+const SILENT_CHARS = new Set([' ', '\t', '\n', '.', ',', '!', '?', ';', ':', '"', "'", '-', ' ']);
+
+// Any letter CHAR_TO_VISEME doesn't cover (most non-Latin scripts: CJK, Cyrillic,
+// Arabic, Devanagari…) still gets a real per-character timestamp from ElevenLabs;
+// it just can't pick a specific phoneme shape from a single codepoint. A generic
+// open-mouth viseme keeps the timing real rather than falling silent on every
+// language the heuristic alphabet-based lookup wasn't written for.
+const DEFAULT_LETTER_VISEME = 'viseme_aa';
+
+/**
+ * Build a timed viseme sequence from ElevenLabs' character-level alignment
+ * (`/with-timestamps`): `{ characters, character_start_times_seconds,
+ * character_end_times_seconds }`. Real per-character timing in whatever
+ * language the text was written in, unlike {@link startLipsync}'s English-letter
+ * duration estimate. Used by the ElevenLabs TTS lane (runtime/speech.js) with
+ * the energy-based {@link LipSyncAnalyser} path as its fallback.
+ *
+ * @param {{characters?: string[], character_start_times_seconds?: number[], character_end_times_seconds?: number[]}} alignment
+ * @returns {Array<{viseme:string,startMs:number,endMs:number}>}
+ */
+export function alignmentToSequence(alignment) {
+ const chars = alignment?.characters;
+ const starts = alignment?.character_start_times_seconds;
+ const ends = alignment?.character_end_times_seconds;
+ if (!Array.isArray(chars) || !Array.isArray(starts) || !Array.isArray(ends)) return [];
+
+ const sequence = [];
+ for (let i = 0; i < chars.length; i++) {
+ const raw = chars[i];
+ if (!raw) continue;
+ const ch = raw.toLowerCase();
+ if (SILENT_CHARS.has(ch)) continue;
+
+ const viseme = CHAR_TO_VISEME[ch] || (/\S/.test(ch) ? DEFAULT_LETTER_VISEME : null);
+ if (!viseme) continue;
+
+ const startMs = (starts[i] ?? 0) * 1000;
+ const endRaw = ends[i] ?? starts[i + 1] ?? starts[i] ?? 0;
+ // A zero-length entry (two characters sharing one timestamp) would never
+ // activate in activeVisemeAt()'s half-open interval check, so floor it to
+ // a visible minimum instead of letting the viseme silently vanish.
+ const endMs = Math.max(startMs + 20, endRaw * 1000);
+ sequence.push({ viseme, startMs, endMs });
+ }
+ return sequence;
+}
+
/**
* Drive jaw/mouth morph targets in sync with speech text via a phoneme heuristic.
* Supports the ARKit viseme_* morph targets that three.ws avatars ship with (and
diff --git a/src/runtime/livekit-voice.js b/src/runtime/livekit-voice.js
index 693fa14f5..7085b3b54 100644
--- a/src/runtime/livekit-voice.js
+++ b/src/runtime/livekit-voice.js
@@ -3,6 +3,7 @@
// data-channel messages with transcript payloads.
import { Room, RoomEvent, Track } from 'livekit-client';
+import { ACTION_TYPES } from '../agent-protocol.js';
export class LiveKitVoice {
/**
@@ -66,12 +67,12 @@ export class LiveKitVoice {
// User speech boundaries (VAD from agent server)
if (msg.type === 'speech_started') {
this._userSpeaking = true;
- this._protocol.emit({ type: 'listen-start', payload: {} });
+ this._protocol.emit({ type: ACTION_TYPES.LISTEN_START, payload: {} });
}
if (msg.type === 'speech_ended') {
this._userSpeaking = false;
this._protocol.emit({
- type: 'listen-end',
+ type: ACTION_TYPES.LISTEN_END,
payload: { text: msg.transcript || '' },
});
}
@@ -83,10 +84,10 @@ export class LiveKitVoice {
const active = speakers.some((p) => p.identity === localId);
if (active && !this._userSpeaking) {
this._userSpeaking = true;
- this._protocol.emit({ type: 'listen-start', payload: {} });
+ this._protocol.emit({ type: ACTION_TYPES.LISTEN_START, payload: {} });
} else if (!active && this._userSpeaking) {
this._userSpeaking = false;
- this._protocol.emit({ type: 'listen-end', payload: { text: '' } });
+ this._protocol.emit({ type: ACTION_TYPES.LISTEN_END, payload: { text: '' } });
}
});
diff --git a/src/runtime/speech.js b/src/runtime/speech.js
index 4ffd23c4b..3ba6190d8 100644
--- a/src/runtime/speech.js
+++ b/src/runtime/speech.js
@@ -5,8 +5,9 @@ export { NeuralTTS } from './neural-tts.js';
export { WhisperSTT } from './whisper-stt.js';
import { NeuralTTS } from './neural-tts.js';
import { WhisperSTT, onDeviceSupported } from './whisper-stt.js';
-import { startLipsync } from './lipsync.js';
+import { startLipsync, alignmentToSequence, playVisemeSequence } from './lipsync.js';
import { ACTION_TYPES } from '../agent-protocol.js';
+import { log } from '../shared/log.js';
export class BrowserTTS {
constructor({ voiceId = 'default', rate = 1, pitch = 1, lang = 'en-US' } = {}) {
@@ -235,10 +236,140 @@ export class ElevenLabsTTS {
this._positionalAudio = pa;
}
+ /**
+ * Speak `text` aloud. Tries real per-character viseme timing first
+ * ({@link _speakWithTimestamps}, ElevenLabs' `/with-timestamps`) whenever
+ * there's an avatar `scene` to animate, since that's the only reason to pay
+ * for the slower non-streaming round trip. Any failure there (network,
+ * malformed response, playback error before a frame plays) falls back to
+ * the original low-latency streaming path with energy-based lipsync
+ * ({@link _speakStreaming}), so a transient ElevenLabs hiccup never silences
+ * the agent.
+ */
async speak(text, { onStart, onEnd, scene } = {}) {
this.cancel();
if (!text) return;
+ if (scene) {
+ try {
+ return await this._speakWithTimestamps(text, { onStart, onEnd, scene });
+ } catch (err) {
+ if (err?.name === 'AbortError') return;
+ log.warn(
+ '[ElevenLabsTTS] with-timestamps failed, falling back to energy-based lipsync:',
+ err?.message || err,
+ );
+ this.cancel();
+ }
+ }
+ return this._speakStreaming(text, { onStart, onEnd, scene });
+ }
+
+ /**
+ * Real multi-language viseme timing: ElevenLabs' `/with-timestamps` endpoint
+ * returns the full clip as `audio_base64` plus character-level `alignment`
+ * (start/end seconds per character, in whatever script the text used). No
+ * MediaSource streaming: the response is one JSON object, not chunked MP3,
+ * so first-audio latency is higher than {@link _speakStreaming}; the real
+ * timing is the point.
+ */
+ async _speakWithTimestamps(text, { onStart, onEnd, scene }) {
+ this._abort = new AbortController();
+ this._speaking = true;
+
+ const url =
+ this.proxyURL ||
+ `https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(this.voiceId)}/with-timestamps`;
+ const headers = { 'content-type': 'application/json', accept: 'application/json' };
+ if (!this.proxyURL && this.apiKey) headers['xi-api-key'] = this.apiKey;
+
+ const rateNorm = Math.min(1.5, Math.max(0.5, this.rate));
+ const styleVal =
+ this.style != null
+ ? Math.min(1, Math.max(0, Number(this.style)))
+ : Math.min(1, Math.max(0, (rateNorm - 0.5) / 1.0));
+
+ const bodyObj = {
+ text,
+ model_id: this.modelId,
+ voice_settings: {
+ stability: this.stability,
+ similarity_boost: this.similarityBoost,
+ style: styleVal,
+ use_speaker_boost: this.useSpeakerBoost,
+ },
+ voiceId: this.voiceId,
+ modelId: this.modelId,
+ ...(this.agentId ? { agentId: this.agentId } : {}),
+ };
+ // Only the proxy understands this flag; the direct ElevenLabs API reads
+ // the /with-timestamps URL path instead.
+ if (this.proxyURL) bodyObj.withTimestamps = true;
+
+ const resp = await fetch(url, {
+ method: 'POST',
+ headers,
+ body: JSON.stringify(bodyObj),
+ signal: this._abort.signal,
+ });
+ if (!resp.ok) throw new Error(`ElevenLabs with-timestamps HTTP ${resp.status}`);
+ const data = await resp.json();
+ if (!data?.audio_base64) throw new Error('ElevenLabs with-timestamps returned no audio');
+
+ const binary = atob(data.audio_base64);
+ const bytes = new Uint8Array(binary.length);
+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+ const blob = new Blob([bytes], { type: 'audio/mpeg' });
+
+ const sequence = alignmentToSequence(data.alignment || data.normalized_alignment);
+
+ const audioURL = URL.createObjectURL(blob);
+ this._mediaSourceURL = audioURL;
+ const audio = new Audio(audioURL);
+ this._audio = audio;
+ // No analyser hookup here: a stale analyser from an earlier energy-path
+ // speak() on this same instance would otherwise leak into AgentAvatar's
+ // connectLipSync() and fight this sequence's direct morph writes for the
+ // same viseme targets every frame.
+ this._analyserNode = null;
+ this._connectPositionalAudio(audio);
+
+ return new Promise((resolve, reject) => {
+ this._onEnd = () => {
+ this._speaking = false;
+ this._lipsync?.stop();
+ this._lipsync = null;
+ this._cleanupAudio();
+ this.onEnd?.();
+ onEnd?.();
+ this._onEnd = null;
+ resolve();
+ };
+ audio.addEventListener(
+ 'playing',
+ () => {
+ if (sequence.length) {
+ this._lipsync = playVisemeSequence(scene, sequence, () => audio.currentTime * 1000);
+ }
+ this.onStart?.();
+ onStart?.();
+ },
+ { once: true },
+ );
+ audio.addEventListener('ended', () => this._onEnd?.(), { once: true });
+ audio.addEventListener(
+ 'error',
+ () => {
+ if (this._onEnd) this._onEnd();
+ },
+ { once: true },
+ );
+ audio.play().catch(reject);
+ });
+ }
+
+ /** Original streaming path: low first-audio latency, energy-based lipsync via LipSyncAnalyser. */
+ async _speakStreaming(text, { onStart, onEnd, scene }) {
this._abort = new AbortController();
this._speaking = true;
@@ -363,6 +494,12 @@ export class ElevenLabsTTS {
try {
if (!this._audioCtx || this._audioCtx.state === 'closed') {
this._audioCtx = new AC();
+ }
+ // Recreated independently of the context: a prior _speakWithTimestamps()
+ // call on this same instance nulls _analyserNode (see that method) while
+ // leaving _audioCtx open, so this must not assume "context exists" implies
+ // "analyser exists".
+ if (!this._analyserNode) {
this._analyserNode = this._audioCtx.createAnalyser();
this._analyserNode.fftSize = 256;
this._analyserNode.smoothingTimeConstant = 0.7;
diff --git a/src/runtime/tools.js b/src/runtime/tools.js
index 48ca72a4d..54f8927c5 100644
--- a/src/runtime/tools.js
+++ b/src/runtime/tools.js
@@ -2,6 +2,13 @@
// See specs/AGENT_MANIFEST.md § Built-in tools.
import { log } from '../shared/log.js';
+import { SLOTS } from './animation-slots.js';
+
+// Gestures are the subset of the slot vocabulary (animation-slots.js) an agent
+// can request by name as a deliberate body-language choice. 'idle' is the rest
+// state, not a performable gesture, so it is excluded from the tool's enum.
+const GESTURE_NAMES = SLOTS.filter((s) => s !== 'idle');
+
export const BUILTIN_TOOLS = [
{
name: 'wave',
@@ -41,6 +48,23 @@ export const BUILTIN_TOOLS = [
required: ['name'],
},
},
+ {
+ name: 'gesture',
+ description:
+ 'Perform a body-language gesture. Use this to react physically while speaking or ' +
+ 'listening: nod to agree or acknowledge, shake the head to disagree, wave to greet or ' +
+ 'say goodbye, shrug when unsure, bow to thank or apologize, point to direct attention, ' +
+ 'celebrate good news, show concern for bad news, think while working something out, ' +
+ 'inspect something closely, present a result, or signal patience, curiosity, or fidgety ' +
+ 'restlessness. The gesture plays once and the body returns to its resting pose.',
+ input_schema: {
+ type: 'object',
+ properties: {
+ name: { type: 'string', enum: GESTURE_NAMES, description: 'Which gesture to perform.' },
+ },
+ required: ['name'],
+ },
+ },
{
name: 'setExpression',
description: 'Set facial expression via morph target preset.',
@@ -155,6 +179,17 @@ export const BUILTIN_HANDLERS = {
return { ok: true, name, resolved };
},
+ async gesture(args, ctx) {
+ const { name } = args;
+ const resolved = ctx.viewer.resolveAnimationSlot?.(name) ?? name;
+ const played = ctx.viewer.playClipByName?.(resolved, { loop: false, fade_ms: 200 });
+ if (!played) {
+ log.warn(`[gesture] slot "${name}" → "${resolved}" not found in library`);
+ return { ok: false, error: `No clip for gesture "${name}"` };
+ }
+ return { ok: true, name, resolved };
+ },
+
async setExpression(args, ctx) {
const { preset, intensity = 1 } = args;
ctx.viewer.setExpression?.(preset, intensity);
diff --git a/tests/agent-avatar-active-listening.test.js b/tests/agent-avatar-active-listening.test.js
new file mode 100644
index 000000000..a4a1634cd
--- /dev/null
+++ b/tests/agent-avatar-active-listening.test.js
@@ -0,0 +1,127 @@
+/**
+ * AgentAvatar, active-listening idle reactions (Stage 2c of `_tickEmotion()`).
+ *
+ * `_onListenStart`/`_onListenEnd` are the handlers the protocol bus invokes
+ * when a real VAD/STT signal fires (never a fabricated loading timer); see
+ * `_sub(ACTION_TYPES.LISTEN_START, ...)` in `attach()`. Called directly here,
+ * mirroring the direct-`_tickEmotion()` approach in
+ * tests/agent-avatar-mood-gesture.test.js, so no protocol bus or DOM is needed.
+ */
+
+import { describe, it, expect, beforeEach } from 'vitest';
+
+let AgentAvatar;
+
+function makeRoot() {
+ return {
+ isObject3D: true,
+ children: [],
+ traverse(fn) {
+ fn(this);
+ },
+ };
+}
+
+function makeFakeAnimationManager(loadedNames = []) {
+ const loaded = new Set(loadedNames);
+ const played = [];
+ return {
+ currentName: null,
+ isLoaded: (name) => loaded.has(name),
+ getAnimationDefs: () => [],
+ play: (name) => {
+ played.push(name);
+ return Promise.resolve(true);
+ },
+ crossfadeTo: (name) => {
+ played.push(name);
+ },
+ played,
+ };
+}
+
+function makeFakeViewer(am) {
+ return {
+ content: makeRoot(),
+ animationManager: am,
+ state: {},
+ };
+}
+
+function makeAvatar(am) {
+ const viewer = makeFakeViewer(am);
+ const protocol = {}; // _onListenStart/_onListenEnd are called directly, bus unused
+ const identity = { id: 'test' };
+ return new AgentAvatar(viewer, protocol, identity);
+}
+
+beforeEach(async () => {
+ if (!AgentAvatar) {
+ const mod = await import('../src/agent-avatar.js');
+ AgentAvatar = mod.AgentAvatar;
+ }
+});
+
+describe('AgentAvatar, active-listening wiring', () => {
+ it('_onListenStart sets the attentive-gaze flag the head-transform priority chain reads', () => {
+ const avatar = makeAvatar(makeFakeAnimationManager(['nod']));
+ expect(avatar._userSpeaking).toBe(false);
+ avatar._onListenStart();
+ expect(avatar._userSpeaking).toBe(true);
+ });
+
+ it('_onListenEnd clears the attentive-gaze flag', () => {
+ const avatar = makeAvatar(makeFakeAnimationManager(['nod']));
+ avatar._onListenStart();
+ avatar._onListenEnd();
+ expect(avatar._userSpeaking).toBe(false);
+ });
+
+ it('nods on a fixed cadence while the user speaks, driven only by setUserSpeaking (no timer of its own)', () => {
+ const am = makeFakeAnimationManager(['nod']);
+ const avatar = makeAvatar(am);
+
+ avatar._onListenStart();
+ // LISTENING_NOD_INTERVAL_S is 2.8s: tick just under it, nothing yet.
+ avatar._tickEmotion(2.7);
+ expect(am.played).not.toContain('nod');
+
+ // Cross the threshold: one nod fires.
+ avatar._tickEmotion(0.2);
+ expect(am.played.filter((n) => n === 'nod')).toHaveLength(1);
+ });
+
+ it('stops nodding immediately once the user stops speaking', () => {
+ const am = makeFakeAnimationManager(['nod']);
+ const avatar = makeAvatar(am);
+
+ avatar._onListenStart();
+ avatar._tickEmotion(2.9); // one nod fires
+ expect(am.played.filter((n) => n === 'nod')).toHaveLength(1);
+
+ avatar._onListenEnd();
+ am.played.length = 0;
+ avatar._tickEmotion(10); // would have nodded 3x more if still "listening"
+ expect(am.played).not.toContain('nod');
+ });
+
+ it('never nods when the user never speaks (idle avatar, no real signal)', () => {
+ const am = makeFakeAnimationManager(['nod']);
+ const avatar = makeAvatar(am);
+
+ for (let i = 0; i < 20; i++) avatar._tickEmotion(1);
+ expect(am.played).not.toContain('nod');
+ });
+
+ it('defers the backchannel nod while a higher-priority one-shot gesture is already playing', () => {
+ const am = makeFakeAnimationManager(['nod', 'wave']);
+ const avatar = makeAvatar(am);
+
+ avatar._onListenStart();
+ avatar._playSlot('wave', 5.0); // long one-shot in flight, sets _isPlayingOneShot
+ am.played.length = 0;
+
+ avatar._tickEmotion(3.0); // would cross the nod threshold if unblocked
+ expect(am.played).not.toContain('nod');
+ });
+});
diff --git a/tests/alignment-to-sequence.test.js b/tests/alignment-to-sequence.test.js
new file mode 100644
index 000000000..6580f6b03
--- /dev/null
+++ b/tests/alignment-to-sequence.test.js
@@ -0,0 +1,71 @@
+// Tests for alignmentToSequence(): turns ElevenLabs' /with-timestamps
+// character-level alignment into the {viseme,startMs,endMs} shape
+// playVisemeSequence() expects. Pure function, no AudioContext/DOM needed.
+
+import { describe, it, expect } from 'vitest';
+import { alignmentToSequence, activeVisemeAt } from '../src/runtime/lipsync.js';
+
+function alignment(chars, startSec, durationSec = 0.08) {
+ const starts = [];
+ const ends = [];
+ let t = startSec;
+ for (let i = 0; i < chars.length; i++) {
+ starts.push(t);
+ ends.push(t + durationSec);
+ t += durationSec;
+ }
+ return { characters: chars.split(''), character_start_times_seconds: starts, character_end_times_seconds: ends };
+}
+
+describe('alignmentToSequence', () => {
+ it('maps known Latin letters to their viseme with real millisecond timing', () => {
+ const seq = alignmentToSequence(alignment('ba', 0, 0.1));
+ expect(seq).toEqual([
+ { viseme: 'viseme_PP', startMs: 0, endMs: 100 },
+ { viseme: 'viseme_aa', startMs: 100, endMs: 200 },
+ ]);
+ });
+
+ it('skips whitespace and punctuation as true silence, not a viseme', () => {
+ const seq = alignmentToSequence(alignment('a, b.', 0, 0.1));
+ const visemes = seq.map((s) => s.viseme);
+ expect(visemes).toEqual(['viseme_aa', 'viseme_PP']);
+ });
+
+ it('falls back to a generic open-mouth viseme for non-Latin scripts', () => {
+ // CJK/Cyrillic/Arabic characters have no entry in CHAR_TO_VISEME, but still
+ // carry a real per-character timestamp from ElevenLabs, so the timing must
+ // not be dropped just because the exact mouth shape is unknown.
+ const seq = alignmentToSequence(alignment('日本', 0, 0.12));
+ expect(seq).toEqual([
+ { viseme: 'viseme_aa', startMs: 0, endMs: 120 },
+ { viseme: 'viseme_aa', startMs: 120, endMs: 240 },
+ ]);
+ });
+
+ it('floors zero-length entries to a visible minimum duration', () => {
+ // Two characters sharing one timestamp (end == start) would never satisfy
+ // activeVisemeAt()'s half-open interval check if left at 0 width.
+ const alignmentData = {
+ characters: ['a', 'b'],
+ character_start_times_seconds: [0, 0.05],
+ character_end_times_seconds: [0.05, 0.05],
+ };
+ const seq = alignmentToSequence(alignmentData);
+ expect(seq[1].endMs).toBeGreaterThan(seq[1].startMs);
+ expect(seq[1].endMs - seq[1].startMs).toBeGreaterThanOrEqual(20);
+ });
+
+ it('produces a sequence activeVisemeAt() can actually scan', () => {
+ const seq = alignmentToSequence(alignment('hi', 0, 0.1));
+ expect(activeVisemeAt(seq, 50)).toBe(seq[0].viseme);
+ expect(activeVisemeAt(seq, 150)).toBe(seq[1].viseme);
+ expect(activeVisemeAt(seq, 1000)).toBeNull();
+ });
+
+ it('returns an empty array when alignment data is missing or malformed', () => {
+ expect(alignmentToSequence(undefined)).toEqual([]);
+ expect(alignmentToSequence({})).toEqual([]);
+ expect(alignmentToSequence({ characters: ['a'] })).toEqual([]);
+ });
+});
diff --git a/tests/e2e/agent-3d-companion-presence.spec.js b/tests/e2e/agent-3d-companion-presence.spec.js
new file mode 100644
index 000000000..ef0ce8f1c
--- /dev/null
+++ b/tests/e2e/agent-3d-companion-presence.spec.js
@@ -0,0 +1,132 @@
+import { test, expect } from '@playwright/test';
+
+// Work order 082: companion presence (mood, emoji-driven expression, the
+// LLM-composable gesture tool, and active-listening idle reactions). Drives
+// a real Chromium against the dev server's /src/element.js (not the
+// published CDN bundle) using public/agent-3d-companion-harness.html, which
+// loads a real production avatar GLB (public/avatars/default.glb) already
+// checked into the repo. No GLB bytes are route-intercepted and no app
+// logic is mocked: every assertion reads the real mesh's
+// `morphTargetInfluences`, the real `AnimationManager`'s loaded clip, or the
+// real `AgentAvatar` emotion state that drives them.
+
+function watchErrors(page) {
+ const errors = [];
+ page.on('pageerror', (err) => errors.push(String(err?.message || err)));
+ return errors;
+}
+
+async function ready(page) {
+ await page.evaluate(() => document.getElementById('a3d')._waitForReady());
+ // AgentAvatar resolves the GLB's morph targets synchronously at
+ // construction time, but construction itself is queued behind model
+ // load; wait for a non-empty resolution before reading morph weights.
+ await expect
+ .poll(() => page.evaluate(() => document.getElementById('a3d')._avatar?._morphResolved?.size ?? 0), {
+ timeout: 60_000,
+ })
+ .toBeGreaterThan(0);
+}
+
+function mouthSmileWeight(page) {
+ return page.evaluate(() => {
+ const avatar = document.getElementById('a3d')._avatar;
+ const slots = avatar._morphResolved.get('mouthSmileLeft') || avatar._morphResolved.get('mouthSmile');
+ if (!slots?.length) return null;
+ return slots[0].mesh.morphTargetInfluences[slots[0].index];
+ });
+}
+
+function mouthFrownWeight(page) {
+ return page.evaluate(() => {
+ const avatar = document.getElementById('a3d')._avatar;
+ const slots = avatar._morphResolved.get('mouthFrownLeft') || avatar._morphResolved.get('mouthFrown');
+ if (!slots?.length) return null;
+ return slots[0].mesh.morphTargetInfluences[slots[0].index];
+ });
+}
+
+test.describe('agent-3d companion presence (mood, emoji, gesture, active listening)', () => {
+ test('setMood drives a real smile onto the loaded avatar mesh', async ({ page }) => {
+ const errors = watchErrors(page);
+ await page.goto('/agent-3d-companion-harness.html', { waitUntil: 'domcontentloaded' });
+ await ready(page);
+
+ // The idle rig carries its own small baseline smile, so assert a real
+ // increase from whatever baseline this GLB renders at rather than an
+ // absolute near-zero floor.
+ const baseline = await mouthSmileWeight(page);
+ await page.evaluate(() => document.getElementById('a3d').setMood(0.9, 0.6));
+ // The mood layer lerps in over ~1s (k = dt*1.2); give it real wall-clock time.
+ await expect.poll(() => mouthSmileWeight(page), { timeout: 5_000 }).toBeGreaterThan(baseline + 0.2);
+
+ expect(errors, `page errors: ${errors.join(' | ')}`).toEqual([]);
+ });
+
+ test('an emoji-only reply with no English sentiment words still smiles the avatar', async ({ page }) => {
+ const errors = watchErrors(page);
+ await page.goto('/agent-3d-companion-harness.html', { waitUntil: 'domcontentloaded' });
+ await ready(page);
+
+ const baseline = await mouthSmileWeight(page);
+ await page.evaluate(() =>
+ document.getElementById('a3d')._avatar._onSpeak({ payload: { text: 'here you go 🎉' } }),
+ );
+ await expect.poll(() => mouthSmileWeight(page), { timeout: 5_000 }).toBeGreaterThan(baseline + 0.1);
+
+ expect(errors, `page errors: ${errors.join(' | ')}`).toEqual([]);
+ });
+
+ test('a sad emoji frowns the avatar instead', async ({ page }) => {
+ const errors = watchErrors(page);
+ await page.goto('/agent-3d-companion-harness.html', { waitUntil: 'domcontentloaded' });
+ await ready(page);
+
+ const baseline = await mouthFrownWeight(page);
+ await page.evaluate(() =>
+ document.getElementById('a3d')._avatar._onSpeak({ payload: { text: 'that did not work out 😢' } }),
+ );
+ await expect.poll(() => mouthFrownWeight(page), { timeout: 5_000 }).toBeGreaterThan(baseline + 0.1);
+
+ expect(errors, `page errors: ${errors.join(' | ')}`).toEqual([]);
+ });
+
+ test('the gesture tool plays a real retargeted baked clip by slot name', async ({ page }) => {
+ const errors = watchErrors(page);
+ await page.goto('/agent-3d-companion-harness.html', { waitUntil: 'domcontentloaded' });
+ await ready(page);
+
+ const resolved = await page.evaluate(() => document.getElementById('a3d')._scene.resolveAnimationSlot('wave'));
+ expect(resolved).toBe('wave');
+
+ await page.evaluate(() => document.getElementById('a3d')._scene.playClipByName('wave', { loop: false }));
+ await expect
+ .poll(() => page.evaluate(() => document.getElementById('a3d')._viewer?.animationManager?.currentName), {
+ timeout: 15_000,
+ })
+ .toBe('wave');
+
+ expect(errors, `page errors: ${errors.join(' | ')}`).toEqual([]);
+ });
+
+ test('the avatar nods on a fixed cadence while the user speaks, and stops when they stop', async ({ page }) => {
+ const errors = watchErrors(page);
+ await page.goto('/agent-3d-companion-harness.html', { waitUntil: 'domcontentloaded' });
+ await ready(page);
+
+ await page.evaluate(() => document.getElementById('a3d')._avatar._onListenStart());
+ // LISTENING_NOD_INTERVAL_S is 2.8s, plus the AnimationManager's own
+ // lazy clip load over the dev server on first use; give both real time.
+ await expect
+ .poll(
+ () => page.evaluate(() => document.getElementById('a3d')._viewer?.animationManager?.currentName),
+ { timeout: 20_000 },
+ )
+ .toBe('nod');
+
+ await page.evaluate(() => document.getElementById('a3d')._avatar._onListenEnd());
+ expect(await page.evaluate(() => document.getElementById('a3d')._avatar._userSpeaking)).toBe(false);
+
+ expect(errors, `page errors: ${errors.join(' | ')}`).toEqual([]);
+ });
+});
diff --git a/tests/src/runtime.test.js b/tests/src/runtime.test.js
index f2c3fbc0a..38324d96e 100644
--- a/tests/src/runtime.test.js
+++ b/tests/src/runtime.test.js
@@ -1,7 +1,7 @@
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { Runtime } from '../../src/runtime/index.js';
import { NullProvider } from '../../src/runtime/providers.js';
-import { BUILTIN_HANDLERS } from '../../src/runtime/tools.js';
+import { BUILTIN_HANDLERS, BUILTIN_TOOLS } from '../../src/runtime/tools.js';
// Disable voice in all Runtime instances — avoids BrowserTTS/BrowserSTT in tests
const NO_VOICE = { tts: { provider: 'none' }, stt: { provider: 'none' } };
@@ -358,6 +358,41 @@ describe('Runtime — busy guard', () => {
});
});
+// ── Runtime.listen() ──────────────────────────────────────────────────────────
+
+describe('Runtime, listen() voice:listen-start/listen-end symmetry', () => {
+ it('fires listen-start then listen-end around a successful transcript', async () => {
+ const rt = makeRuntime(mockProvider([textResp('ok')]));
+ rt.stt = { listen: vi.fn().mockResolvedValue('hello') };
+ const events = [];
+ rt.addEventListener('voice:listen-start', () => events.push('start'));
+ rt.addEventListener('voice:listen-end', () => events.push('end'));
+
+ const text = await rt.listen();
+ expect(text).toBe('hello');
+ expect(events).toEqual(['start', 'end']);
+ });
+
+ it('still fires listen-end when the STT call rejects (try/finally, not just the happy path)', async () => {
+ const rt = makeRuntime(mockProvider([textResp('ok')]));
+ rt.stt = { listen: vi.fn().mockRejectedValue(new Error('mic denied')) };
+ const events = [];
+ rt.addEventListener('voice:listen-start', () => events.push('start'));
+ rt.addEventListener('voice:listen-end', () => events.push('end'));
+
+ await expect(rt.listen()).rejects.toThrow('mic denied');
+ expect(events).toEqual(['start', 'end']);
+ });
+
+ it('throws without dispatching anything when no STT is configured', async () => {
+ const rt = makeRuntime(mockProvider([textResp('ok')]));
+ rt.stt = null;
+ const events = collect(rt, 'voice:listen-start');
+ await expect(rt.listen()).rejects.toThrow('STT not configured');
+ expect(events).toEqual([]);
+ });
+});
+
// ── BUILTIN_HANDLERS ──────────────────────────────────────────────────────────
describe('BUILTIN_HANDLERS.speak', () => {
@@ -420,3 +455,40 @@ describe('BUILTIN_HANDLERS.setExpression', () => {
expect(ctx.viewer.setExpression).toHaveBeenCalledWith('sad', 1);
});
});
+
+describe('gesture tool, LLM-composable body language', () => {
+ it('enumerates every animation slot except the rest state "idle"', () => {
+ const tool = BUILTIN_TOOLS.find((t) => t.name === 'gesture');
+ const names = tool.input_schema.properties.name.enum;
+ expect(names).toContain('nod');
+ expect(names).toContain('wave');
+ expect(names).not.toContain('idle');
+ });
+
+ it('resolves the slot name through resolveAnimationSlot before playing it', async () => {
+ const ctx = {
+ viewer: {
+ resolveAnimationSlot: vi.fn().mockReturnValue('Armature|Nod'),
+ playClipByName: vi.fn().mockReturnValue(true),
+ },
+ };
+ const result = await BUILTIN_HANDLERS.gesture({ name: 'nod' }, ctx);
+ expect(ctx.viewer.resolveAnimationSlot).toHaveBeenCalledWith('nod');
+ expect(ctx.viewer.playClipByName).toHaveBeenCalledWith('Armature|Nod', {
+ loop: false,
+ fade_ms: 200,
+ });
+ expect(result).toEqual({ ok: true, name: 'nod', resolved: 'Armature|Nod' });
+ });
+
+ it('reports failure instead of throwing when the resolved clip is missing from the library', async () => {
+ const ctx = {
+ viewer: {
+ resolveAnimationSlot: vi.fn().mockReturnValue('nod'),
+ playClipByName: vi.fn().mockReturnValue(false),
+ },
+ };
+ const result = await BUILTIN_HANDLERS.gesture({ name: 'nod' }, ctx);
+ expect(result).toEqual({ ok: false, error: 'No clip for gesture "nod"' });
+ });
+});