The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
import debug from 'debug';
import { useEffect, useRef, useState } from 'react';
import { useSelector } from 'react-redux';
import { type ChatSubagentDoneEvent, subscribeChatEvents } from '../../services/chatService';
import { selectEffectiveMascotVoiceId } from '../../store/mascotSlice';
import type { MascotFace } from './Mascot';
import { lerpViseme, VISEMES, type VisemeShape } from './Mascot/visemes';
import {
isAudioStopped,
type PlaybackHandle,
playBase64Audio,
swallowAudioStop,
} from './voice/audioPlayer';
import {
hasUsableStarts,
normalizeVisemeTimeline,
proceduralVisemes,
synthesizeSpeech,
type VisemeFrame,
} from './voice/ttsClient';
import { selectVisemeSource, splitIntoSentences } from './voice/ttsQueue';
import { findActiveFrame, oculusVisemeToShape } from './voice/visemeMap';
const mascotLog = debug('human:mascot');
/** ms the mouth holds the target viseme before decaying back to rest. */
const VISEME_DECAY_MS = 180;
/**
* Safety ceiling for a single TTS utterance. Runaway audio (network stall,
* decoder that never emits `ended`) is auto-stopped at this limit so the
* mascot never gets stuck in a permanent `speaking` pose.
* 5 minutes comfortably covers any real reply; exported for tests.
*/
export const TTS_MAX_PLAYBACK_MS = 5 * 60 * 1_000;
const TTS_ESTIMATED_MS_PER_CHAR = 55;
const TTS_MIN_ESTIMATED_PLAYBACK_MS = 600;
function estimateTtsPlaybackMs(text: string, frames: VisemeFrame[]): number {
const frameEnd = frames.at(-1)?.end_ms ?? 0;
if (frameEnd > 0 && hasUsableStarts(frames)) return frameEnd;
return Math.max(TTS_MIN_ESTIMATED_PLAYBACK_MS, text.trim().length * TTS_ESTIMATED_MS_PER_CHAR);
}
/**
* How long to hold a transient acknowledgement face (`happy`, `concerned`)
* before decaying back to `idle`. Tuned to feel like a soft beat rather than
* a snap. Exported for tests.
*/
export const ACK_FACE_HOLD_MS = 700;
/**
* Pick a viseme from the trailing letter of a text delta. Heuristic — we
* have no phoneme data — but it gives the mouth varied motion that tracks
* the streaming text instead of just opening and closing the same way.
*/
export function pickViseme(delta: string): VisemeShape {
const ch = delta
.replace(/[^a-zA-Z]/g, '')
.slice(-1)
.toLowerCase();
switch (ch) {
case 'a':
return VISEMES.A;
case 'e':
return VISEMES.E;
case 'i':
case 'y':
return VISEMES.I;
case 'o':
return VISEMES.O;
case 'u':
case 'w':
return VISEMES.U;
case 'm':
case 'b':
case 'p':
return VISEMES.M;
case 'f':
case 'v':
return VISEMES.F;
default:
return VISEMES.E;
}
}
/**
* Pick a raw Oculus viseme code from a text delta character. Used to drive
* `mouthVisemeCode` on the Rive state machine during pseudo-lipsync (no TTS).
* Returns Oculus 15-set codes; `RiveMascot` normalises the close vowels
* (`I`→`ih`, `O`→`oh`, `U`→`ou`) to the asset's `visme_codes` vocabulary at
* render time, keeping `E`/`aa` and the consonants as-is.
*/
export function pickVisemeCode(delta: string): string {
const ch = delta
.replace(/[^a-zA-Z]/g, '')
.slice(-1)
.toLowerCase();
switch (ch) {
case 'a':
return 'aa';
case 'e':
return 'E';
case 'i':
case 'y':
return 'I';
case 'o':
return 'O';
case 'u':
case 'w':
return 'U';
case 'm':
case 'b':
case 'p':
return 'PP';
case 'f':
case 'v':
return 'FF';
case 's':
case 'z':
return 'SS';
case 'n':
case 'l':
return 'nn';
case 't':
case 'd':
return 'DD';
case 'k':
case 'g':
return 'kk';
case 'r':
return 'RR';
default:
return 'E';
}
}
type ConversationAckFace = Extract<
MascotFace,
| 'happy'
| 'confused'
| 'concerned'
| 'curious'
| 'proud'
| 'cautious'
| 'celebrating'
| 'dancing'
| 'waving'
>;
type ConversationAckEvent = { full_response?: string | null };
const CONCERNED_TEXT_RE =
/\b(sorry|apolog(?:y|ize|ise)|failed|failure|error|cannot|can't|unable|blocked|problem)\b/i;
const CONFUSED_TEXT_RE =
/\b(not sure|unclear|ambiguous|clarify|which one|need more|can you confirm|maybe)\b/i;
const HAPPY_TEXT_RE = /\b(done|completed|fixed|success|successful|ready|all set|great|nice)\b/i;
const PROUD_TEXT_RE =
/\b(successfully completed|all tasks? (done|finished)|mission accomplished|everything (works?|is working)|all (checks?|tests?) pass(ed)?)\b/i;
const CURIOUS_TEXT_RE =
/\b(interesting|fascinating|curious(ly)?|let me (check|look|investigate)|i('ll)? (look|check) into|turns? out to be)\b/i;
const CAUTIOUS_TEXT_RE =
/\b(be careful|warning|caution|heads? up|please note|make sure|important to note|note that|worth (noting|mentioning))\b/i;
const CELEBRATING_TEXT_RE =
/\b(congrat(ulations|s)?|well done|bravo|hooray|woohoo|amazing|fantastic|incredible|awesome work)\b/i;
const GREETING_TEXT_RE =
/^(hello|hey|hi there|good (morning|afternoon|evening)|welcome back|greetings|howdy)[!.,]?(?:\s|$)/i;
/**
* Map conversation-level meaning into the short acknowledgement face that
* follows a completed turn. Runtime activity still owns thinking/speaking
* states; this only decides the post-turn emotional beat.
*/
export function pickConversationAckFace(event: ConversationAckEvent): ConversationAckFace | null {
const text = event.full_response?.trim() ?? '';
if (!text) return null;
// Priority: concerned > cautious > proud > confused > curious > happy.
// Concerned and cautious share some vocabulary; check concerned first so
// outright failures don't get softened to a heads-up.
if (CONCERNED_TEXT_RE.test(text)) return 'concerned';
if (CAUTIOUS_TEXT_RE.test(text)) return 'cautious';
if (CELEBRATING_TEXT_RE.test(text)) return 'celebrating';
if (PROUD_TEXT_RE.test(text)) return 'proud';
if (CONFUSED_TEXT_RE.test(text)) return 'confused';
if (CURIOUS_TEXT_RE.test(text)) return 'curious';
if (GREETING_TEXT_RE.test(text)) return 'waving';
if (HAPPY_TEXT_RE.test(text)) return 'happy';
return null;
}
/**
* Map a tool name to an activity pose. Returns null when the tool doesn't
* have a strong visual association — the caller falls back to a generic face.
*/
function toolToActivityFace(toolName: string): MascotFace | null {
const name = toolName.toLowerCase();
if (
name.includes('file_write') ||
name.includes('edit_file') ||
name.includes('apply_patch') ||
name.includes('create_file') ||
name.includes('write')
) {
return 'writing';
}
if (
name.includes('browser') ||
name.includes('web_search') ||
name.includes('web_fetch') ||
name.includes('read_file') ||
name.includes('search') ||
name.includes('grep') ||
name.includes('find')
) {
return 'reading';
}
return null;
}
interface UseHumanMascotOptions {
/** When true, post-stream replies are sent to ElevenLabs and the mouth
* follows the returned viseme timeline while the audio plays. */
speakReplies?: boolean;
/** When true, force the mascot into a `listening` pose. Caller is responsible
* for setting this while the mic is hot (e.g. from voice dictation state). */
listening?: boolean;
}
interface UseHumanMascotResult {
face: MascotFace;
viseme: VisemeShape;
/** Raw Oculus 15-set viseme code for Rive's `mouthVisemeCode` input. */
visemeCode: string;
}
/** Result of a chunk's speech synthesis — `ok:false` on failure so the pump
* can skip a bad sentence without aborting the whole reply. */
type ChunkSynth = { ok: true; tts: Awaited<ReturnType<typeof synthesizeSpeech>> } | { ok: false };
/** One queued sentence: its spoken text plus the eagerly-fired synth promise. */
interface TtsChunk {
text: string;
synth: Promise<ChunkSynth>;
}
/**
* Drives the mascot's face/mouth from agent + voice lifecycle events.
*
* Mapping (kept in one place so the visual model stays coherent):
*
* - `inference_start` → `thinking`
* - `iteration_start` round > 1 or `tool_call` → activity pose based on tool
* name (writing/reading) or `confused` as fallback
* - `tool_result success=false` → `concerned` (held briefly)
* - `text_delta` → `speaking`, pseudo-lipsync from the trailing letter
* - `chat_done` (no TTS) → message-aware ack face (held briefly), then `idle`
* - `chat_done` (TTS enabled) → `thinking` while synthesizing → `speaking`
* with real visemes → message-aware ack face when the audio ends
* - `chat_error`, TTS failure → `concerned` (held briefly), then `idle`
* - `listening` option override → `listening` (highest priority)
*
* Errors and unavailable voice degrade cleanly: speech failures fall through
* to text-only behavior and surface as a brief `concerned` beat.
*/
export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMascotResult {
const { speakReplies = false, listening = false } = options;
const speakRef = useRef(speakReplies);
speakRef.current = speakReplies;
const listeningRef = useRef(listening);
listeningRef.current = listening;
const effectiveMascotVoiceId = useSelector(selectEffectiveMascotVoiceId);
const mascotVoiceIdRef = useRef<string>(effectiveMascotVoiceId);
mascotVoiceIdRef.current = effectiveMascotVoiceId;
const [face, setFace] = useState<MascotFace>('idle');
const targetRef = useRef<VisemeShape>(VISEMES.REST);
const visemeCodeRef = useRef<string>('sil');
const lastDeltaAtRef = useRef(0);
const ackTimerRef = useRef<number | null>(null);
const toolSucceededRef = useRef(false);
const subagentSucceededRef = useRef(false);
const playbackRef = useRef<PlaybackHandle | null>(null);
const visemeFramesRef = useRef<{ viseme: string; start_ms: number; end_ms: number }[]>([]);
const visemeCursorRef = useRef(0);
const playbackSeqRef = useRef(0);
// Wall-clock anchor (performance.now() at the instant playback became
// current) used to index the viseme timeline. We deliberately do NOT key off
// `audio.currentTime`: in the embedded CEF webview it can fail to advance for
// in-memory blob audio, which freezes the mouth on a single viseme even
// though the audio plays. A monotonic clock always advances, and because the
// viseme frames are rescaled to the measured audio duration it stays in sync.
const playbackStartedAtRef = useRef(0);
// Throttle marker for the lipsync diagnostic log (last logged ms).
const lastLipsyncLogRef = useRef(0);
// ── Streaming, sentence-chunked TTS state (#5358) ──────────────────────────
// The reply is spoken sentence-by-sentence as it streams instead of waiting
// for the whole response, so the first audio starts within a sentence's
// synth latency rather than after the entire reply + one big round trip.
//
// `turnSeqRef` mirrors the `playbackSeqRef` value captured when the current
// TTS turn began — every async synth/play continuation re-checks it so a
// superseded or cancelled turn drops its work instead of speaking over the
// next one. A chunk holds the raw sentence text plus the in-flight synthesis
// promise (fired eagerly at enqueue so sentence N+1 synthesizes while sentence
// N plays); the pump plays them strictly in enqueue order, single-flight, so
// only one `<audio>` element is ever live and there are no orphan blob URLs.
const streamActiveRef = useRef(false);
const turnSeqRef = useRef(0);
const sawDeltaThisTurnRef = useRef(false);
const sentenceBufferRef = useRef('');
const chunkQueueRef = useRef<TtsChunk[]>([]);
const queueClosedRef = useRef(false);
const queuePumpingRef = useRef(false);
const queueAckRef = useRef<ConversationAckFace>('happy');
const chunkAttemptedRef = useRef(false);
const chunkPlayedRef = useRef(false);
const [, force] = useState(0);
function clearAckTimer() {
if (ackTimerRef.current != null) {
window.clearTimeout(ackTimerRef.current);
ackTimerRef.current = null;
}
}
function holdThenIdle(ackFace: MascotFace, ms = ACK_FACE_HOLD_MS) {
clearAckTimer();
setFace(ackFace);
ackTimerRef.current = window.setTimeout(() => {
ackTimerRef.current = null;
setFace('idle');
}, ms);
}
useEffect(() => {
const unsub = subscribeChatEvents({
onInferenceStart: () => {
clearAckTimer();
toolSucceededRef.current = false;
subagentSucceededRef.current = false;
// A new agent turn supersedes any reply still being spoken — stop the
// previous turn's audio and drop its queue so the next turn's deltas
// start a fresh stream instead of appending onto stale state.
cancelTtsPlayback();
mascotLog('voice-session transition → thinking (inference_start)');
setFace('thinking');
},
onIterationStart: e => {
if (e.round > 1) {
clearAckTimer();
mascotLog('voice-session transition → drinking_coffee (iteration round=%d)', e.round);
setFace('drinking_coffee');
}
},
onToolCall: e => {
clearAckTimer();
const activityFace = toolToActivityFace(e.tool_name);
if (activityFace) {
mascotLog('voice-session transition → %s (tool_call %s)', activityFace, e.tool_name);
setFace(activityFace);
} else {
mascotLog('voice-session transition → thinking (tool_call %s)', e.tool_name);
setFace('thinking');
}
},
onToolResult: e => {
if (!e.success) {
mascotLog('voice-session transition → concerned (tool_result failed)');
setFace('concerned');
} else {
toolSucceededRef.current = true;
setFace('thinking');
}
},
onSubagentDone: (e: ChatSubagentDoneEvent) => {
if (e.success) {
mascotLog('voice-session subagent_done success tool=%s', e.tool_name);
subagentSucceededRef.current = true;
} else {
mascotLog(
'voice-session transition → concerned (subagent_done failed tool=%s)',
e.tool_name
);
setFace('concerned');
}
},
onTextDelta: e => {
if (listeningRef.current) {
mascotLog('voice-session text_delta suppressed — listening is active');
return;
}
// In speak mode we stream the reply sentence-by-sentence: buffer the
// deltas and enqueue each complete sentence for synthesis + playback the
// moment it lands, so the first audio starts within one sentence's
// synth latency instead of after the whole reply + one big round trip
// (#5358). The text-delta pseudo-lipsync below is the no-audio path
// only — letting it run while replies are spoken would flap the mouth
// ahead of the voice.
if (speakRef.current) {
ensureTtsTurn();
sawDeltaThisTurnRef.current = true;
enqueueSpeech(e.delta, turnSeqRef.current, false);
return;
}
if (playbackRef.current) return;
clearAckTimer();
setFace('speaking');
targetRef.current = pickViseme(e.delta);
visemeCodeRef.current = pickVisemeCode(e.delta);
lastDeltaAtRef.current = window.performance.now();
},
onDone: e => {
if (listeningRef.current) {
mascotLog('voice-session onDone suppressed — listening is active');
return;
}
const didMeaningfulWork = toolSucceededRef.current || subagentSucceededRef.current;
const explicitAck = pickConversationAckFace(e);
const ackFace: ConversationAckFace =
(explicitAck === 'happy' || explicitAck === null) && didMeaningfulWork
? 'celebrating'
: (explicitAck ?? 'happy');
toolSucceededRef.current = false;
subagentSucceededRef.current = false;
mascotLog(
'voice-session onDone ackFace=%s (explicit=%s didWork=%s)',
ackFace,
explicitAck ?? 'none',
didMeaningfulWork
);
if (!speakRef.current || !e.full_response?.trim()) {
holdThenIdle(ackFace);
return;
}
finalizeTtsTurn(e.full_response, ackFace);
},
onError: () => {
mascotLog('voice-session transition → concerned (chat_error), cancelling in-flight TTS');
cancelTtsPlayback();
holdThenIdle('concerned');
},
});
return () => {
unsub();
clearAckTimer();
cancelTtsPlayback();
};
}, []);
useEffect(() => {
if (!listening) return;
clearAckTimer();
const ttsWasInFlight = playbackRef.current != null || streamActiveRef.current;
mascotLog(
'voice-session listening-active tts-in-flight=%s — %s',
ttsWasInFlight,
ttsWasInFlight
? 'user started recording while TTS was playing (interrupted)'
: 'mic activated, no TTS to cancel'
);
// Barge-in: drain the whole streaming queue (active clip + any pending
// sentences), not just the current handle, so a mid-reply interruption
// stops everything at once.
cancelTtsPlayback();
targetRef.current = VISEMES.REST;
lastDeltaAtRef.current = 0;
setFace('idle');
}, [listening]);
// Turning speech off mid-reply must silence the *current* utterance, not just
// suppress the next one. `speakRef` is only consulted when a new reply
// arrives, so without this the chat mascot keeps talking over the text
// conversation after the user collapses its voice stage (which is what sets
// `speakReplies` false). Same drain as barge-in above.
useEffect(() => {
if (speakReplies) return;
if (playbackRef.current == null && !streamActiveRef.current) return;
mascotLog('speakReplies disabled mid-reply — cancelling in-flight TTS');
cancelTtsPlayback();
targetRef.current = VISEMES.REST;
lastDeltaAtRef.current = 0;
setFace('idle');
}, [speakReplies]);
function isTurnCurrent(seq: number): boolean {
return playbackSeqRef.current === seq;
}
/**
* Reset all playback + queue state and begin a fresh TTS turn. Cancels any
* clip still playing from a prior turn and bumps the seq token so its
* in-flight synth/play continuations no-op. Returns the new turn's seq.
*/
function beginTtsTurn(): number {
const orphan = playbackRef.current;
playbackRef.current = null;
playbackStartedAtRef.current = 0;
if (orphan) {
orphan.stop();
orphan.ended.catch(swallowAudioStop);
}
const seq = ++playbackSeqRef.current;
turnSeqRef.current = seq;
streamActiveRef.current = true;
sawDeltaThisTurnRef.current = false;
sentenceBufferRef.current = '';
chunkQueueRef.current = [];
queueClosedRef.current = false;
queuePumpingRef.current = false;
chunkAttemptedRef.current = false;
chunkPlayedRef.current = false;
queueAckRef.current = 'happy';
visemeFramesRef.current = [];
visemeCursorRef.current = 0;
visemeCodeRef.current = 'sil';
lastLipsyncLogRef.current = -1_000;
clearAckTimer();
setFace('thinking');
mascotLog('tts turn %d started', seq);
return seq;
}
/**
* Start a streaming turn on the first delta of a spoken reply. No-op if one
* is already open (subsequent deltas of the same reply).
*/
function ensureTtsTurn(): void {
// Reuse the open turn only while it is still accepting sentences. Once
// `chat_done` has closed it (queueClosedRef), a delta belongs to a NEW turn
// — start fresh so it never appends onto the previous reply's queue/ack.
if (streamActiveRef.current && !queueClosedRef.current) return;
beginTtsTurn();
}
/**
* Cancel everything: the active clip, any queued sentences, and the buffer.
* Bumps the seq so pending synth/play continuations drop their work; leaves
* the face for the caller to set. Idempotent.
*/
function cancelTtsPlayback(): void {
playbackSeqRef.current++;
streamActiveRef.current = false;
queueClosedRef.current = false;
queuePumpingRef.current = false;
sawDeltaThisTurnRef.current = false;
sentenceBufferRef.current = '';
chunkQueueRef.current = [];
const orphan = playbackRef.current;
playbackRef.current = null;
playbackStartedAtRef.current = 0;
if (orphan) {
orphan.stop();
orphan.ended.catch(swallowAudioStop);
}
visemeFramesRef.current = [];
visemeCursorRef.current = 0;
visemeCodeRef.current = 'sil';
}
/**
* Buffer streamed text, carve out complete sentences, and enqueue each for
* synthesis. `flush` also enqueues the trailing partial sentence (used at
* end-of-turn).
*/
function enqueueSpeech(text: string, seq: number, flush: boolean): void {
if (!isTurnCurrent(seq)) return;
sentenceBufferRef.current += text;
const { sentences, rest } = splitIntoSentences(sentenceBufferRef.current);
sentenceBufferRef.current = rest;
for (const sentence of sentences) enqueueSentence(sentence, seq);
if (flush) {
const tail = sentenceBufferRef.current.trim();
sentenceBufferRef.current = '';
if (tail) enqueueSentence(tail, seq);
}
}
/** Fire a sentence's synthesis eagerly and queue it for in-order playback. */
function enqueueSentence(sentence: string, seq: number): void {
const spoken = sentence.trim();
if (!spoken) return;
chunkAttemptedRef.current = true;
const synth: Promise<ChunkSynth> = synthesizeSpeech(spoken, {
voiceId: mascotVoiceIdRef.current,
})
.then(tts => ({ ok: true, tts }) as ChunkSynth)
.catch(() => ({ ok: false }) as ChunkSynth);
chunkQueueRef.current.push({ text: spoken, synth });
mascotLog(
'tts enqueued sentence chars=%d queue=%d',
spoken.length,
chunkQueueRef.current.length
);
void pumpQueue(seq);
}
/**
* Finalize the turn on `chat_done`. If the reply streamed as deltas, flush
* the trailing fragment and close the queue; otherwise (no deltas were seen)
* speak the whole response as a fresh turn.
*/
function finalizeTtsTurn(fullResponse: string, ackFace: ConversationAckFace): void {
if (sawDeltaThisTurnRef.current && streamActiveRef.current) {
queueAckRef.current = ackFace;
const seq = turnSeqRef.current;
enqueueSpeech('', seq, true);
queueClosedRef.current = true;
void pumpQueue(seq);
return;
}
// No deltas arrived (some backends only emit the final message) — start a
// fresh turn and speak the whole response, still chunked into sentences.
const seq = beginTtsTurn();
queueAckRef.current = ackFace;
enqueueSpeech(fullResponse, seq, true);
queueClosedRef.current = true;
void pumpQueue(seq);
}
/**
* Play queued chunks strictly in order, one at a time. Runs until the queue
* drains; a later enqueue restarts it. Finishes the turn once the queue is
* drained and closed.
*/
async function pumpQueue(seq: number): Promise<void> {
if (queuePumpingRef.current) return;
if (!isTurnCurrent(seq)) return;
queuePumpingRef.current = true;
try {
while (chunkQueueRef.current.length > 0) {
if (!isTurnCurrent(seq)) return;
const chunk = chunkQueueRef.current.shift()!;
const result = await chunk.synth;
if (!isTurnCurrent(seq)) return;
if (!result.ok) {
mascotLog('tts chunk synth failed — skipping');
continue;
}
await playChunk(result.tts, chunk.text, seq);
}
} finally {
// Only act if this pump still owns the turn — a superseding turn resets
// state explicitly, so a stale pump must not touch it. Finalizing here in
// the `finally` guarantees the turn always ends (mouth rests, ack fires)
// even if a chunk unexpectedly throws, instead of stranding it speaking.
if (isTurnCurrent(seq)) {
queuePumpingRef.current = false;
maybeFinishTurn(seq);
}
}
}
/**
* Play one chunk: derive its own viseme timeline, play the audio, drive the
* mouth off it, and resolve when it ends. Keeps `face='speaking'` across
* chunk boundaries so the RAF loop never tears down between sentences.
*/
async function playChunk(
tts: Awaited<ReturnType<typeof synthesizeSpeech>>,
text: string,
seq: number
): Promise<void> {
let { frames, source } = selectVisemeSource(tts);
let handle: PlaybackHandle;
try {
handle = await playBase64Audio(tts.audio_base64, tts.audio_mime ?? 'audio/mpeg', {
maxDurationMs: TTS_MAX_PLAYBACK_MS,
});
} catch (err) {
// A decode/autoplay failure degrades this chunk; the turn ends concerned
// only if no chunk ever played (see maybeFinishTurn).
mascotLog('tts chunk playback could not start: %s', String(err));
return;
}
if (!isTurnCurrent(seq)) {
handle.stop();
handle.ended.catch(swallowAudioStop);
return;
}
// Start lipsync as soon as play() succeeds; metadata can lag by the 500ms
// decoder fallback, so estimate first and refine once it resolves.
let audioMs = handle.durationMs();
const waitingForMetadata = audioMs <= 0;
if (waitingForMetadata) audioMs = estimateTtsPlaybackMs(text, frames);
if (frames.length === 0) {
frames = proceduralVisemes(text, audioMs);
source = 'procedural';
}
frames = normalizeVisemeTimeline(frames, audioMs);
visemeFramesRef.current = frames;
visemeCursorRef.current = 0;
playbackRef.current = handle;
playbackStartedAtRef.current = window.performance.now();
lastLipsyncLogRef.current = -1_000;
chunkPlayedRef.current = true;
setFace('speaking');
mascotLog('tts chunk playback started (%s) frames=%d', source, frames.length);
if (waitingForMetadata) {
await handle.metadataReady;
if (!isTurnCurrent(seq)) {
handle.stop();
handle.ended.catch(swallowAudioStop);
return;
}
const measuredAudioMs = handle.durationMs();
if (measuredAudioMs > 0 && playbackRef.current === handle) {
const measuredFrames =
source === 'procedural'
? proceduralVisemes(text, measuredAudioMs)
: normalizeVisemeTimeline(visemeFramesRef.current, measuredAudioMs);
visemeFramesRef.current =
source === 'procedural'
? normalizeVisemeTimeline(measuredFrames, measuredAudioMs)
: measuredFrames;
visemeCursorRef.current = 0;
}
}
try {
await handle.ended;
} catch (err) {
// A real decoder/playback error (not the stop sentinel) degrades this
// chunk. Do NOT rethrow — that would escape the pump, skip the cleanup
// below, and strand the mascot in the speaking state. Log and fall
// through so the handle is released and the queue finishes normally.
if (!isAudioStopped(err)) {
mascotLog('tts chunk ended with a playback error: %s', String(err));
}
}
// Release the finished clip so the between-chunk gap rests the mouth. The
// face stays 'speaking'; the next chunk or the finish transition follows.
if (isTurnCurrent(seq) && playbackRef.current === handle) {
playbackRef.current = null;
playbackStartedAtRef.current = 0;
}
}
/**
* Once the queue is drained and closed, end the turn: rest the mouth and play
* the acknowledgement beat — or a concerned beat if a sentence was attempted
* but nothing ever spoke (total synth/playback failure).
*/
function maybeFinishTurn(seq: number): void {
if (!isTurnCurrent(seq)) return;
if (!streamActiveRef.current) return;
if (!queueClosedRef.current) return;
if (chunkQueueRef.current.length > 0) return;
if (queuePumpingRef.current) return;
streamActiveRef.current = false;
playbackRef.current = null;
playbackStartedAtRef.current = 0;
visemeFramesRef.current = [];
visemeCodeRef.current = 'sil';
const degraded = chunkAttemptedRef.current && !chunkPlayedRef.current;
mascotLog('tts turn %d finished degraded=%s', seq, degraded);
holdThenIdle(degraded ? 'concerned' : queueAckRef.current);
}
useEffect(() => {
if (face !== 'speaking') return;
let raf = 0;
const loop = () => {
force(t => t + 1);
raf = window.requestAnimationFrame(loop);
};
raf = window.requestAnimationFrame(loop);
return () => window.cancelAnimationFrame(raf);
}, [face]);
let viseme: VisemeShape = VISEMES.REST;
let visemeCode = 'sil';
const playback = playbackRef.current;
if (playback) {
const audioMs = playback.currentMs();
const wallClockMs =
playbackStartedAtRef.current > 0
? window.performance.now() - playbackStartedAtRef.current
: audioMs;
const ms = audioMs < 0 ? -1 : Math.max(audioMs, wallClockMs);
if (ms >= 0) {
const { frame, cursor } = findActiveFrame(
visemeFramesRef.current,
ms,
visemeCursorRef.current
);
visemeCursorRef.current = cursor;
viseme = frame ? oculusVisemeToShape(frame.viseme) : VISEMES.REST;
visemeCode = frame ? frame.viseme : 'sil';
if (ms - lastLipsyncLogRef.current >= 500) {
lastLipsyncLogRef.current = ms;
mascotLog(
'lipsync ms=%d cursor=%d/%d code=%s',
Math.round(ms),
cursor,
visemeFramesRef.current.length,
visemeCode
);
}
}
} else if (face === 'speaking') {
const since = window.performance.now() - lastDeltaAtRef.current;
const decay = Math.max(0, Math.min(1, since / VISEME_DECAY_MS));
viseme = lerpViseme(targetRef.current, VISEMES.REST, decay);
visemeCode = decay > 0.5 ? 'sil' : visemeCodeRef.current;
}
const effectiveFace: MascotFace = listening ? 'listening' : face;
const effectiveViseme: VisemeShape = listening ? VISEMES.REST : viseme;
const effectiveVisemeCode: string = listening ? 'sil' : visemeCode;
return { face: effectiveFace, viseme: effectiveViseme, visemeCode: effectiveVisemeCode };
}