Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(voice): route non-English speech to the multilingual ASR model
nirholasbc7b0cbd56b6
diff --git a/api/_lib/asr-nvidia.js b/api/_lib/asr-nvidia.jsindex fcc385357..fd7669af6 100644--- a/api/_lib/asr-nvidia.js+++ b/api/_lib/asr-nvidia.js@@ -53,7 +53,73 @@ export function resolveAsrEncoding(encoding) {}export function nvidiaAsrConfigured() {- return Boolean(env.NVIDIA_API_KEY && env.NVIDIA_ASR_FUNCTION_ID);+ return Boolean(+ env.NVIDIA_API_KEY &&+ (env.NVIDIA_ASR_FUNCTION_ID || env.NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL),+ );+}++// ── Language routing ────────────────────────────────────────────────────────+// One hosted function does not cover every language. The English-tuned Parakeet+// models reject a non-`en` language_code outright (gRPC 3 INVALID_ARGUMENT), so+// a Mandarin utterance sent to the English pin fails rather than degrading. A+// multilingual model (ai-canary-1b-asr on our account) transcribes the languages+// below, and English at the same quality as the English pin. We therefore keep+// the English pin for `en*` (better casing and punctuation on English) and route+// every other language to the multilingual id when one is configured.+//+// The list is empirical, not aspirational: every tag here was verified by+// synthesizing a sentence in that language and reading back a correct+// transcript (`node scripts/verify-nvidia-asr.mjs --languages`). Languages the+// model rejects (Arabic, Malay, Tamil, Filipino, Norwegian) or garbles+// (Indonesian, Vietnamese, Finnish, Danish, Greek, Hungarian) are deliberately+// absent, so the client never offers a language the lane cannot hear.+export const MULTILINGUAL_ASR_LANGUAGES = [+ 'zh-CN',+ 'zh-TW',+ 'ja-JP',+ 'ko-KR',+ 'es-ES',+ 'fr-FR',+ 'de-DE',+ 'it-IT',+ 'pt-BR',+ 'ru-RU',+ 'hi-IN',+ 'nl-NL',+ 'pl-PL',+ 'cs-CZ',+];++/** True when `language` is an English tag (the English pin's territory). */+function isEnglish(language) {+ return String(language || '').trim().toLowerCase().startsWith('en');+}++/**+ * NVCF function id to use for `language`, or null when nothing is configured.+ * English (and any language with no multilingual id configured) stays on+ * NVIDIA_ASR_FUNCTION_ID; everything else prefers the multilingual id.+ */+export function asrFunctionIdFor(language) {+ const multilingual = env.NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL || null;+ const primary = env.NVIDIA_ASR_FUNCTION_ID || null;+ if (!isEnglish(language) && multilingual) return multilingual;+ return primary || multilingual;+}++/**+ * BCP-47 tags this deployment can actually recognize, for the capability probe.+ * Always includes en-US (the base lane); the multilingual tags appear only when+ * a multilingual function id is configured.+ */+export function asrLanguages() {+ const langs = env.NVIDIA_ASR_FUNCTION_ID ? ['en-US', 'en-GB'] : [];+ if (env.NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL) {+ if (!langs.length) langs.push('en-US');+ langs.push(...MULTILINGUAL_ASR_LANGUAGES);+ }+ return langs;}// Parse a RIFF/WAVE header → { sampleRateHz, channels, pcm } so a WAV upload can@@ -167,9 +233,10 @@ export async function transcribeNvidiaAsr({model = '',timeoutMs = DEFAULT_TIMEOUT_MS,apiKey,+ functionId: functionIdOverride,} = {}) {const key = apiKey || env.NVIDIA_API_KEY;- const functionId = env.NVIDIA_ASR_FUNCTION_ID;+ const functionId = functionIdOverride || asrFunctionIdFor(language);if (!key || !functionId) {const err = new Error(!key ? 'NVIDIA_API_KEY not set' : 'NVIDIA_ASR_FUNCTION_ID not set (run scripts/verify-nvidia-asr.mjs --list to discover it)',diff --git a/api/_lib/env.js b/api/_lib/env.jsindex 43243c866..8ec199036 100644--- a/api/_lib/env.js+++ b/api/_lib/env.js@@ -666,6 +666,18 @@ export const env = {return opt('NVIDIA_ASR_FUNCTION_ID');},+ // NVCF function id for a MULTILINGUAL hosted ASR model, used when a caller asks+ // for a non-English language (api/_lib/asr-nvidia.js). The English-tuned+ // Parakeet functions reject a non-`en` language_code outright (gRPC 3+ // INVALID_ARGUMENT), so without this id the free lane is English-only and a+ // Mandarin/Japanese/Spanish speaker falls back to the browser recognizer.+ // Discover the live id with `node scripts/verify-nvidia-asr.mjs --list` and+ // pick a multilingual model (ai-canary-1b-asr on our account). Optional: when+ // unset, non-English requests keep using NVIDIA_ASR_FUNCTION_ID.+ get NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL() {+ return opt('NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL');+ },+// NVCF function id for the hosted Audio2Face-3D model used by the free facial-// animation lane (api/_lib/a2f-nvidia.js, api/a2f.js) that turns Magpie speech// into a per-frame ARKit blendshape track. Unlike the ASR id this is OPTIONAL:diff --git a/api/asr.js b/api/asr.jsindex 1f71738bb..465e482c4 100644--- a/api/asr.js+++ b/api/asr.js@@ -17,7 +17,8 @@// audio/flac → FLAC// audio/ogg → Ogg/Opus// or a JSON body { audio: <base64>, format?, language?, sampleRate?, words? }.-// Query/JSON params: language (BCP-47, default en-US), rate (PCM sample rate),+// Query/JSON params: language (BCP-47, default en-US; a non-English tag routes+// to the multilingual model, see asrFunctionIdFor), rate (PCM sample rate),// words ("1" for word-level timestamps), model (override Riva model name).//// Response: { text, confidence, language, model, durationSec, words? }.@@ -30,6 +31,7 @@ import {nvidiaAsrConfigured,resolveAsrEncoding,parseWav,+ asrLanguages,} from './_lib/asr-nvidia.js';export const maxDuration = 30;@@ -86,6 +88,10 @@ export default wrap(async function handler(req, res) {// 16 kHz mono is the Riva acoustic-model rate and the rate the client// downsamples its capture to — surface it so the two never drift.sampleRate: 16000,+ // BCP-47 tags this deployment can recognize. A client builds its+ // language picker from this rather than guessing: offering a language+ // the lane rejects is worse than not offering it at all.+ languages: asrLanguages(),},{ 'cache-control': 'public, max-age=60' },);@@ -201,6 +207,17 @@ export default wrap(async function handler(req, res) {code === 'invalid_argument' ? 400 :code === 'not_configured' ? 503 :502;+ // An `invalid_argument` on a non-English language is nearly always the+ // hosted model refusing that language_code, which reads as a generic+ // provider failure unless we say so. Name the language and the way out.+ if (code === 'invalid_argument' && !String(language).toLowerCase().startsWith('en')) {+ return error(+ res,+ 400,+ 'unsupported_language',+ `The speech-to-text lane could not recognize "${language}". Supported: ${asrLanguages().join(', ') || 'none'}.`,+ );+ }return error(res, status, code, `Speech recognition failed: ${e?.message || 'unknown error'}`);}});diff --git a/src/voice/talk-controller.js b/src/voice/talk-controller.jsindex a01a14403..81a6cfcff 100644--- a/src/voice/talk-controller.js+++ b/src/voice/talk-controller.js@@ -27,6 +27,12 @@import { LipsyncDriver, tapAudioElement } from './lipsync-driver.js';import { MicCapture } from './mic-capture.js';+import {+ detectTalkLanguage,+ edgeVoiceFor,+ languageInstruction,+ resolveTalkLanguage,+} from './talk-languages.js';import { log } from '../shared/log.js';// Riva interim recognition cadence. While holding to talk we fire a recognition@@ -36,12 +42,6 @@ import { log } from '../shared/log.js';const INTERIM_INTERVAL_MS = 1400;const MAX_INTERIMS = 4;-const EDGE_VOICES_BY_GENDER = {- female: 'en-US-AriaNeural',- male: 'en-US-GuyNeural',- neutral: 'en-US-AriaNeural',-};-const ELEVEN_DEFAULT_VOICE = 'EXAVITQu4vr4xnSDxMaL'; // Bella — ElevenLabs default voiceexport class TalkController {@@ -56,7 +56,7 @@ export class TalkController {* @param {(err: Error) => void} [opts.onError]* @param {{ attach: Function, setMouthShape: Function }} opts.mouthTarget*/- constructor({ avatar, systemPromptFn, onMessage, onStateChange, onInterim, onError, mouthTarget, commandInterceptor }) {+ constructor({ avatar, systemPromptFn, onMessage, onStateChange, onInterim, onError, mouthTarget, commandInterceptor, language }) {if (!avatar?.id) throw new Error('TalkController: avatar.id required');if (!mouthTarget) throw new Error('TalkController: mouthTarget required');this.avatar = avatar;@@ -82,24 +82,69 @@ export class TalkController {// Speech-to-text routing. 'riva' = server-side NVIDIA Riva (cross-browser),// 'browser' = window.SpeechRecognition (Chrome/Edge/Safari), 'none' = text- // only. Resolved once by prepare(); language tracks the avatar's locale.- this._sttMode = null;- this._sttModePromise = null;+ // only. The right path depends on the CONVERSATION LANGUAGE, not just the+ // browser: the server lane recognizes the languages it advertises on+ // /api/asr, and the browser recognizer covers the rest. prepare() probes+ // once; the mode is then derived per language.+ this._probePromise = null;+ this._probeSettledFlag = false;+ this._serverAsr = null; // { configured, languages } from GET /api/asr+ this._hasBrowserSR = !!(+ typeof window !== 'undefined' &&+ (window.SpeechRecognition || window.webkitSpeechRecognition)+ );this._listenMode = null; // mode of the in-flight turnthis._mic = null;this._interimTimer = null;this._interimBusy = false;this._interimCount = 0;- this.language = 'en-US';++ // Conversation language: an explicit choice wins, else the language the+ // site is already displayed in, else the browser's own preference. Every+ // leg of the loop reads it: recognition, the reply, and the voice.+ this.language = resolveTalkLanguage(+ language ||+ detectTalkLanguage({+ uiLocale: siteLocale(),+ navLangs:+ typeof navigator !== 'undefined'+ ? navigator.languages || [navigator.language]+ : [],+ }),+ );+ }++ /**+ * Switch the conversation language mid-session. Takes effect on the next turn:+ * recognition, the reply, and the voice all follow it.+ * @returns {string} the resolved tag (never an unsupported one).+ */+ setLanguage(tag) {+ this.language = resolveTalkLanguage(tag);+ return this.language;}get state() {return this._state;}- /** Resolved STT path: 'riva' | 'browser' | 'none' (null until prepare()). */+ /** STT path for the CURRENT language: 'riva' | 'browser' | 'none'. */get sttMode() {- return this._sttMode;+ return this._modeFor(this.language);+ }++ /** True when the free server lane can hear the current language. */+ get serverHearsLanguage() {+ return this._serverSupports(this.language);+ }++ /**+ * Recognizer a given language would use right now: 'riva' | 'browser' | 'none'.+ * A picker calls this to label each option honestly before the user commits to+ * one, instead of discovering the dead end by holding the mic.+ */+ sttModeFor(language) {+ return this._modeFor(resolveTalkLanguage(language));}/** Live mic level (0..1) while the Riva lane is capturing; 0 otherwise. */@@ -108,32 +153,61 @@ export class TalkController {}/**- * Decide the STT path once, before the first turn. Probes /api/asr for the- * free NVIDIA Riva lane (works in every browser, including Firefox); falls- * back to the browser's own SpeechRecognition, then to text-only. Safe to- * call repeatedly — the probe runs at most once. Returns the resolved mode.+ * Decide the STT path before the first turn. Probes /api/asr once for the free+ * NVIDIA Riva lane (works in every browser, including Firefox) and the set of+ * languages that lane recognizes; falls back to the browser's own+ * SpeechRecognition, then to text-only. Safe to call repeatedly; the probe+ * runs at most once. Returns the mode for the current language.*/async prepare() {- if (this._sttMode) return this._sttMode;- if (!this._sttModePromise) {- this._sttModePromise = (async () => {- const hasBrowserSR = !!(window.SpeechRecognition || window.webkitSpeechRecognition);- const canCapture = MicCapture.isSupported();- if (canCapture) {- try {- const r = await fetch('/api/asr', { headers: { accept: 'application/json' } });- if (r.ok) {- const j = await r.json();- if (j?.configured) return (this._sttMode = 'riva');- }- } catch {- // Probe failure is not fatal — fall back to whatever the browser offers.- }+ if (!this._probePromise) {+ this._probePromise = (async () => {+ if (!MicCapture.isSupported()) return;+ try {+ const r = await fetch('/api/asr', { headers: { accept: 'application/json' } });+ if (!r.ok) return;+ const j = await r.json();+ if (!j?.configured) return;+ this._serverAsr = {+ configured: true,+ // A deployment that predates language routing advertises no list;+ // treat that as the English-only lane it is rather than assuming+ // it can hear Mandarin and failing the first turn.+ languages: Array.isArray(j.languages) && j.languages.length+ ? j.languages.map((t) => String(t).toLowerCase())+ : ['en-us'],+ };+ } catch {+ // Probe failure is not fatal: fall back to whatever the browser offers.+ } finally {+ this._probeSettledFlag = true;}- return (this._sttMode = hasBrowserSR ? 'browser' : 'none');})();}- return this._sttModePromise;+ await this._probePromise;+ return this.sttMode;+ }++ /** Does the probed server lane advertise recognition for this language? */+ _serverSupports(language) {+ if (!this._serverAsr?.configured) return false;+ const tag = String(language || '').toLowerCase();+ const primary = tag.split('-')[0];+ return this._serverAsr.languages.some(+ (l) => l === tag || l.split('-')[0] === primary,+ );+ }++ /**+ * Best recognizer for a language. The server lane leads where it can hear the+ * language (cross-browser, and it does not ship the user's audio to Google);+ * the browser's recognizer covers everything else, which is what makes a+ * language the server lane never learned still usable in Chrome and Safari.+ */+ _modeFor(language) {+ if (this._serverSupports(language)) return 'riva';+ if (this._hasBrowserSR) return 'browser';+ return 'none';}/**@@ -145,7 +219,7 @@ export class TalkController {startListening() {if (this._state !== 'idle') return false;- const mode = this._sttMode || this._fallbackSttMode();+ const mode = this.probeSettled ? this.sttMode : this._fallbackSttMode();if (mode === 'riva') {this._listenMode = 'riva';this._startRivaListening();@@ -180,10 +254,19 @@ export class TalkController {// recognizer (zero setup, instant) when present; otherwise attempt Riva if the// environment can capture audio at all._fallbackSttMode() {- if (window.SpeechRecognition || window.webkitSpeechRecognition) return 'browser';+ if (this._hasBrowserSR) return 'browser';return MicCapture.isSupported() ? 'riva' : 'none';}+ /**+ * True once the /api/asr probe has answered (or failed). Until it has, the+ * language→recognizer mapping is a guess, so a UI should not tell the user a+ * language is unusable yet.+ */+ get probeSettled() {+ return this._probeSettledFlag;+ }+// ── Browser SpeechRecognition path (Chrome/Edge/Safari) ────────────────_startBrowserListening() {const RecCls = window.SpeechRecognition || window.webkitSpeechRecognition;@@ -460,6 +543,14 @@ export class TalkController {}}+ // The agent's own persona plus, for a non-English conversation, the one line+ // that makes the reply come back in the language the user is speaking.+ _systemPrompt() {+ return [this.systemPromptFn(), languageInstruction(this.language)]+ .filter(Boolean)+ .join('\n\n');+ }+async _streamChat(message) {const isUuid =typeof this.avatar.id === 'string' &&@@ -471,7 +562,7 @@ export class TalkController {credentials: 'include',body: JSON.stringify({message,- system_prompt: this.systemPromptFn(),+ system_prompt: this._systemPrompt(),history: this._history.slice(-10, -1),...(isUuid ? { agentId: this.avatar.id } : {}),...(this.avatar.agent_id ? { agentId: this.avatar.agent_id } : {}),@@ -634,7 +725,10 @@ export class TalkController {this.avatar?.source_meta?.gender ||this.avatar?.source_meta?.bodyType ||'neutral';- const voice = EDGE_VOICES_BY_GENDER[gender] || EDGE_VOICES_BY_GENDER.neutral;+ // Language first, body type second: an English voice reading Mandarin is+ // unintelligible, while a female voice reading a male avatar's line is+ // merely a mismatch.+ const voice = edgeVoiceFor(this.language, gender);const r = await fetch('/api/tts/edge', {method: 'POST',headers: { 'content-type': 'application/json' },@@ -652,6 +746,19 @@ export class TalkController {}}+// The language the site UI is already displayed in, when the i18n runtime is on+// the page. Read through the global instead of importing src/i18n.js: talk mode+// also runs inside embeds that never boot the translation runtime, and a hard+// import would pull the whole catalogue loader into those bundles.+function siteLocale() {+ if (typeof window === 'undefined') return '';+ try {+ return window.threewsI18n?.getLocale?.() || document.documentElement?.lang || '';+ } catch {+ return '';+ }+}+// Error carrying a machine-readable `.code` so the host UI can route each// failure precisely (mic denial → text input, rate limit → "try later", etc).function coded(message, code) {diff --git a/src/voice/talk-languages.js b/src/voice/talk-languages.jsnew file mode 100644index 000000000..1007de3c3--- /dev/null+++ b/src/voice/talk-languages.js@@ -0,0 +1,107 @@+/**+ * Languages the avatar voice loop can hold a conversation in.+ *+ * One turn crosses three systems, and a language only counts as supported when+ * all three handle it:+ *+ * 1. speech in: server-side Riva ASR (/api/asr, multilingual model) or the+ * browser's own SpeechRecognition+ * 2. the reply: /api/chat, told which language to answer in+ * 3. speech out: ElevenLabs (a cloned voice speaks every language below via+ * the multilingual model) with Microsoft Edge Neural TTS as+ * the fallback when no voice is cloned+ *+ * `asr: true` marks the tags the free server lane transcribes accurately, each+ * verified against real synthesized speech (`node scripts/verify-nvidia-asr.mjs+ * --languages`). The rest still work when the browser's recognizer speaks them,+ * so they stay offerable in Chrome/Edge/Safari rather than being hidden.+ *+ * `edge` is the Microsoft Neural voice used when the avatar has no cloned voice:+ * without it a Mandarin reply comes out of an American English voice reading+ * pinyin-shaped noise. Female/male pairs mirror the avatar's own body type.+ */++export const TALK_LANGUAGES = [+ { tag: 'en-US', native: 'English (US)', english: 'English (US)', asr: true, edge: { female: 'en-US-AriaNeural', male: 'en-US-GuyNeural' } },+ { tag: 'en-GB', native: 'English (UK)', english: 'English (UK)', asr: true, edge: { female: 'en-GB-SoniaNeural', male: 'en-GB-RyanNeural' } },+ { tag: 'zh-CN', native: '简体中文', english: 'Chinese (Mandarin)', asr: true, edge: { female: 'zh-CN-XiaoxiaoNeural', male: 'zh-CN-YunxiNeural' } },+ { tag: 'zh-TW', native: '繁體中文', english: 'Chinese (Taiwan)', asr: true, edge: { female: 'zh-TW-HsiaoChenNeural', male: 'zh-TW-YunJheNeural' } },+ { tag: 'ja-JP', native: '日本語', english: 'Japanese', asr: true, edge: { female: 'ja-JP-NanamiNeural', male: 'ja-JP-KeitaNeural' } },+ { tag: 'ko-KR', native: '한국어', english: 'Korean', asr: true, edge: { female: 'ko-KR-SunHiNeural', male: 'ko-KR-InJoonNeural' } },+ { tag: 'es-ES', native: 'Español', english: 'Spanish', asr: true, edge: { female: 'es-ES-ElviraNeural', male: 'es-ES-AlvaroNeural' } },+ { tag: 'fr-FR', native: 'Français', english: 'French', asr: true, edge: { female: 'fr-FR-DeniseNeural', male: 'fr-FR-HenriNeural' } },+ { tag: 'de-DE', native: 'Deutsch', english: 'German', asr: true, edge: { female: 'de-DE-KatjaNeural', male: 'de-DE-ConradNeural' } },+ { tag: 'it-IT', native: 'Italiano', english: 'Italian', asr: true, edge: { female: 'it-IT-ElsaNeural', male: 'it-IT-DiegoNeural' } },+ { tag: 'pt-BR', native: 'Português (BR)', english: 'Portuguese (Brazil)', asr: true, edge: { female: 'pt-BR-FranciscaNeural', male: 'pt-BR-AntonioNeural' } },+ { tag: 'ru-RU', native: 'Русский', english: 'Russian', asr: true, edge: { female: 'ru-RU-SvetlanaNeural', male: 'ru-RU-DmitryNeural' } },+ { tag: 'hi-IN', native: 'हिन्दी', english: 'Hindi', asr: true, edge: { female: 'hi-IN-SwaraNeural', male: 'hi-IN-MadhurNeural' } },+ { tag: 'nl-NL', native: 'Nederlands', english: 'Dutch', asr: true, edge: { female: 'nl-NL-ColetteNeural', male: 'nl-NL-MaartenNeural' } },+ { tag: 'pl-PL', native: 'Polski', english: 'Polish', asr: true, edge: { female: 'pl-PL-ZofiaNeural', male: 'pl-PL-MarekNeural' } },+ { tag: 'cs-CZ', native: 'Čeština', english: 'Czech', asr: true, edge: { female: 'cs-CZ-VlastaNeural', male: 'cs-CZ-AntoninNeural' } },+ { tag: 'ar-SA', native: 'العربية', english: 'Arabic', asr: false, edge: { female: 'ar-SA-ZariyahNeural', male: 'ar-SA-HamedNeural' } },+ { tag: 'tr-TR', native: 'Türkçe', english: 'Turkish', asr: false, edge: { female: 'tr-TR-EmelNeural', male: 'tr-TR-AhmetNeural' } },+ { tag: 'id-ID', native: 'Bahasa Indonesia', english: 'Indonesian', asr: false, edge: { female: 'id-ID-GadisNeural', male: 'id-ID-ArdiNeural' } },+ { tag: 'sv-SE', native: 'Svenska', english: 'Swedish', asr: false, edge: { female: 'sv-SE-SofieNeural', male: 'sv-SE-MattiasNeural' } },+ { tag: 'uk-UA', native: 'Українська', english: 'Ukrainian', asr: false, edge: { female: 'uk-UA-PolinaNeural', male: 'uk-UA-OstapNeural' } },+ { tag: 'ro-RO', native: 'Română', english: 'Romanian', asr: false, edge: { female: 'ro-RO-AlinaNeural', male: 'ro-RO-EmilNeural' } },+];++export const DEFAULT_TALK_LANGUAGE = 'en-US';++const BY_TAG = new Map(TALK_LANGUAGES.map((l) => [l.tag.toLowerCase(), l]));++/** The language entry for a tag, or null when we do not carry that language. */+export function talkLanguage(tag) {+ const key = String(tag || '').trim().toLowerCase();+ if (!key) return null;+ if (BY_TAG.has(key)) return BY_TAG.get(key);+ // "zh", "zh-Hans-CN", "pt" → the first entry sharing the primary subtag, so a+ // browser locale we do not list exactly still lands on the right language.+ const primary = key.split('-')[0];+ return TALK_LANGUAGES.find((l) => l.tag.toLowerCase().split('-')[0] === primary) || null;+}++/**+ * Normalize any locale-ish input to a tag we support.+ * @returns {string} a supported BCP-47 tag; DEFAULT_TALK_LANGUAGE when unknown.+ */+export function resolveTalkLanguage(tag) {+ return talkLanguage(tag)?.tag || DEFAULT_TALK_LANGUAGE;+}++/**+ * Pick the conversation language for a first visit, in preference order:+ * an explicit stored choice, then the language the site UI is already in, then+ * the browser's own languages. Pure, so the ordering is unit-testable.+ *+ * @param {{ stored?: string, uiLocale?: string, navLangs?: string[] }} input+ */+export function detectTalkLanguage({ stored, uiLocale, navLangs } = {}) {+ const candidates = [stored, uiLocale, ...(navLangs || [])];+ for (const c of candidates) {+ const hit = talkLanguage(c);+ if (hit) return hit.tag;+ }+ return DEFAULT_TALK_LANGUAGE;+}++/**+ * Microsoft Edge Neural voice for a language, matched to the avatar's body type.+ * Falls back to the language's female voice, then to US English.+ */+export function edgeVoiceFor(tag, gender = 'neutral') {+ const entry = talkLanguage(tag) || talkLanguage(DEFAULT_TALK_LANGUAGE);+ const key = gender === 'male' ? 'male' : 'female';+ return entry.edge[key] || entry.edge.female;+}++/**+ * The instruction appended to the agent's system prompt so the reply comes back+ * in the conversation language. English needs no instruction: it is what every+ * agent persona already writes in, and a redundant line only dilutes the prompt.+ */+export function languageInstruction(tag) {+ const entry = talkLanguage(tag);+ if (!entry || entry.tag.startsWith('en')) return '';+ return `Reply only in ${entry.english} (${entry.native}). The user is speaking ${entry.english}; answer naturally in that language, never in English, and never translate your own reply. Keep replies short enough to be spoken aloud.`;+}diff --git a/src/voice/talk-mode.js b/src/voice/talk-mode.jsindex 69d2fb796..3d56397ac 100644--- a/src/voice/talk-mode.js+++ b/src/voice/talk-mode.js@@ -19,6 +19,7 @@ import { TalkScene } from './talk-scene.js';import { AvatarMouthTarget } from './avatar-morph-target.js';import { TalkController } from './talk-controller.js';import { openVoiceCloneModal } from './voice-clone-modal.js';+import { TALK_LANGUAGES, detectTalkLanguage, talkLanguage } from './talk-languages.js';import { nextPreset, PRESET_LABELS } from './camera-presets.js';import { WalletIntentController } from './wallet-intent.js';import { log } from '../shared/log.js';@@ -67,6 +68,7 @@ export function openTalkMode({ avatar, systemPromptFn }) {const textbar = overlay.querySelector('.tws-talk-textbar');const textInput = overlay.querySelector('.tws-talk-input');const walletHintBtn = overlay.querySelector('.tws-talk-wallet-hint');+ const langSel = overlay.querySelector('.tws-talk-lang');nameEl.textContent = avatar.name || 'Avatar';// Owner-only affordance: `owner_id` is stripped from the API response for@@ -129,6 +131,7 @@ export function openTalkMode({ avatar, systemPromptFn }) {controller = new TalkController({avatar,systemPromptFn,+ language: readStoredTalkLanguage(),mouthTarget,onMessage: (m) => appendTranscript(transcriptEl, m),onInterim: (t) => setPartial(partialEl, t),@@ -156,18 +159,65 @@ export function openTalkMode({ avatar, systemPromptFn }) {}}- // Resolve the speech-to-text path up front (probes the free Riva lane). When no- // recognizer exists at all, retire the mic button and lead with text input.- controller.prepare().then((mode) => {+ // ── Conversation language ──────────────────────────────────────────+ // The picker drives all three legs of the loop: what the recognizer listens+ // for, what language the agent answers in, and which voice speaks the reply+ // (a cloned voice speaks every language here; the fallback voice switches to+ // a native one). Options are labelled with the recognizer each language will+ // actually get, so a "type only" language is visible before it is chosen.+ function renderLanguageOptions() {+ if (!langSel) return;+ const current = controller.language;+ langSel.innerHTML = TALK_LANGUAGES.map((l) => {+ const mode = controller.sttModeFor(l.tag);+ const suffix = mode === 'none' ? ' (type only)' : '';+ const label = l.native === l.english ? l.native : `${l.native} · ${l.english}`;+ return `<option value="${l.tag}"${l.tag === current ? ' selected' : ''}>${label}${suffix}</option>`;+ }).join('');+ langSel.value = current;+ }++ // Reflect the recognizer available for the chosen language: no recognizer at+ // all retires the mic button and leads with the text input rather than leaving+ // a button that can only fail.+ function applySttMode() {+ const mode = controller.sttMode;+ const name = talkLanguage(controller.language)?.english || 'this language';+ // Before the probe answers, the mapping is a guess, so never retire the mic+ // button or accuse a language of being unusable on a guess.+ if (!controller.probeSettled) {+ holdBtn.title = `Hold to talk in ${name}`;+ return;+ }+ holdBtn.hidden = mode === 'none';if (mode === 'none') {- holdBtn.hidden = true;textInput?.focus();+ showError(errEl, `Speech input isn't available for ${name} in this browser. Type instead and the avatar still answers out loud.`);} else {holdBtn.title =mode === 'riva'- ? 'Hold to talk — powered by NVIDIA Riva speech-to-text'- : 'Hold to talk';+ ? `Hold to talk in ${name}, powered by NVIDIA Riva speech-to-text`+ : `Hold to talk in ${name}`;}+ }++ renderLanguageOptions();+ applySttMode();++ langSel?.addEventListener('change', () => {+ const tag = controller.setLanguage(langSel.value);+ storeTalkLanguage(tag);+ hideError(errEl);+ renderLanguageOptions();+ applySttMode();+ });++ // Resolve the speech-to-text path up front (probes the free Riva lane): the+ // probe tells us which languages the free server lane can hear, so the labels+ // and the mic button both settle once it answers.+ controller.prepare().then(() => {+ renderLanguageOptions();+ applySttMode();});// ── Conversational Wallet (owner-only) ──────────────────────────────@@ -340,6 +390,10 @@ const TEMPLATE = `<span class="tws-talk-name"></span></div><div class="tws-talk-header-actions">+ <label class="tws-talk-lang-wrap" title="Language you speak. The avatar answers in it, in its own voice">+ <span class="tws-talk-lang-icon" aria-hidden="true">🌐</span>+ <select class="tws-talk-lang" aria-label="Conversation language"></select>+ </label><button class="tws-talk-frame" type="button" title="Cycle framing: half → headshot → full" aria-label="Cycle camera framing"><span aria-hidden="true">⛶</span><span class="tws-talk-frame-label">Half body</span>@@ -446,6 +500,33 @@ function appendTranscript(el, msg) {el.scrollTop = el.scrollHeight;}+function hideError(el) {+ if (!el) return;+ el.hidden = true;+ el.textContent = '';+}++// Conversation-language preference. Per browser, not per avatar: someone who+// talks to their agents in Mandarin does so with all of them, and re-picking the+// language on every avatar page would be the worst part of the feature.+const LANG_STORAGE_KEY = 'tws_talk_language';++function readStoredTalkLanguage() {+ try {+ return localStorage.getItem(LANG_STORAGE_KEY) || '';+ } catch {+ return ''; // private mode / storage blocked, so fall back to detection+ }+}++function storeTalkLanguage(tag) {+ try {+ localStorage.setItem(LANG_STORAGE_KEY, tag);+ } catch {+ // Storage blocked: the choice still applies for this session.+ }+}+function showError(el, message) {if (!el) return;el.textContent = message;@@ -522,6 +603,36 @@ const TALK_CSS = `gap: 8px;flex-shrink: 0;}+.tws-talk-lang-wrap {+ display: inline-flex;+ align-items: center;+ gap: 6px;+ background: rgba(255,255,255,0.06);+ border: 1px solid rgba(255,255,255,0.14);+ border-radius: 999px;+ padding: 5px 10px 5px 12px;+ transition: background 0.15s, border-color 0.15s;+ flex-shrink: 0;+}+.tws-talk-lang-wrap:hover, .tws-talk-lang-wrap:focus-within {+ background: rgba(255,255,255,0.10);+ border-color: rgba(255,255,255,0.25);+}+.tws-talk-lang-icon { font-size: 12px; line-height: 1; }+.tws-talk-lang {+ appearance: none;+ background: transparent;+ border: 0;+ color: #fafafa;+ font-family: inherit;+ font-size: 12px;+ font-weight: 600;+ padding: 2px 2px;+ cursor: pointer;+ max-width: 150px;+}+.tws-talk-lang:focus-visible { outline: 2px solid #7c5cff; outline-offset: 3px; border-radius: 4px; }+.tws-talk-lang option { background: #131316; color: #fafafa; }.tws-talk-clone, .tws-talk-frame {background: rgba(255,255,255,0.06);border: 1px solid rgba(255,255,255,0.14);