Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(voice): route non-English speech to the multilingual ASR model

nirholasbc7b0cbd56b6
diff --git a/api/_lib/asr-nvidia.js b/api/_lib/asr-nvidia.js
index fcc385357..fd7669af6 100644
--- a/api/_lib/asr-nvidia.js
+++ b/api/_lib/asr-nvidia.js
@@ -53,7 +53,73 @@ export function resolveAsrEncoding(encoding) {
}
export function nvidiaAsrConfigured() {
- return Boolean(env.NVIDIA_API_KEY && env.NVIDIA_ASR_FUNCTION_ID);
+ return Boolean(
+ env.NVIDIA_API_KEY &&
+ (env.NVIDIA_ASR_FUNCTION_ID || env.NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL),
+ );
+}
+
+// ── Language routing ────────────────────────────────────────────────────────
+// One hosted function does not cover every language. The English-tuned Parakeet
+// models reject a non-`en` language_code outright (gRPC 3 INVALID_ARGUMENT), so
+// a Mandarin utterance sent to the English pin fails rather than degrading. A
+// multilingual model (ai-canary-1b-asr on our account) transcribes the languages
+// below, and English at the same quality as the English pin. We therefore keep
+// the English pin for `en*` (better casing and punctuation on English) and route
+// every other language to the multilingual id when one is configured.
+//
+// The list is empirical, not aspirational: every tag here was verified by
+// synthesizing a sentence in that language and reading back a correct
+// transcript (`node scripts/verify-nvidia-asr.mjs --languages`). Languages the
+// model rejects (Arabic, Malay, Tamil, Filipino, Norwegian) or garbles
+// (Indonesian, Vietnamese, Finnish, Danish, Greek, Hungarian) are deliberately
+// absent, so the client never offers a language the lane cannot hear.
+export const MULTILINGUAL_ASR_LANGUAGES = [
+ 'zh-CN',
+ 'zh-TW',
+ 'ja-JP',
+ 'ko-KR',
+ 'es-ES',
+ 'fr-FR',
+ 'de-DE',
+ 'it-IT',
+ 'pt-BR',
+ 'ru-RU',
+ 'hi-IN',
+ 'nl-NL',
+ 'pl-PL',
+ 'cs-CZ',
+];
+
+/** True when `language` is an English tag (the English pin's territory). */
+function isEnglish(language) {
+ return String(language || '').trim().toLowerCase().startsWith('en');
+}
+
+/**
+ * NVCF function id to use for `language`, or null when nothing is configured.
+ * English (and any language with no multilingual id configured) stays on
+ * NVIDIA_ASR_FUNCTION_ID; everything else prefers the multilingual id.
+ */
+export function asrFunctionIdFor(language) {
+ const multilingual = env.NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL || null;
+ const primary = env.NVIDIA_ASR_FUNCTION_ID || null;
+ if (!isEnglish(language) && multilingual) return multilingual;
+ return primary || multilingual;
+}
+
+/**
+ * BCP-47 tags this deployment can actually recognize, for the capability probe.
+ * Always includes en-US (the base lane); the multilingual tags appear only when
+ * a multilingual function id is configured.
+ */
+export function asrLanguages() {
+ const langs = env.NVIDIA_ASR_FUNCTION_ID ? ['en-US', 'en-GB'] : [];
+ if (env.NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL) {
+ if (!langs.length) langs.push('en-US');
+ langs.push(...MULTILINGUAL_ASR_LANGUAGES);
+ }
+ return langs;
}
// Parse a RIFF/WAVE header → { sampleRateHz, channels, pcm } so a WAV upload can
@@ -167,9 +233,10 @@ export async function transcribeNvidiaAsr({
model = '',
timeoutMs = DEFAULT_TIMEOUT_MS,
apiKey,
+ functionId: functionIdOverride,
} = {}) {
const key = apiKey || env.NVIDIA_API_KEY;
- const functionId = env.NVIDIA_ASR_FUNCTION_ID;
+ const functionId = functionIdOverride || asrFunctionIdFor(language);
if (!key || !functionId) {
const err = new Error(
!key ? 'NVIDIA_API_KEY not set' : 'NVIDIA_ASR_FUNCTION_ID not set (run scripts/verify-nvidia-asr.mjs --list to discover it)',
diff --git a/api/_lib/env.js b/api/_lib/env.js
index 43243c866..8ec199036 100644
--- a/api/_lib/env.js
+++ b/api/_lib/env.js
@@ -666,6 +666,18 @@ export const env = {
return opt('NVIDIA_ASR_FUNCTION_ID');
},
+ // NVCF function id for a MULTILINGUAL hosted ASR model, used when a caller asks
+ // for a non-English language (api/_lib/asr-nvidia.js). The English-tuned
+ // Parakeet functions reject a non-`en` language_code outright (gRPC 3
+ // INVALID_ARGUMENT), so without this id the free lane is English-only and a
+ // Mandarin/Japanese/Spanish speaker falls back to the browser recognizer.
+ // Discover the live id with `node scripts/verify-nvidia-asr.mjs --list` and
+ // pick a multilingual model (ai-canary-1b-asr on our account). Optional: when
+ // unset, non-English requests keep using NVIDIA_ASR_FUNCTION_ID.
+ get NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL() {
+ return opt('NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL');
+ },
+
// NVCF function id for the hosted Audio2Face-3D model used by the free facial-
// animation lane (api/_lib/a2f-nvidia.js, api/a2f.js) that turns Magpie speech
// into a per-frame ARKit blendshape track. Unlike the ASR id this is OPTIONAL:
diff --git a/api/asr.js b/api/asr.js
index 1f71738bb..465e482c4 100644
--- a/api/asr.js
+++ b/api/asr.js
@@ -17,7 +17,8 @@
// audio/flac → FLAC
// audio/ogg → Ogg/Opus
// or a JSON body { audio: <base64>, format?, language?, sampleRate?, words? }.
-// Query/JSON params: language (BCP-47, default en-US), rate (PCM sample rate),
+// Query/JSON params: language (BCP-47, default en-US; a non-English tag routes
+// to the multilingual model, see asrFunctionIdFor), rate (PCM sample rate),
// words ("1" for word-level timestamps), model (override Riva model name).
//
// Response: { text, confidence, language, model, durationSec, words? }.
@@ -30,6 +31,7 @@ import {
nvidiaAsrConfigured,
resolveAsrEncoding,
parseWav,
+ asrLanguages,
} from './_lib/asr-nvidia.js';
export const maxDuration = 30;
@@ -86,6 +88,10 @@ export default wrap(async function handler(req, res) {
// 16 kHz mono is the Riva acoustic-model rate and the rate the client
// downsamples its capture to — surface it so the two never drift.
sampleRate: 16000,
+ // BCP-47 tags this deployment can recognize. A client builds its
+ // language picker from this rather than guessing: offering a language
+ // the lane rejects is worse than not offering it at all.
+ languages: asrLanguages(),
},
{ 'cache-control': 'public, max-age=60' },
);
@@ -201,6 +207,17 @@ export default wrap(async function handler(req, res) {
code === 'invalid_argument' ? 400 :
code === 'not_configured' ? 503 :
502;
+ // An `invalid_argument` on a non-English language is nearly always the
+ // hosted model refusing that language_code, which reads as a generic
+ // provider failure unless we say so. Name the language and the way out.
+ if (code === 'invalid_argument' && !String(language).toLowerCase().startsWith('en')) {
+ return error(
+ res,
+ 400,
+ 'unsupported_language',
+ `The speech-to-text lane could not recognize "${language}". Supported: ${asrLanguages().join(', ') || 'none'}.`,
+ );
+ }
return error(res, status, code, `Speech recognition failed: ${e?.message || 'unknown error'}`);
}
});
diff --git a/src/voice/talk-controller.js b/src/voice/talk-controller.js
index a01a14403..81a6cfcff 100644
--- a/src/voice/talk-controller.js
+++ b/src/voice/talk-controller.js
@@ -27,6 +27,12 @@
import { LipsyncDriver, tapAudioElement } from './lipsync-driver.js';
import { MicCapture } from './mic-capture.js';
+import {
+ detectTalkLanguage,
+ edgeVoiceFor,
+ languageInstruction,
+ resolveTalkLanguage,
+} from './talk-languages.js';
import { log } from '../shared/log.js';
// Riva interim recognition cadence. While holding to talk we fire a recognition
@@ -36,12 +42,6 @@ import { log } from '../shared/log.js';
const INTERIM_INTERVAL_MS = 1400;
const MAX_INTERIMS = 4;
-const EDGE_VOICES_BY_GENDER = {
- female: 'en-US-AriaNeural',
- male: 'en-US-GuyNeural',
- neutral: 'en-US-AriaNeural',
-};
-
const ELEVEN_DEFAULT_VOICE = 'EXAVITQu4vr4xnSDxMaL'; // Bella — ElevenLabs default voice
export class TalkController {
@@ -56,7 +56,7 @@ export class TalkController {
* @param {(err: Error) => void} [opts.onError]
* @param {{ attach: Function, setMouthShape: Function }} opts.mouthTarget
*/
- constructor({ avatar, systemPromptFn, onMessage, onStateChange, onInterim, onError, mouthTarget, commandInterceptor }) {
+ constructor({ avatar, systemPromptFn, onMessage, onStateChange, onInterim, onError, mouthTarget, commandInterceptor, language }) {
if (!avatar?.id) throw new Error('TalkController: avatar.id required');
if (!mouthTarget) throw new Error('TalkController: mouthTarget required');
this.avatar = avatar;
@@ -82,24 +82,69 @@ export class TalkController {
// Speech-to-text routing. 'riva' = server-side NVIDIA Riva (cross-browser),
// 'browser' = window.SpeechRecognition (Chrome/Edge/Safari), 'none' = text
- // only. Resolved once by prepare(); language tracks the avatar's locale.
- this._sttMode = null;
- this._sttModePromise = null;
+ // only. The right path depends on the CONVERSATION LANGUAGE, not just the
+ // browser: the server lane recognizes the languages it advertises on
+ // /api/asr, and the browser recognizer covers the rest. prepare() probes
+ // once; the mode is then derived per language.
+ this._probePromise = null;
+ this._probeSettledFlag = false;
+ this._serverAsr = null; // { configured, languages } from GET /api/asr
+ this._hasBrowserSR = !!(
+ typeof window !== 'undefined' &&
+ (window.SpeechRecognition || window.webkitSpeechRecognition)
+ );
this._listenMode = null; // mode of the in-flight turn
this._mic = null;
this._interimTimer = null;
this._interimBusy = false;
this._interimCount = 0;
- this.language = 'en-US';
+
+ // Conversation language: an explicit choice wins, else the language the
+ // site is already displayed in, else the browser's own preference. Every
+ // leg of the loop reads it: recognition, the reply, and the voice.
+ this.language = resolveTalkLanguage(
+ language ||
+ detectTalkLanguage({
+ uiLocale: siteLocale(),
+ navLangs:
+ typeof navigator !== 'undefined'
+ ? navigator.languages || [navigator.language]
+ : [],
+ }),
+ );
+ }
+
+ /**
+ * Switch the conversation language mid-session. Takes effect on the next turn:
+ * recognition, the reply, and the voice all follow it.
+ * @returns {string} the resolved tag (never an unsupported one).
+ */
+ setLanguage(tag) {
+ this.language = resolveTalkLanguage(tag);
+ return this.language;
}
get state() {
return this._state;
}
- /** Resolved STT path: 'riva' | 'browser' | 'none' (null until prepare()). */
+ /** STT path for the CURRENT language: 'riva' | 'browser' | 'none'. */
get sttMode() {
- return this._sttMode;
+ return this._modeFor(this.language);
+ }
+
+ /** True when the free server lane can hear the current language. */
+ get serverHearsLanguage() {
+ return this._serverSupports(this.language);
+ }
+
+ /**
+ * Recognizer a given language would use right now: 'riva' | 'browser' | 'none'.
+ * A picker calls this to label each option honestly before the user commits to
+ * one, instead of discovering the dead end by holding the mic.
+ */
+ sttModeFor(language) {
+ return this._modeFor(resolveTalkLanguage(language));
}
/** Live mic level (0..1) while the Riva lane is capturing; 0 otherwise. */
@@ -108,32 +153,61 @@ export class TalkController {
}
/**
- * Decide the STT path once, before the first turn. Probes /api/asr for the
- * free NVIDIA Riva lane (works in every browser, including Firefox); falls
- * back to the browser's own SpeechRecognition, then to text-only. Safe to
- * call repeatedly — the probe runs at most once. Returns the resolved mode.
+ * Decide the STT path before the first turn. Probes /api/asr once for the free
+ * NVIDIA Riva lane (works in every browser, including Firefox) and the set of
+ * languages that lane recognizes; falls back to the browser's own
+ * SpeechRecognition, then to text-only. Safe to call repeatedly; the probe
+ * runs at most once. Returns the mode for the current language.
*/
async prepare() {
- if (this._sttMode) return this._sttMode;
- if (!this._sttModePromise) {
- this._sttModePromise = (async () => {
- const hasBrowserSR = !!(window.SpeechRecognition || window.webkitSpeechRecognition);
- const canCapture = MicCapture.isSupported();
- if (canCapture) {
- try {
- const r = await fetch('/api/asr', { headers: { accept: 'application/json' } });
- if (r.ok) {
- const j = await r.json();
- if (j?.configured) return (this._sttMode = 'riva');
- }
- } catch {
- // Probe failure is not fatal — fall back to whatever the browser offers.
- }
+ if (!this._probePromise) {
+ this._probePromise = (async () => {
+ if (!MicCapture.isSupported()) return;
+ try {
+ const r = await fetch('/api/asr', { headers: { accept: 'application/json' } });
+ if (!r.ok) return;
+ const j = await r.json();
+ if (!j?.configured) return;
+ this._serverAsr = {
+ configured: true,
+ // A deployment that predates language routing advertises no list;
+ // treat that as the English-only lane it is rather than assuming
+ // it can hear Mandarin and failing the first turn.
+ languages: Array.isArray(j.languages) && j.languages.length
+ ? j.languages.map((t) => String(t).toLowerCase())
+ : ['en-us'],
+ };
+ } catch {
+ // Probe failure is not fatal: fall back to whatever the browser offers.
+ } finally {
+ this._probeSettledFlag = true;
}
- return (this._sttMode = hasBrowserSR ? 'browser' : 'none');
})();
}
- return this._sttModePromise;
+ await this._probePromise;
+ return this.sttMode;
+ }
+
+ /** Does the probed server lane advertise recognition for this language? */
+ _serverSupports(language) {
+ if (!this._serverAsr?.configured) return false;
+ const tag = String(language || '').toLowerCase();
+ const primary = tag.split('-')[0];
+ return this._serverAsr.languages.some(
+ (l) => l === tag || l.split('-')[0] === primary,
+ );
+ }
+
+ /**
+ * Best recognizer for a language. The server lane leads where it can hear the
+ * language (cross-browser, and it does not ship the user's audio to Google);
+ * the browser's recognizer covers everything else, which is what makes a
+ * language the server lane never learned still usable in Chrome and Safari.
+ */
+ _modeFor(language) {
+ if (this._serverSupports(language)) return 'riva';
+ if (this._hasBrowserSR) return 'browser';
+ return 'none';
}
/**
@@ -145,7 +219,7 @@ export class TalkController {
startListening() {
if (this._state !== 'idle') return false;
- const mode = this._sttMode || this._fallbackSttMode();
+ const mode = this.probeSettled ? this.sttMode : this._fallbackSttMode();
if (mode === 'riva') {
this._listenMode = 'riva';
this._startRivaListening();
@@ -180,10 +254,19 @@ export class TalkController {
// recognizer (zero setup, instant) when present; otherwise attempt Riva if the
// environment can capture audio at all.
_fallbackSttMode() {
- if (window.SpeechRecognition || window.webkitSpeechRecognition) return 'browser';
+ if (this._hasBrowserSR) return 'browser';
return MicCapture.isSupported() ? 'riva' : 'none';
}
+ /**
+ * True once the /api/asr probe has answered (or failed). Until it has, the
+ * language→recognizer mapping is a guess, so a UI should not tell the user a
+ * language is unusable yet.
+ */
+ get probeSettled() {
+ return this._probeSettledFlag;
+ }
+
// ── Browser SpeechRecognition path (Chrome/Edge/Safari) ────────────────
_startBrowserListening() {
const RecCls = window.SpeechRecognition || window.webkitSpeechRecognition;
@@ -460,6 +543,14 @@ export class TalkController {
}
}
+ // The agent's own persona plus, for a non-English conversation, the one line
+ // that makes the reply come back in the language the user is speaking.
+ _systemPrompt() {
+ return [this.systemPromptFn(), languageInstruction(this.language)]
+ .filter(Boolean)
+ .join('\n\n');
+ }
+
async _streamChat(message) {
const isUuid =
typeof this.avatar.id === 'string' &&
@@ -471,7 +562,7 @@ export class TalkController {
credentials: 'include',
body: JSON.stringify({
message,
- system_prompt: this.systemPromptFn(),
+ system_prompt: this._systemPrompt(),
history: this._history.slice(-10, -1),
...(isUuid ? { agentId: this.avatar.id } : {}),
...(this.avatar.agent_id ? { agentId: this.avatar.agent_id } : {}),
@@ -634,7 +725,10 @@ export class TalkController {
this.avatar?.source_meta?.gender ||
this.avatar?.source_meta?.bodyType ||
'neutral';
- const voice = EDGE_VOICES_BY_GENDER[gender] || EDGE_VOICES_BY_GENDER.neutral;
+ // Language first, body type second: an English voice reading Mandarin is
+ // unintelligible, while a female voice reading a male avatar's line is
+ // merely a mismatch.
+ const voice = edgeVoiceFor(this.language, gender);
const r = await fetch('/api/tts/edge', {
method: 'POST',
headers: { 'content-type': 'application/json' },
@@ -652,6 +746,19 @@ export class TalkController {
}
}
+// The language the site UI is already displayed in, when the i18n runtime is on
+// the page. Read through the global instead of importing src/i18n.js: talk mode
+// also runs inside embeds that never boot the translation runtime, and a hard
+// import would pull the whole catalogue loader into those bundles.
+function siteLocale() {
+ if (typeof window === 'undefined') return '';
+ try {
+ return window.threewsI18n?.getLocale?.() || document.documentElement?.lang || '';
+ } catch {
+ return '';
+ }
+}
+
// Error carrying a machine-readable `.code` so the host UI can route each
// failure precisely (mic denial → text input, rate limit → "try later", etc).
function coded(message, code) {
diff --git a/src/voice/talk-languages.js b/src/voice/talk-languages.js
new file mode 100644
index 000000000..1007de3c3
--- /dev/null
+++ b/src/voice/talk-languages.js
@@ -0,0 +1,107 @@
+/**
+ * Languages the avatar voice loop can hold a conversation in.
+ *
+ * One turn crosses three systems, and a language only counts as supported when
+ * all three handle it:
+ *
+ * 1. speech in: server-side Riva ASR (/api/asr, multilingual model) or the
+ * browser's own SpeechRecognition
+ * 2. the reply: /api/chat, told which language to answer in
+ * 3. speech out: ElevenLabs (a cloned voice speaks every language below via
+ * the multilingual model) with Microsoft Edge Neural TTS as
+ * the fallback when no voice is cloned
+ *
+ * `asr: true` marks the tags the free server lane transcribes accurately, each
+ * verified against real synthesized speech (`node scripts/verify-nvidia-asr.mjs
+ * --languages`). The rest still work when the browser's recognizer speaks them,
+ * so they stay offerable in Chrome/Edge/Safari rather than being hidden.
+ *
+ * `edge` is the Microsoft Neural voice used when the avatar has no cloned voice:
+ * without it a Mandarin reply comes out of an American English voice reading
+ * pinyin-shaped noise. Female/male pairs mirror the avatar's own body type.
+ */
+
+export const TALK_LANGUAGES = [
+ { tag: 'en-US', native: 'English (US)', english: 'English (US)', asr: true, edge: { female: 'en-US-AriaNeural', male: 'en-US-GuyNeural' } },
+ { tag: 'en-GB', native: 'English (UK)', english: 'English (UK)', asr: true, edge: { female: 'en-GB-SoniaNeural', male: 'en-GB-RyanNeural' } },
+ { tag: 'zh-CN', native: '简体中文', english: 'Chinese (Mandarin)', asr: true, edge: { female: 'zh-CN-XiaoxiaoNeural', male: 'zh-CN-YunxiNeural' } },
+ { tag: 'zh-TW', native: '繁體中文', english: 'Chinese (Taiwan)', asr: true, edge: { female: 'zh-TW-HsiaoChenNeural', male: 'zh-TW-YunJheNeural' } },
+ { tag: 'ja-JP', native: '日本語', english: 'Japanese', asr: true, edge: { female: 'ja-JP-NanamiNeural', male: 'ja-JP-KeitaNeural' } },
+ { tag: 'ko-KR', native: '한국어', english: 'Korean', asr: true, edge: { female: 'ko-KR-SunHiNeural', male: 'ko-KR-InJoonNeural' } },
+ { tag: 'es-ES', native: 'Español', english: 'Spanish', asr: true, edge: { female: 'es-ES-ElviraNeural', male: 'es-ES-AlvaroNeural' } },
+ { tag: 'fr-FR', native: 'Français', english: 'French', asr: true, edge: { female: 'fr-FR-DeniseNeural', male: 'fr-FR-HenriNeural' } },
+ { tag: 'de-DE', native: 'Deutsch', english: 'German', asr: true, edge: { female: 'de-DE-KatjaNeural', male: 'de-DE-ConradNeural' } },
+ { tag: 'it-IT', native: 'Italiano', english: 'Italian', asr: true, edge: { female: 'it-IT-ElsaNeural', male: 'it-IT-DiegoNeural' } },
+ { tag: 'pt-BR', native: 'Português (BR)', english: 'Portuguese (Brazil)', asr: true, edge: { female: 'pt-BR-FranciscaNeural', male: 'pt-BR-AntonioNeural' } },
+ { tag: 'ru-RU', native: 'Русский', english: 'Russian', asr: true, edge: { female: 'ru-RU-SvetlanaNeural', male: 'ru-RU-DmitryNeural' } },
+ { tag: 'hi-IN', native: 'हिन्दी', english: 'Hindi', asr: true, edge: { female: 'hi-IN-SwaraNeural', male: 'hi-IN-MadhurNeural' } },
+ { tag: 'nl-NL', native: 'Nederlands', english: 'Dutch', asr: true, edge: { female: 'nl-NL-ColetteNeural', male: 'nl-NL-MaartenNeural' } },
+ { tag: 'pl-PL', native: 'Polski', english: 'Polish', asr: true, edge: { female: 'pl-PL-ZofiaNeural', male: 'pl-PL-MarekNeural' } },
+ { tag: 'cs-CZ', native: 'Čeština', english: 'Czech', asr: true, edge: { female: 'cs-CZ-VlastaNeural', male: 'cs-CZ-AntoninNeural' } },
+ { tag: 'ar-SA', native: 'العربية', english: 'Arabic', asr: false, edge: { female: 'ar-SA-ZariyahNeural', male: 'ar-SA-HamedNeural' } },
+ { tag: 'tr-TR', native: 'Türkçe', english: 'Turkish', asr: false, edge: { female: 'tr-TR-EmelNeural', male: 'tr-TR-AhmetNeural' } },
+ { tag: 'id-ID', native: 'Bahasa Indonesia', english: 'Indonesian', asr: false, edge: { female: 'id-ID-GadisNeural', male: 'id-ID-ArdiNeural' } },
+ { tag: 'sv-SE', native: 'Svenska', english: 'Swedish', asr: false, edge: { female: 'sv-SE-SofieNeural', male: 'sv-SE-MattiasNeural' } },
+ { tag: 'uk-UA', native: 'Українська', english: 'Ukrainian', asr: false, edge: { female: 'uk-UA-PolinaNeural', male: 'uk-UA-OstapNeural' } },
+ { tag: 'ro-RO', native: 'Română', english: 'Romanian', asr: false, edge: { female: 'ro-RO-AlinaNeural', male: 'ro-RO-EmilNeural' } },
+];
+
+export const DEFAULT_TALK_LANGUAGE = 'en-US';
+
+const BY_TAG = new Map(TALK_LANGUAGES.map((l) => [l.tag.toLowerCase(), l]));
+
+/** The language entry for a tag, or null when we do not carry that language. */
+export function talkLanguage(tag) {
+ const key = String(tag || '').trim().toLowerCase();
+ if (!key) return null;
+ if (BY_TAG.has(key)) return BY_TAG.get(key);
+ // "zh", "zh-Hans-CN", "pt" → the first entry sharing the primary subtag, so a
+ // browser locale we do not list exactly still lands on the right language.
+ const primary = key.split('-')[0];
+ return TALK_LANGUAGES.find((l) => l.tag.toLowerCase().split('-')[0] === primary) || null;
+}
+
+/**
+ * Normalize any locale-ish input to a tag we support.
+ * @returns {string} a supported BCP-47 tag; DEFAULT_TALK_LANGUAGE when unknown.
+ */
+export function resolveTalkLanguage(tag) {
+ return talkLanguage(tag)?.tag || DEFAULT_TALK_LANGUAGE;
+}
+
+/**
+ * Pick the conversation language for a first visit, in preference order:
+ * an explicit stored choice, then the language the site UI is already in, then
+ * the browser's own languages. Pure, so the ordering is unit-testable.
+ *
+ * @param {{ stored?: string, uiLocale?: string, navLangs?: string[] }} input
+ */
+export function detectTalkLanguage({ stored, uiLocale, navLangs } = {}) {
+ const candidates = [stored, uiLocale, ...(navLangs || [])];
+ for (const c of candidates) {
+ const hit = talkLanguage(c);
+ if (hit) return hit.tag;
+ }
+ return DEFAULT_TALK_LANGUAGE;
+}
+
+/**
+ * Microsoft Edge Neural voice for a language, matched to the avatar's body type.
+ * Falls back to the language's female voice, then to US English.
+ */
+export function edgeVoiceFor(tag, gender = 'neutral') {
+ const entry = talkLanguage(tag) || talkLanguage(DEFAULT_TALK_LANGUAGE);
+ const key = gender === 'male' ? 'male' : 'female';
+ return entry.edge[key] || entry.edge.female;
+}
+
+/**
+ * The instruction appended to the agent's system prompt so the reply comes back
+ * in the conversation language. English needs no instruction: it is what every
+ * agent persona already writes in, and a redundant line only dilutes the prompt.
+ */
+export function languageInstruction(tag) {
+ const entry = talkLanguage(tag);
+ if (!entry || entry.tag.startsWith('en')) return '';
+ return `Reply only in ${entry.english} (${entry.native}). The user is speaking ${entry.english}; answer naturally in that language, never in English, and never translate your own reply. Keep replies short enough to be spoken aloud.`;
+}
diff --git a/src/voice/talk-mode.js b/src/voice/talk-mode.js
index 69d2fb796..3d56397ac 100644
--- a/src/voice/talk-mode.js
+++ b/src/voice/talk-mode.js
@@ -19,6 +19,7 @@ import { TalkScene } from './talk-scene.js';
import { AvatarMouthTarget } from './avatar-morph-target.js';
import { TalkController } from './talk-controller.js';
import { openVoiceCloneModal } from './voice-clone-modal.js';
+import { TALK_LANGUAGES, detectTalkLanguage, talkLanguage } from './talk-languages.js';
import { nextPreset, PRESET_LABELS } from './camera-presets.js';
import { WalletIntentController } from './wallet-intent.js';
import { log } from '../shared/log.js';
@@ -67,6 +68,7 @@ export function openTalkMode({ avatar, systemPromptFn }) {
const textbar = overlay.querySelector('.tws-talk-textbar');
const textInput = overlay.querySelector('.tws-talk-input');
const walletHintBtn = overlay.querySelector('.tws-talk-wallet-hint');
+ const langSel = overlay.querySelector('.tws-talk-lang');
nameEl.textContent = avatar.name || 'Avatar';
// Owner-only affordance: `owner_id` is stripped from the API response for
@@ -129,6 +131,7 @@ export function openTalkMode({ avatar, systemPromptFn }) {
controller = new TalkController({
avatar,
systemPromptFn,
+ language: readStoredTalkLanguage(),
mouthTarget,
onMessage: (m) => appendTranscript(transcriptEl, m),
onInterim: (t) => setPartial(partialEl, t),
@@ -156,18 +159,65 @@ export function openTalkMode({ avatar, systemPromptFn }) {
}
}
- // Resolve the speech-to-text path up front (probes the free Riva lane). When no
- // recognizer exists at all, retire the mic button and lead with text input.
- controller.prepare().then((mode) => {
+ // ── Conversation language ──────────────────────────────────────────
+ // The picker drives all three legs of the loop: what the recognizer listens
+ // for, what language the agent answers in, and which voice speaks the reply
+ // (a cloned voice speaks every language here; the fallback voice switches to
+ // a native one). Options are labelled with the recognizer each language will
+ // actually get, so a "type only" language is visible before it is chosen.
+ function renderLanguageOptions() {
+ if (!langSel) return;
+ const current = controller.language;
+ langSel.innerHTML = TALK_LANGUAGES.map((l) => {
+ const mode = controller.sttModeFor(l.tag);
+ const suffix = mode === 'none' ? ' (type only)' : '';
+ const label = l.native === l.english ? l.native : `${l.native} · ${l.english}`;
+ return `<option value="${l.tag}"${l.tag === current ? ' selected' : ''}>${label}${suffix}</option>`;
+ }).join('');
+ langSel.value = current;
+ }
+
+ // Reflect the recognizer available for the chosen language: no recognizer at
+ // all retires the mic button and leads with the text input rather than leaving
+ // a button that can only fail.
+ function applySttMode() {
+ const mode = controller.sttMode;
+ const name = talkLanguage(controller.language)?.english || 'this language';
+ // Before the probe answers, the mapping is a guess, so never retire the mic
+ // button or accuse a language of being unusable on a guess.
+ if (!controller.probeSettled) {
+ holdBtn.title = `Hold to talk in ${name}`;
+ return;
+ }
+ holdBtn.hidden = mode === 'none';
if (mode === 'none') {
- holdBtn.hidden = true;
textInput?.focus();
+ showError(errEl, `Speech input isn't available for ${name} in this browser. Type instead and the avatar still answers out loud.`);
} else {
holdBtn.title =
mode === 'riva'
- ? 'Hold to talk — powered by NVIDIA Riva speech-to-text'
- : 'Hold to talk';
+ ? `Hold to talk in ${name}, powered by NVIDIA Riva speech-to-text`
+ : `Hold to talk in ${name}`;
}
+ }
+
+ renderLanguageOptions();
+ applySttMode();
+
+ langSel?.addEventListener('change', () => {
+ const tag = controller.setLanguage(langSel.value);
+ storeTalkLanguage(tag);
+ hideError(errEl);
+ renderLanguageOptions();
+ applySttMode();
+ });
+
+ // Resolve the speech-to-text path up front (probes the free Riva lane): the
+ // probe tells us which languages the free server lane can hear, so the labels
+ // and the mic button both settle once it answers.
+ controller.prepare().then(() => {
+ renderLanguageOptions();
+ applySttMode();
});
// ── Conversational Wallet (owner-only) ──────────────────────────────
@@ -340,6 +390,10 @@ const TEMPLATE = `
<span class="tws-talk-name"></span>
</div>
<div class="tws-talk-header-actions">
+ <label class="tws-talk-lang-wrap" title="Language you speak. The avatar answers in it, in its own voice">
+ <span class="tws-talk-lang-icon" aria-hidden="true">🌐</span>
+ <select class="tws-talk-lang" aria-label="Conversation language"></select>
+ </label>
<button class="tws-talk-frame" type="button" title="Cycle framing: half → headshot → full" aria-label="Cycle camera framing">
<span aria-hidden="true">⛶</span>
<span class="tws-talk-frame-label">Half body</span>
@@ -446,6 +500,33 @@ function appendTranscript(el, msg) {
el.scrollTop = el.scrollHeight;
}
+function hideError(el) {
+ if (!el) return;
+ el.hidden = true;
+ el.textContent = '';
+}
+
+// Conversation-language preference. Per browser, not per avatar: someone who
+// talks to their agents in Mandarin does so with all of them, and re-picking the
+// language on every avatar page would be the worst part of the feature.
+const LANG_STORAGE_KEY = 'tws_talk_language';
+
+function readStoredTalkLanguage() {
+ try {
+ return localStorage.getItem(LANG_STORAGE_KEY) || '';
+ } catch {
+ return ''; // private mode / storage blocked, so fall back to detection
+ }
+}
+
+function storeTalkLanguage(tag) {
+ try {
+ localStorage.setItem(LANG_STORAGE_KEY, tag);
+ } catch {
+ // Storage blocked: the choice still applies for this session.
+ }
+}
+
function showError(el, message) {
if (!el) return;
el.textContent = message;
@@ -522,6 +603,36 @@ const TALK_CSS = `
gap: 8px;
flex-shrink: 0;
}
+.tws-talk-lang-wrap {
+ display: inline-flex;
+ align-items: center;
+ gap: 6px;
+ background: rgba(255,255,255,0.06);
+ border: 1px solid rgba(255,255,255,0.14);
+ border-radius: 999px;
+ padding: 5px 10px 5px 12px;
+ transition: background 0.15s, border-color 0.15s;
+ flex-shrink: 0;
+}
+.tws-talk-lang-wrap:hover, .tws-talk-lang-wrap:focus-within {
+ background: rgba(255,255,255,0.10);
+ border-color: rgba(255,255,255,0.25);
+}
+.tws-talk-lang-icon { font-size: 12px; line-height: 1; }
+.tws-talk-lang {
+ appearance: none;
+ background: transparent;
+ border: 0;
+ color: #fafafa;
+ font-family: inherit;
+ font-size: 12px;
+ font-weight: 600;
+ padding: 2px 2px;
+ cursor: pointer;
+ max-width: 150px;
+}
+.tws-talk-lang:focus-visible { outline: 2px solid #7c5cff; outline-offset: 3px; border-radius: 4px; }
+.tws-talk-lang option { background: #131316; color: #fafafa; }
.tws-talk-clone, .tws-talk-frame {
background: rgba(255,255,255,0.06);
border: 1px solid rgba(255,255,255,0.14);