Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(voice): run agent speech-to-text on-device with Whisper when the manifest asks for it
nirholas169d6a4c449b
diff --git a/src/element.js b/src/element.jsindex 9f96c6014..211548aba 100644--- a/src/element.js+++ b/src/element.js@@ -1283,20 +1283,7 @@ class Agent3DElement extends HTMLElement {});row.appendChild(input);- if (this._speechInputAvailable()) {- const mic = document.createElement('button');- mic.type = 'button';- mic.className = 'icon mic';- mic.title = 'Push to talk';- mic.setAttribute('aria-label', 'Push to talk');- mic.innerHTML =- '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">' +- '<rect x="9" y="3" width="6" height="11" rx="3"></rect>' +- '<path d="M5 11a7 7 0 0 0 14 0"></path><path d="M12 18v3"></path></svg>';- mic.addEventListener('click', () => this._toggleMic());- row.appendChild(mic);- this._micEl = mic;- }+ if (this._speechInputAvailable()) row.appendChild(this._createMicButton());row.appendChild(send);this._inputEl = input;return row;@@ -1309,6 +1296,57 @@ class Agent3DElement extends HTMLElement {return name && name !== 'Agent' ? `Message ${name}…` : 'Say something...';}+ _createMicButton() {+ const mic = document.createElement('button');+ mic.type = 'button';+ mic.className = 'icon mic';+ mic.title = 'Push to talk';+ mic.setAttribute('aria-label', 'Push to talk');+ mic.innerHTML =+ '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">' ++ '<rect x="9" y="3" width="6" height="11" rx="3"></rect>' ++ '<path d="M5 11a7 7 0 0 0 14 0"></path><path d="M12 18v3"></path></svg>';+ mic.addEventListener('click', () => this._toggleMic());+ this._micEl = mic;+ return mic;+ }++ // The composer is built before the manifest is known, so an agent whose STT+ // runs on-device (manifest voice.stt.provider "whisper") gets its mic here,+ // once the runtime exists. That is what gives Firefox and Safari visitors push+ // to talk: they have no browser recognizer for _speechInputAvailable to find.+ _ensureOnDeviceMic() {+ if (this._micEl || !this._inputEl?.parentElement) return;+ if (typeof this._runtime?.stt?.warm !== 'function') return;+ this._inputEl.after(this._createMicButton());+ }++ // Reflect what the recognizer is doing on the mic, so a first-use model+ // download or a slow decode reads as progress instead of a frozen button.+ _renderSttStatus(status) {+ const mic = this._micEl;+ if (!mic) return;+ const phase = status?.phase;+ if (phase === 'loading-model') {+ const pct = Number.isFinite(status.progress) ? ` ${Math.round(status.progress * 100)}%` : '';+ mic.dataset.voiceState = 'thinking';+ mic.title = `Loading the on-device speech model${pct}`;+ } else if (phase === 'transcribing') {+ mic.dataset.voiceState = 'thinking';+ mic.title = 'Transcribing';+ } else if (phase === 'listening') {+ mic.dataset.voiceState = 'listening';+ const dl = Number.isFinite(status.modelProgress)+ ? `, speech model ${Math.round(status.modelProgress * 100)}% downloaded`+ : '';+ mic.title = `Listening${dl}. Click to stop`;+ } else {+ delete mic.dataset.voiceState;+ mic.title = 'Push to talk';+ }+ mic.setAttribute('aria-label', mic.title);+ }+// Push-to-talk works through a voice server or the browser's own speech// recognition. Where neither exists the mic would be a dead button._speechInputAvailable() {@@ -2016,6 +2054,7 @@ class Agent3DElement extends HTMLElement {agentId: _backendId || undefined,skillAccess: _skillAccess,});+ this._ensureOnDeviceMic();// ── Empathy + lipsync layer ───────────────────────────────────────// AgentAvatar subscribes to the agent-protocol bus, runs the emotion@@ -2088,6 +2127,7 @@ class Agent3DElement extends HTMLElement {'voice:speech-end','voice:transcript','voice:listen-start',+ 'voice:stt-status','memory:write',]) {this._runtime.addEventListener(ev, (e) => {@@ -2981,13 +3021,14 @@ class Agent3DElement extends HTMLElement {this._listening = true;this._micEl.dataset.listening = 'true';try {- const text = await this._runtime.listen();+ const text = await this._runtime.listen({ onStatus: (s) => this._renderSttStatus(s) });if (text) this.say(text, { voice: true });} catch (e) {log.warn('[agent-3d] listen failed', e);} finally {this._listening = false;this._micEl.dataset.listening = 'false';+ this._renderSttStatus(null);}}diff --git a/src/runtime/index.js b/src/runtime/index.jsindex af0a561a4..17ecddc8c 100644--- a/src/runtime/index.js+++ b/src/runtime/index.js@@ -53,7 +53,11 @@ export class Runtime extends EventTarget {...this.manifest.voice?.tts,...voiceConfig.tts,});- this.stt = createSTT({ ...this.manifest.voice?.stt, ...voiceConfig.stt });+ this.stt = createSTT({+ ...(this.apiOrigin ? { apiOrigin: this.apiOrigin } : {}),+ ...this.manifest.voice?.stt,+ ...voiceConfig.stt,+ });this.messages = [];this._busy = false;@@ -296,10 +300,16 @@ export class Runtime extends EventTarget {};}- async listen({ onInterim, onFinal } = {}) {+ async listen({ onInterim, onFinal, onStatus } = {}) {if (!this.stt) throw new Error('STT not configured');this.dispatchEvent(new CustomEvent('voice:listen-start', {}));const text = await this.stt.listen({+ // Recognizers that do real work before text exists (an on-device model+ // downloading, an utterance decoding) report it, so the UI can say so.+ onStatus: (status) => {+ this.dispatchEvent(new CustomEvent('voice:stt-status', { detail: status }));+ onStatus?.(status);+ },onInterim: (t) => {this.dispatchEvent(new CustomEvent('voice:transcript', { detail: { text: t, final: false } }),diff --git a/src/runtime/speech.js b/src/runtime/speech.jsindex 5c2a43695..4ffd23c4b 100644--- a/src/runtime/speech.js+++ b/src/runtime/speech.js@@ -2,7 +2,9 @@export { LiveKitVoice } from './livekit-voice.js';export { NeuralTTS } from './neural-tts.js';+export { WhisperSTT } from './whisper-stt.js';import { NeuralTTS } from './neural-tts.js';+import { WhisperSTT, onDeviceSupported } from './whisper-stt.js';import { startLipsync } from './lipsync.js';import { ACTION_TYPES } from '../agent-protocol.js';@@ -528,6 +530,9 @@ const TTS_FACTORIES = {const STT_FACTORIES = {none: () => null,browser: (cfg) => new BrowserSTT(cfg),+ // On-device recognition (src/runtime/whisper-stt.js). Where the browser cannot+ // capture audio or run WebAssembly, its own recognizer is the honest fallback.+ whisper: (cfg) => (onDeviceSupported() ? new WhisperSTT(cfg) : new BrowserSTT(cfg)),};export function createTTS(config = {}) {diff --git a/src/runtime/whisper-stt.js b/src/runtime/whisper-stt.jsnew file mode 100644index 000000000..0591f05a6--- /dev/null+++ b/src/runtime/whisper-stt.js@@ -0,0 +1,427 @@+/**+ * WhisperSTT: the `whisper` speech-to-text provider of the agent manifest.+ *+ * Speech is recognized on the visitor's own device. No audio leaves the browser+ * on the primary path, nothing is billed, and unlike `BrowserSTT` it works in+ * every browser with WebAssembly, Firefox and Safari included, where+ * window.SpeechRecognition is missing or ships the audio to a vendor.+ *+ * Pipeline:+ *+ * mic ─▶ silero VAD (src/voice/vad.js) ─▶ one utterance, 16 kHz Float32+ * ─▶ on-device transformers.js ASR ─▶ text+ * └▶ (on-device failed) POST /api/asr, the NVIDIA Riva lane ─▶ text+ *+ * Models (both MIT, ONNX q8, fetched once from the Hugging Face CDN and kept in+ * the browser Cache API, so the second visit loads from disk):+ *+ * English onnx-community/moonshine-base-ONNX about 63 MB+ * Other onnx-community/whisper-base about 77 MB, 99 languages+ *+ * Measured on the JFK reference clip (11 s, CPU, 4 threads): Moonshine 2.0 s and+ * Whisper 3.0 s per utterance, both transcribing it word-perfect.+ *+ * The model download starts the moment listening starts, so it overlaps the+ * user speaking instead of following it. Progress is reported through+ * `onStatus` so a caller can say what is happening on a first, slow load.+ *+ * Same contract as BrowserSTT: listen({ onInterim, onFinal }) resolves with the+ * final transcript, stop() ends listening and still transcribes whatever was+ * said. `continuous: true` keeps listening across utterances until stop(),+ * firing onFinal per utterance.+ */++import { log } from '../shared/log.js';+import { VoiceActivityDetector, VAD_ASSET_PATH, VAD_SAMPLE_RATE, float32ToWav } from '../voice/vad.js';++export const ON_DEVICE_MODELS = Object.freeze({+ english: 'onnx-community/moonshine-base-ONNX',+ multilingual: 'onnx-community/whisper-base',+});++/** Whisper reads a 30 s window; cut a little short so padding never truncates speech. */+export const MAX_UTTERANCE_SAMPLES = 28 * VAD_SAMPLE_RATE;++/**+ * Trailing silence that ends an utterance. The home voice loop runs at 352 ms+ * because hands-free turn-taking has to feel instant; push-to-talk does not, and+ * at 352 ms an ordinary mid-sentence pause ends the turn. The JFK reference clip+ * pauses 1.0 to 1.2 s after "my fellow Americans": 352 ms and 1200 ms both cut+ * there, 1500 ms keeps the sentence whole. Clicking the mic again ends the turn+ * at once and still transcribes it, so the longer window never traps anyone.+ */+export const END_OF_SPEECH_SILENCE_MS = 1500;++/** Give up on a single-shot listen when nobody speaks for this long. */+export const NO_SPEECH_TIMEOUT_MS = 12_000;++/** BCP-47 tag to its primary language subtag: 'pt-BR' to 'pt'. */+export function primaryLanguage(tag) {+ return String(tag || 'en').toLowerCase().split(/[-_]/)[0] || 'en';+}++/**+ * Which on-device model serves a language, and the generate options it needs.+ * English gets Moonshine (smaller, faster, English-only); everything else gets+ * multilingual Whisper pinned to the requested language so it never guesses.+ */+export function onDeviceModelFor(language) {+ const lang = primaryLanguage(language);+ if (lang === 'en') return { model: ON_DEVICE_MODELS.english, options: {} };+ return { model: ON_DEVICE_MODELS.multilingual, options: { language: lang, task: 'transcribe' } };+}++/** Can this browser capture audio and run the on-device recognizer at all? */+export function onDeviceSupported() {+ if (typeof window === 'undefined' || typeof WebAssembly !== 'object') return false;+ if (!navigator?.mediaDevices?.getUserMedia) return false;+ return Boolean(window.AudioContext || window.webkitAudioContext);+}++/**+ * Run each engine in order and return the first transcript. An engine that+ * throws hands over to the next; the last error surfaces only when all fail.+ *+ * @param {Float32Array} audio 16 kHz mono+ * @param {{id: string, transcribe: (audio: Float32Array) => Promise<string>}[]} engines+ * @returns {Promise<{text: string, engine: string}>}+ */+export async function transcribeWithFallback(audio, engines) {+ let lastError = null;+ for (const engine of engines) {+ try {+ const text = await engine.transcribe(audio);+ return { text: String(text || '').trim(), engine: engine.id };+ } catch (err) {+ lastError = err;+ log.warn(`[whisper-stt] ${engine.id} recognizer failed, trying the next`, err?.message || err);+ }+ }+ throw lastError || new Error('No speech recognizer is available');+}++/** Join utterance frames into one buffer. */+export function concatFrames(frames) {+ let length = 0;+ for (const f of frames) length += f.length;+ const out = new Float32Array(length);+ let offset = 0;+ for (const f of frames) {+ out.set(f, offset);+ offset += f.length;+ }+ return out;+}++// One transformers.js import and one pipeline per model for the whole page, so+// two agents on the same page share a single download and a single session.+// Every caller still hears the download progress, not only the first.+let transformersModule = null;+const transcribers = new Map();++function loadTranscriber(model, onProgress) {+ let entry = transcribers.get(model);+ if (entry) {+ if (onProgress && !entry.settled) entry.listeners.add(onProgress);+ return entry.promise;+ }+ entry = { promise: null, listeners: new Set(onProgress ? [onProgress] : []), settled: false };+ const fanOut = (event) => {+ for (const listener of entry.listeners) listener(event);+ };+ entry.promise = (async () => {+ transformersModule ||= import('@huggingface/transformers');+ const { pipeline, env } = await transformersModule;+ // Models come from the Hugging Face CDN only. Without this the library first+ // probes /models/<id>/ on the current origin, which on a host page that+ // embeds an agent is someone else's server.+ env.allowLocalModels = false;+ return pipeline('automatic-speech-recognition', model, {+ dtype: 'q8',+ device: 'wasm',+ progress_callback: fanOut,+ });+ })();+ const settle = () => {+ entry.settled = true;+ entry.listeners.clear();+ };+ entry.promise.then(settle, () => {+ settle();+ transcribers.delete(model);+ });+ transcribers.set(model, entry);+ return entry.promise;+}++/** Download progress events summed across a model's files into one 0..1 figure. */+function progressTracker(report) {+ const files = new Map();+ return (event) => {+ if (event?.status !== 'progress' || !event.file) return;+ files.set(event.file, { loaded: event.loaded || 0, total: event.total || 0 });+ let loaded = 0;+ let total = 0;+ for (const f of files.values()) {+ loaded += f.loaded;+ total += f.total;+ }+ if (total > 0) report(Math.min(1, loaded / total));+ };+}++export class WhisperSTT {+ /**+ * @param {object} [opts]+ * @param {string} [opts.language] BCP-47 tag, default en-US.+ * @param {boolean} [opts.continuous] Keep listening across utterances until stop().+ * @param {string} [opts.apiOrigin] three.ws origin for the server fallback and+ * the VAD assets. Empty means same-origin; an embed passes its API origin.+ * @param {number} [opts.endSilenceMs] Trailing silence that ends an utterance.+ * @param {(status: SttStatus) => void} [opts.onStatus]+ *+ * @typedef {object} SttStatus+ * @property {'idle'|'listening'|'transcribing'|'loading-model'} phase+ * 'loading-model' means an utterance is waiting on the model download.+ * @property {number} [progress] 0..1, with 'loading-model'.+ * @property {number} [modelProgress] 0..1, while the model downloads in the+ * background of another phase.+ */+ constructor({+ language = 'en-US',+ continuous = false,+ apiOrigin = '',+ endSilenceMs = END_OF_SPEECH_SILENCE_MS,+ onStatus = null,+ } = {}) {+ this.language = language;+ this.continuous = continuous;+ this.apiOrigin = String(apiOrigin || '').replace(/\/+$/, '');+ this.endSilenceMs = endSilenceMs;+ this.onStatus = onStatus;+ this._session = null;+ this._phase = 'idle';+ /** null until a download is seen, then 0..1; 1 once the model is ready. */+ this._modelProgress = null;+ }++ get listening() {+ return Boolean(this._session);+ }++ /** The recognizers tried for each utterance, best first. */+ engines() {+ const { model, options } = onDeviceModelFor(this.language);+ return [+ {+ id: 'on-device',+ transcribe: async (audio) => {+ const asr = await loadTranscriber(model, () => {});+ try {+ return (await asr(audio, options)).text;+ } catch (err) {+ // A language Whisper does not know throws before decoding; let it+ // detect the language itself rather than failing the utterance.+ if (!options.language) throw err;+ return (await asr(audio, { task: 'transcribe' })).text;+ }+ },+ },+ {+ id: 'server',+ transcribe: async (audio) => {+ const lang = encodeURIComponent(this.language);+ const res = await fetch(`${this.apiOrigin}/api/asr?language=${lang}`, {+ method: 'POST',+ headers: { 'content-type': 'audio/wav' },+ body: float32ToWav(audio, VAD_SAMPLE_RATE),+ });+ if (!res.ok) throw new Error(`/api/asr answered ${res.status}`);+ return (await res.json()).text;+ },+ },+ ];+ }++ /** Start the model download without listening, e.g. ahead of a first turn. */+ warm(onStatus) {+ const { model } = onDeviceModelFor(this.language);+ const pending = loadTranscriber(+ model,+ progressTracker((progress) => {+ this._modelProgress = progress;+ this._report(onStatus);+ }),+ );+ pending.then(+ () => {+ const changed = this._modelProgress !== 1;+ this._modelProgress = 1;+ if (changed && this._phase !== 'idle') this._report(onStatus);+ },+ () => {+ this._modelProgress = null;+ },+ );+ return pending;+ }++ /**+ * Report what the session is doing, folded together with the model download.+ * An utterance waiting on the download reads as 'loading-model' (that is what+ * the user is actually waiting for); otherwise the download rides along as+ * `modelProgress` so "listening" is never hidden behind a progress figure.+ */+ _report(onStatus, phase = this._phase) {+ this._phase = phase;+ const downloading = this._modelProgress != null && this._modelProgress < 1;+ let status;+ if (phase === 'transcribing' && downloading) status = { phase: 'loading-model', progress: this._modelProgress };+ else status = downloading ? { phase, modelProgress: this._modelProgress } : { phase };+ onStatus?.(status);+ this.onStatus?.(status);+ }++ // The on-device models decode whole utterances, so there is no partial text+ // and `onInterim` from the shared contract is never called.+ async listen({ onFinal, onStatus } = {}) {+ if (this._session) throw new Error('Already listening');+ const stream = await navigator.mediaDevices.getUserMedia({+ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },+ });+ const Ctx = window.AudioContext || window.webkitAudioContext;+ const audioContext = new Ctx();++ // Overlap the model download with the user speaking. A failure here is not+ // fatal: the server rung still answers, so it is only logged.+ this.warm(onStatus).catch((err) => log.warn('[whisper-stt] on-device model unavailable', err?.message || err));++ return new Promise((resolve, reject) => {+ const session = {+ stream,+ audioContext,+ vad: null,+ frames: [],+ samples: 0,+ speaking: false,+ // Set once a long utterance has been cut at the model window: the VAD's+ // own end-of-speech audio would then repeat what was already sent.+ cut: false,+ texts: [],+ errors: [],+ work: Promise.resolve(),+ finished: false,+ noSpeechTimer: null,+ };+ this._session = session;++ const transcribe = (audio) => {+ if (!audio?.length) return;+ session.work = session.work.then(async () => {+ this._report(onStatus, 'transcribing');+ try {+ const { text } = await transcribeWithFallback(audio, this.engines());+ if (text) {+ session.texts.push(text);+ onFinal?.(text);+ }+ } catch (err) {+ session.errors.push(err);+ }+ if (this.continuous && !session.finished) this._report(onStatus, 'listening');+ });+ };++ const finish = async () => {+ if (session.finished) return;+ session.finished = true;+ clearTimeout(session.noSpeechTimer);+ if (session.speaking && session.frames.length) transcribe(concatFrames(session.frames));+ session.frames = [];+ await teardown(session);+ await session.work;+ if (this._session === session) this._session = null;+ this._report(onStatus, 'idle');+ const text = session.texts.join(' ').trim();+ if (!text && session.errors.length) reject(session.errors[session.errors.length - 1]);+ else resolve(text);+ };+ session.finish = finish;++ const resetUtterance = () => {+ session.speaking = false;+ session.cut = false;+ session.frames = [];+ session.samples = 0;+ };++ const endUtterance = (audio) => {+ resetUtterance();+ transcribe(audio);+ if (!this.continuous) finish();+ else this._report(onStatus, 'listening');+ };++ // Longer than the model's window: send what is buffered and keep listening+ // to the rest of the same utterance.+ const cutUtterance = () => {+ transcribe(concatFrames(session.frames));+ session.frames = [];+ session.samples = 0;+ session.cut = true;+ if (!this.continuous) finish();+ };++ session.vad = new VoiceActivityDetector({+ stream,+ audioContext,+ assetPath: `${this.apiOrigin}${VAD_ASSET_PATH}`,+ redemptionMs: this.endSilenceMs,+ onSpeechStart: () => {+ clearTimeout(session.noSpeechTimer);+ resetUtterance();+ session.speaking = true;+ },+ onFrame: (frame) => {+ if (!session.speaking || session.finished) return;+ session.frames.push(frame);+ session.samples += frame.length;+ if (session.samples >= MAX_UTTERANCE_SAMPLES) cutUtterance();+ },+ onSpeechEnd: (audio) => {+ if (session.finished || !session.speaking) return;+ endUtterance(session.cut ? concatFrames(session.frames) : audio);+ },+ onMisfire: () => resetUtterance(),+ });++ session.vad+ .start()+ .then(() => {+ this._report(onStatus, 'listening');+ if (!this.continuous) session.noSpeechTimer = setTimeout(finish, NO_SPEECH_TIMEOUT_MS);+ })+ .catch(async (err) => {+ session.finished = true;+ await teardown(session);+ if (this._session === session) this._session = null;+ this._report(onStatus, 'idle');+ reject(err);+ });+ });+ }++ stop() {+ this._session?.finish?.();+ }+}++async function teardown(session) {+ try {+ await session.vad?.destroy();+ } catch (err) {+ log.warn('[whisper-stt] VAD teardown failed', err?.message || err);+ }+ for (const track of session.stream.getTracks()) track.stop();+ if (session.audioContext.state !== 'closed') await session.audioContext.close().catch(() => {});+}diff --git a/src/voice/vad.js b/src/voice/vad.jsindex 056f1de54..e3b73a9eb 100644--- a/src/voice/vad.js+++ b/src/voice/vad.js@@ -93,6 +93,9 @@ export class VoiceActivityDetector {* The finished utterance, pre-roll included, at 16 kHz.* @param {() => void} [opts.onMisfire] Speech started but was too short.* @param {number} [opts.redemptionMs] Trailing silence before end-of-speech.+ * @param {string} [opts.assetPath] Where the model and worklet are served.+ * Root-relative by default; an embed on someone else's page passes the+ * absolute three.ws path, since the host page does not serve these files.*/constructor({stream,@@ -103,6 +106,7 @@ export class VoiceActivityDetector {onSpeechEnd,onMisfire,redemptionMs = DEFAULT_REDEMPTION_MS,+ assetPath = VAD_ASSET_PATH,} = {}) {if (!stream) throw new Error('VoiceActivityDetector requires a MediaStream');if (!audioContext) throw new Error('VoiceActivityDetector requires an AudioContext');@@ -114,6 +118,7 @@ export class VoiceActivityDetector {this.onSpeechEnd = onSpeechEnd || (() => {});this.onMisfire = onMisfire || (() => {});this.redemptionMs = redemptionMs;+ this.assetPath = assetPath;this._vad = null;this._running = false;this._destroyed = false;@@ -138,8 +143,8 @@ export class VoiceActivityDetector {this._vad = await MicVAD.new({model: 'v5',- baseAssetPath: VAD_ASSET_PATH,- onnxWASMBasePath: VAD_ASSET_PATH,+ baseAssetPath: this.assetPath,+ onnxWASMBasePath: this.assetPath,// The caller owns the mic. Handing MicVAD the existing stream and// context is what keeps this to one getUserMedia for the whole loop.getStream: async () => this.stream,diff --git a/tests/whisper-stt.test.js b/tests/whisper-stt.test.jsnew file mode 100644index 000000000..63d0e2cca--- /dev/null+++ b/tests/whisper-stt.test.js@@ -0,0 +1,105 @@+import { describe, it, expect } from 'vitest';+import { createSTT, BrowserSTT } from '../src/runtime/speech.js';+import {+ ON_DEVICE_MODELS,+ MAX_UTTERANCE_SAMPLES,+ WhisperSTT,+ concatFrames,+ onDeviceModelFor,+ onDeviceSupported,+ primaryLanguage,+ transcribeWithFallback,+} from '../src/runtime/whisper-stt.js';++describe('createSTT("whisper")', () => {+ it('no longer throws for the provider the manifest spec has always listed', () => {+ expect(() => createSTT({ provider: 'whisper' })).not.toThrow();+ });++ it('falls back to the browser recognizer where audio capture is impossible', () => {+ // Node has no window, mic or AudioContext: the honest answer is BrowserSTT.+ expect(onDeviceSupported()).toBe(false);+ expect(createSTT({ provider: 'whisper' })).toBeInstanceOf(BrowserSTT);+ });+});++describe('language routing', () => {+ it('reduces a BCP-47 tag to its primary subtag', () => {+ expect(primaryLanguage('pt-BR')).toBe('pt');+ expect(primaryLanguage('zh_Hans')).toBe('zh');+ expect(primaryLanguage('')).toBe('en');+ expect(primaryLanguage(undefined)).toBe('en');+ });++ it('sends English to Moonshine with no language hint', () => {+ expect(onDeviceModelFor('en-US')).toEqual({ model: ON_DEVICE_MODELS.english, options: {} });+ expect(onDeviceModelFor('en-GB').model).toBe(ON_DEVICE_MODELS.english);+ });++ it('sends every other language to Whisper pinned to that language', () => {+ expect(onDeviceModelFor('fr-FR')).toEqual({+ model: ON_DEVICE_MODELS.multilingual,+ options: { language: 'fr', task: 'transcribe' },+ });+ expect(onDeviceModelFor('ja').options.language).toBe('ja');+ });++ it('keeps an utterance inside the 30 s Whisper window', () => {+ expect(MAX_UTTERANCE_SAMPLES).toBeLessThan(30 * 16000);+ });+});++describe('transcribeWithFallback', () => {+ const audio = new Float32Array(16000);++ it('returns the first engine that answers, trimmed, with its id', async () => {+ const out = await transcribeWithFallback(audio, [+ { id: 'on-device', transcribe: async () => ' hello there ' },+ { id: 'server', transcribe: async () => 'never reached' },+ ]);+ expect(out).toEqual({ text: 'hello there', engine: 'on-device' });+ });++ it('hands over to the next engine when one throws', async () => {+ const out = await transcribeWithFallback(audio, [+ { id: 'on-device', transcribe: async () => { throw new Error('wasm failed'); } },+ { id: 'server', transcribe: async () => 'from the server' },+ ]);+ expect(out).toEqual({ text: 'from the server', engine: 'server' });+ });++ it('surfaces the last error only when every engine fails', async () => {+ await expect(+ transcribeWithFallback(audio, [+ { id: 'on-device', transcribe: async () => { throw new Error('first'); } },+ { id: 'server', transcribe: async () => { throw new Error('/api/asr answered 503'); } },+ ]),+ ).rejects.toThrow('/api/asr answered 503');+ });++ it('treats silence as an empty transcript, not a failure', async () => {+ const out = await transcribeWithFallback(audio, [{ id: 'on-device', transcribe: async () => null }]);+ expect(out.text).toBe('');+ });+});++describe('WhisperSTT', () => {+ it('tries on-device first and the /api/asr lane second', () => {+ expect(new WhisperSTT().engines().map((e) => e.id)).toEqual(['on-device', 'server']);+ });++ it('normalizes the API origin so the fallback URL never doubles a slash', () => {+ expect(new WhisperSTT({ apiOrigin: 'https://three.ws/' }).apiOrigin).toBe('https://three.ws');+ });++ it('is idle until listen() is called', () => {+ expect(new WhisperSTT().listening).toBe(false);+ });+});++describe('concatFrames', () => {+ it('joins frames in order', () => {+ const out = concatFrames([Float32Array.of(1, 2), Float32Array.of(3), new Float32Array(0)]);+ expect(Array.from(out)).toEqual([1, 2, 3]);+ });+});