Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(voice): run agent speech-to-text on-device with Whisper when the manifest asks for it

nirholas169d6a4c449b
diff --git a/src/element.js b/src/element.js
index 9f96c6014..211548aba 100644
--- a/src/element.js
+++ b/src/element.js
@@ -1283,20 +1283,7 @@ class Agent3DElement extends HTMLElement {
});
row.appendChild(input);
- if (this._speechInputAvailable()) {
- const mic = document.createElement('button');
- mic.type = 'button';
- mic.className = 'icon mic';
- mic.title = 'Push to talk';
- mic.setAttribute('aria-label', 'Push to talk');
- mic.innerHTML =
- '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">' +
- '<rect x="9" y="3" width="6" height="11" rx="3"></rect>' +
- '<path d="M5 11a7 7 0 0 0 14 0"></path><path d="M12 18v3"></path></svg>';
- mic.addEventListener('click', () => this._toggleMic());
- row.appendChild(mic);
- this._micEl = mic;
- }
+ if (this._speechInputAvailable()) row.appendChild(this._createMicButton());
row.appendChild(send);
this._inputEl = input;
return row;
@@ -1309,6 +1296,57 @@ class Agent3DElement extends HTMLElement {
return name && name !== 'Agent' ? `Message ${name}…` : 'Say something...';
}
+ _createMicButton() {
+ const mic = document.createElement('button');
+ mic.type = 'button';
+ mic.className = 'icon mic';
+ mic.title = 'Push to talk';
+ mic.setAttribute('aria-label', 'Push to talk');
+ mic.innerHTML =
+ '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">' +
+ '<rect x="9" y="3" width="6" height="11" rx="3"></rect>' +
+ '<path d="M5 11a7 7 0 0 0 14 0"></path><path d="M12 18v3"></path></svg>';
+ mic.addEventListener('click', () => this._toggleMic());
+ this._micEl = mic;
+ return mic;
+ }
+
+ // The composer is built before the manifest is known, so an agent whose STT
+ // runs on-device (manifest voice.stt.provider "whisper") gets its mic here,
+ // once the runtime exists. That is what gives Firefox and Safari visitors push
+ // to talk: they have no browser recognizer for _speechInputAvailable to find.
+ _ensureOnDeviceMic() {
+ if (this._micEl || !this._inputEl?.parentElement) return;
+ if (typeof this._runtime?.stt?.warm !== 'function') return;
+ this._inputEl.after(this._createMicButton());
+ }
+
+ // Reflect what the recognizer is doing on the mic, so a first-use model
+ // download or a slow decode reads as progress instead of a frozen button.
+ _renderSttStatus(status) {
+ const mic = this._micEl;
+ if (!mic) return;
+ const phase = status?.phase;
+ if (phase === 'loading-model') {
+ const pct = Number.isFinite(status.progress) ? ` ${Math.round(status.progress * 100)}%` : '';
+ mic.dataset.voiceState = 'thinking';
+ mic.title = `Loading the on-device speech model${pct}`;
+ } else if (phase === 'transcribing') {
+ mic.dataset.voiceState = 'thinking';
+ mic.title = 'Transcribing';
+ } else if (phase === 'listening') {
+ mic.dataset.voiceState = 'listening';
+ const dl = Number.isFinite(status.modelProgress)
+ ? `, speech model ${Math.round(status.modelProgress * 100)}% downloaded`
+ : '';
+ mic.title = `Listening${dl}. Click to stop`;
+ } else {
+ delete mic.dataset.voiceState;
+ mic.title = 'Push to talk';
+ }
+ mic.setAttribute('aria-label', mic.title);
+ }
+
// Push-to-talk works through a voice server or the browser's own speech
// recognition. Where neither exists the mic would be a dead button.
_speechInputAvailable() {
@@ -2016,6 +2054,7 @@ class Agent3DElement extends HTMLElement {
agentId: _backendId || undefined,
skillAccess: _skillAccess,
});
+ this._ensureOnDeviceMic();
// ── Empathy + lipsync layer ───────────────────────────────────────
// AgentAvatar subscribes to the agent-protocol bus, runs the emotion
@@ -2088,6 +2127,7 @@ class Agent3DElement extends HTMLElement {
'voice:speech-end',
'voice:transcript',
'voice:listen-start',
+ 'voice:stt-status',
'memory:write',
]) {
this._runtime.addEventListener(ev, (e) => {
@@ -2981,13 +3021,14 @@ class Agent3DElement extends HTMLElement {
this._listening = true;
this._micEl.dataset.listening = 'true';
try {
- const text = await this._runtime.listen();
+ const text = await this._runtime.listen({ onStatus: (s) => this._renderSttStatus(s) });
if (text) this.say(text, { voice: true });
} catch (e) {
log.warn('[agent-3d] listen failed', e);
} finally {
this._listening = false;
this._micEl.dataset.listening = 'false';
+ this._renderSttStatus(null);
}
}
diff --git a/src/runtime/index.js b/src/runtime/index.js
index af0a561a4..17ecddc8c 100644
--- a/src/runtime/index.js
+++ b/src/runtime/index.js
@@ -53,7 +53,11 @@ export class Runtime extends EventTarget {
...this.manifest.voice?.tts,
...voiceConfig.tts,
});
- this.stt = createSTT({ ...this.manifest.voice?.stt, ...voiceConfig.stt });
+ this.stt = createSTT({
+ ...(this.apiOrigin ? { apiOrigin: this.apiOrigin } : {}),
+ ...this.manifest.voice?.stt,
+ ...voiceConfig.stt,
+ });
this.messages = [];
this._busy = false;
@@ -296,10 +300,16 @@ export class Runtime extends EventTarget {
};
}
- async listen({ onInterim, onFinal } = {}) {
+ async listen({ onInterim, onFinal, onStatus } = {}) {
if (!this.stt) throw new Error('STT not configured');
this.dispatchEvent(new CustomEvent('voice:listen-start', {}));
const text = await this.stt.listen({
+ // Recognizers that do real work before text exists (an on-device model
+ // downloading, an utterance decoding) report it, so the UI can say so.
+ onStatus: (status) => {
+ this.dispatchEvent(new CustomEvent('voice:stt-status', { detail: status }));
+ onStatus?.(status);
+ },
onInterim: (t) => {
this.dispatchEvent(
new CustomEvent('voice:transcript', { detail: { text: t, final: false } }),
diff --git a/src/runtime/speech.js b/src/runtime/speech.js
index 5c2a43695..4ffd23c4b 100644
--- a/src/runtime/speech.js
+++ b/src/runtime/speech.js
@@ -2,7 +2,9 @@
export { LiveKitVoice } from './livekit-voice.js';
export { NeuralTTS } from './neural-tts.js';
+export { WhisperSTT } from './whisper-stt.js';
import { NeuralTTS } from './neural-tts.js';
+import { WhisperSTT, onDeviceSupported } from './whisper-stt.js';
import { startLipsync } from './lipsync.js';
import { ACTION_TYPES } from '../agent-protocol.js';
@@ -528,6 +530,9 @@ const TTS_FACTORIES = {
const STT_FACTORIES = {
none: () => null,
browser: (cfg) => new BrowserSTT(cfg),
+ // On-device recognition (src/runtime/whisper-stt.js). Where the browser cannot
+ // capture audio or run WebAssembly, its own recognizer is the honest fallback.
+ whisper: (cfg) => (onDeviceSupported() ? new WhisperSTT(cfg) : new BrowserSTT(cfg)),
};
export function createTTS(config = {}) {
diff --git a/src/runtime/whisper-stt.js b/src/runtime/whisper-stt.js
new file mode 100644
index 000000000..0591f05a6
--- /dev/null
+++ b/src/runtime/whisper-stt.js
@@ -0,0 +1,427 @@
+/**
+ * WhisperSTT: the `whisper` speech-to-text provider of the agent manifest.
+ *
+ * Speech is recognized on the visitor's own device. No audio leaves the browser
+ * on the primary path, nothing is billed, and unlike `BrowserSTT` it works in
+ * every browser with WebAssembly, Firefox and Safari included, where
+ * window.SpeechRecognition is missing or ships the audio to a vendor.
+ *
+ * Pipeline:
+ *
+ * mic ─▶ silero VAD (src/voice/vad.js) ─▶ one utterance, 16 kHz Float32
+ * ─▶ on-device transformers.js ASR ─▶ text
+ * └▶ (on-device failed) POST /api/asr, the NVIDIA Riva lane ─▶ text
+ *
+ * Models (both MIT, ONNX q8, fetched once from the Hugging Face CDN and kept in
+ * the browser Cache API, so the second visit loads from disk):
+ *
+ * English onnx-community/moonshine-base-ONNX about 63 MB
+ * Other onnx-community/whisper-base about 77 MB, 99 languages
+ *
+ * Measured on the JFK reference clip (11 s, CPU, 4 threads): Moonshine 2.0 s and
+ * Whisper 3.0 s per utterance, both transcribing it word-perfect.
+ *
+ * The model download starts the moment listening starts, so it overlaps the
+ * user speaking instead of following it. Progress is reported through
+ * `onStatus` so a caller can say what is happening on a first, slow load.
+ *
+ * Same contract as BrowserSTT: listen({ onInterim, onFinal }) resolves with the
+ * final transcript, stop() ends listening and still transcribes whatever was
+ * said. `continuous: true` keeps listening across utterances until stop(),
+ * firing onFinal per utterance.
+ */
+
+import { log } from '../shared/log.js';
+import { VoiceActivityDetector, VAD_ASSET_PATH, VAD_SAMPLE_RATE, float32ToWav } from '../voice/vad.js';
+
+export const ON_DEVICE_MODELS = Object.freeze({
+ english: 'onnx-community/moonshine-base-ONNX',
+ multilingual: 'onnx-community/whisper-base',
+});
+
+/** Whisper reads a 30 s window; cut a little short so padding never truncates speech. */
+export const MAX_UTTERANCE_SAMPLES = 28 * VAD_SAMPLE_RATE;
+
+/**
+ * Trailing silence that ends an utterance. The home voice loop runs at 352 ms
+ * because hands-free turn-taking has to feel instant; push-to-talk does not, and
+ * at 352 ms an ordinary mid-sentence pause ends the turn. The JFK reference clip
+ * pauses 1.0 to 1.2 s after "my fellow Americans": 352 ms and 1200 ms both cut
+ * there, 1500 ms keeps the sentence whole. Clicking the mic again ends the turn
+ * at once and still transcribes it, so the longer window never traps anyone.
+ */
+export const END_OF_SPEECH_SILENCE_MS = 1500;
+
+/** Give up on a single-shot listen when nobody speaks for this long. */
+export const NO_SPEECH_TIMEOUT_MS = 12_000;
+
+/** BCP-47 tag to its primary language subtag: 'pt-BR' to 'pt'. */
+export function primaryLanguage(tag) {
+ return String(tag || 'en').toLowerCase().split(/[-_]/)[0] || 'en';
+}
+
+/**
+ * Which on-device model serves a language, and the generate options it needs.
+ * English gets Moonshine (smaller, faster, English-only); everything else gets
+ * multilingual Whisper pinned to the requested language so it never guesses.
+ */
+export function onDeviceModelFor(language) {
+ const lang = primaryLanguage(language);
+ if (lang === 'en') return { model: ON_DEVICE_MODELS.english, options: {} };
+ return { model: ON_DEVICE_MODELS.multilingual, options: { language: lang, task: 'transcribe' } };
+}
+
+/** Can this browser capture audio and run the on-device recognizer at all? */
+export function onDeviceSupported() {
+ if (typeof window === 'undefined' || typeof WebAssembly !== 'object') return false;
+ if (!navigator?.mediaDevices?.getUserMedia) return false;
+ return Boolean(window.AudioContext || window.webkitAudioContext);
+}
+
+/**
+ * Run each engine in order and return the first transcript. An engine that
+ * throws hands over to the next; the last error surfaces only when all fail.
+ *
+ * @param {Float32Array} audio 16 kHz mono
+ * @param {{id: string, transcribe: (audio: Float32Array) => Promise<string>}[]} engines
+ * @returns {Promise<{text: string, engine: string}>}
+ */
+export async function transcribeWithFallback(audio, engines) {
+ let lastError = null;
+ for (const engine of engines) {
+ try {
+ const text = await engine.transcribe(audio);
+ return { text: String(text || '').trim(), engine: engine.id };
+ } catch (err) {
+ lastError = err;
+ log.warn(`[whisper-stt] ${engine.id} recognizer failed, trying the next`, err?.message || err);
+ }
+ }
+ throw lastError || new Error('No speech recognizer is available');
+}
+
+/** Join utterance frames into one buffer. */
+export function concatFrames(frames) {
+ let length = 0;
+ for (const f of frames) length += f.length;
+ const out = new Float32Array(length);
+ let offset = 0;
+ for (const f of frames) {
+ out.set(f, offset);
+ offset += f.length;
+ }
+ return out;
+}
+
+// One transformers.js import and one pipeline per model for the whole page, so
+// two agents on the same page share a single download and a single session.
+// Every caller still hears the download progress, not only the first.
+let transformersModule = null;
+const transcribers = new Map();
+
+function loadTranscriber(model, onProgress) {
+ let entry = transcribers.get(model);
+ if (entry) {
+ if (onProgress && !entry.settled) entry.listeners.add(onProgress);
+ return entry.promise;
+ }
+ entry = { promise: null, listeners: new Set(onProgress ? [onProgress] : []), settled: false };
+ const fanOut = (event) => {
+ for (const listener of entry.listeners) listener(event);
+ };
+ entry.promise = (async () => {
+ transformersModule ||= import('@huggingface/transformers');
+ const { pipeline, env } = await transformersModule;
+ // Models come from the Hugging Face CDN only. Without this the library first
+ // probes /models/<id>/ on the current origin, which on a host page that
+ // embeds an agent is someone else's server.
+ env.allowLocalModels = false;
+ return pipeline('automatic-speech-recognition', model, {
+ dtype: 'q8',
+ device: 'wasm',
+ progress_callback: fanOut,
+ });
+ })();
+ const settle = () => {
+ entry.settled = true;
+ entry.listeners.clear();
+ };
+ entry.promise.then(settle, () => {
+ settle();
+ transcribers.delete(model);
+ });
+ transcribers.set(model, entry);
+ return entry.promise;
+}
+
+/** Download progress events summed across a model's files into one 0..1 figure. */
+function progressTracker(report) {
+ const files = new Map();
+ return (event) => {
+ if (event?.status !== 'progress' || !event.file) return;
+ files.set(event.file, { loaded: event.loaded || 0, total: event.total || 0 });
+ let loaded = 0;
+ let total = 0;
+ for (const f of files.values()) {
+ loaded += f.loaded;
+ total += f.total;
+ }
+ if (total > 0) report(Math.min(1, loaded / total));
+ };
+}
+
+export class WhisperSTT {
+ /**
+ * @param {object} [opts]
+ * @param {string} [opts.language] BCP-47 tag, default en-US.
+ * @param {boolean} [opts.continuous] Keep listening across utterances until stop().
+ * @param {string} [opts.apiOrigin] three.ws origin for the server fallback and
+ * the VAD assets. Empty means same-origin; an embed passes its API origin.
+ * @param {number} [opts.endSilenceMs] Trailing silence that ends an utterance.
+ * @param {(status: SttStatus) => void} [opts.onStatus]
+ *
+ * @typedef {object} SttStatus
+ * @property {'idle'|'listening'|'transcribing'|'loading-model'} phase
+ * 'loading-model' means an utterance is waiting on the model download.
+ * @property {number} [progress] 0..1, with 'loading-model'.
+ * @property {number} [modelProgress] 0..1, while the model downloads in the
+ * background of another phase.
+ */
+ constructor({
+ language = 'en-US',
+ continuous = false,
+ apiOrigin = '',
+ endSilenceMs = END_OF_SPEECH_SILENCE_MS,
+ onStatus = null,
+ } = {}) {
+ this.language = language;
+ this.continuous = continuous;
+ this.apiOrigin = String(apiOrigin || '').replace(/\/+$/, '');
+ this.endSilenceMs = endSilenceMs;
+ this.onStatus = onStatus;
+ this._session = null;
+ this._phase = 'idle';
+ /** null until a download is seen, then 0..1; 1 once the model is ready. */
+ this._modelProgress = null;
+ }
+
+ get listening() {
+ return Boolean(this._session);
+ }
+
+ /** The recognizers tried for each utterance, best first. */
+ engines() {
+ const { model, options } = onDeviceModelFor(this.language);
+ return [
+ {
+ id: 'on-device',
+ transcribe: async (audio) => {
+ const asr = await loadTranscriber(model, () => {});
+ try {
+ return (await asr(audio, options)).text;
+ } catch (err) {
+ // A language Whisper does not know throws before decoding; let it
+ // detect the language itself rather than failing the utterance.
+ if (!options.language) throw err;
+ return (await asr(audio, { task: 'transcribe' })).text;
+ }
+ },
+ },
+ {
+ id: 'server',
+ transcribe: async (audio) => {
+ const lang = encodeURIComponent(this.language);
+ const res = await fetch(`${this.apiOrigin}/api/asr?language=${lang}`, {
+ method: 'POST',
+ headers: { 'content-type': 'audio/wav' },
+ body: float32ToWav(audio, VAD_SAMPLE_RATE),
+ });
+ if (!res.ok) throw new Error(`/api/asr answered ${res.status}`);
+ return (await res.json()).text;
+ },
+ },
+ ];
+ }
+
+ /** Start the model download without listening, e.g. ahead of a first turn. */
+ warm(onStatus) {
+ const { model } = onDeviceModelFor(this.language);
+ const pending = loadTranscriber(
+ model,
+ progressTracker((progress) => {
+ this._modelProgress = progress;
+ this._report(onStatus);
+ }),
+ );
+ pending.then(
+ () => {
+ const changed = this._modelProgress !== 1;
+ this._modelProgress = 1;
+ if (changed && this._phase !== 'idle') this._report(onStatus);
+ },
+ () => {
+ this._modelProgress = null;
+ },
+ );
+ return pending;
+ }
+
+ /**
+ * Report what the session is doing, folded together with the model download.
+ * An utterance waiting on the download reads as 'loading-model' (that is what
+ * the user is actually waiting for); otherwise the download rides along as
+ * `modelProgress` so "listening" is never hidden behind a progress figure.
+ */
+ _report(onStatus, phase = this._phase) {
+ this._phase = phase;
+ const downloading = this._modelProgress != null && this._modelProgress < 1;
+ let status;
+ if (phase === 'transcribing' && downloading) status = { phase: 'loading-model', progress: this._modelProgress };
+ else status = downloading ? { phase, modelProgress: this._modelProgress } : { phase };
+ onStatus?.(status);
+ this.onStatus?.(status);
+ }
+
+ // The on-device models decode whole utterances, so there is no partial text
+ // and `onInterim` from the shared contract is never called.
+ async listen({ onFinal, onStatus } = {}) {
+ if (this._session) throw new Error('Already listening');
+ const stream = await navigator.mediaDevices.getUserMedia({
+ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
+ });
+ const Ctx = window.AudioContext || window.webkitAudioContext;
+ const audioContext = new Ctx();
+
+ // Overlap the model download with the user speaking. A failure here is not
+ // fatal: the server rung still answers, so it is only logged.
+ this.warm(onStatus).catch((err) => log.warn('[whisper-stt] on-device model unavailable', err?.message || err));
+
+ return new Promise((resolve, reject) => {
+ const session = {
+ stream,
+ audioContext,
+ vad: null,
+ frames: [],
+ samples: 0,
+ speaking: false,
+ // Set once a long utterance has been cut at the model window: the VAD's
+ // own end-of-speech audio would then repeat what was already sent.
+ cut: false,
+ texts: [],
+ errors: [],
+ work: Promise.resolve(),
+ finished: false,
+ noSpeechTimer: null,
+ };
+ this._session = session;
+
+ const transcribe = (audio) => {
+ if (!audio?.length) return;
+ session.work = session.work.then(async () => {
+ this._report(onStatus, 'transcribing');
+ try {
+ const { text } = await transcribeWithFallback(audio, this.engines());
+ if (text) {
+ session.texts.push(text);
+ onFinal?.(text);
+ }
+ } catch (err) {
+ session.errors.push(err);
+ }
+ if (this.continuous && !session.finished) this._report(onStatus, 'listening');
+ });
+ };
+
+ const finish = async () => {
+ if (session.finished) return;
+ session.finished = true;
+ clearTimeout(session.noSpeechTimer);
+ if (session.speaking && session.frames.length) transcribe(concatFrames(session.frames));
+ session.frames = [];
+ await teardown(session);
+ await session.work;
+ if (this._session === session) this._session = null;
+ this._report(onStatus, 'idle');
+ const text = session.texts.join(' ').trim();
+ if (!text && session.errors.length) reject(session.errors[session.errors.length - 1]);
+ else resolve(text);
+ };
+ session.finish = finish;
+
+ const resetUtterance = () => {
+ session.speaking = false;
+ session.cut = false;
+ session.frames = [];
+ session.samples = 0;
+ };
+
+ const endUtterance = (audio) => {
+ resetUtterance();
+ transcribe(audio);
+ if (!this.continuous) finish();
+ else this._report(onStatus, 'listening');
+ };
+
+ // Longer than the model's window: send what is buffered and keep listening
+ // to the rest of the same utterance.
+ const cutUtterance = () => {
+ transcribe(concatFrames(session.frames));
+ session.frames = [];
+ session.samples = 0;
+ session.cut = true;
+ if (!this.continuous) finish();
+ };
+
+ session.vad = new VoiceActivityDetector({
+ stream,
+ audioContext,
+ assetPath: `${this.apiOrigin}${VAD_ASSET_PATH}`,
+ redemptionMs: this.endSilenceMs,
+ onSpeechStart: () => {
+ clearTimeout(session.noSpeechTimer);
+ resetUtterance();
+ session.speaking = true;
+ },
+ onFrame: (frame) => {
+ if (!session.speaking || session.finished) return;
+ session.frames.push(frame);
+ session.samples += frame.length;
+ if (session.samples >= MAX_UTTERANCE_SAMPLES) cutUtterance();
+ },
+ onSpeechEnd: (audio) => {
+ if (session.finished || !session.speaking) return;
+ endUtterance(session.cut ? concatFrames(session.frames) : audio);
+ },
+ onMisfire: () => resetUtterance(),
+ });
+
+ session.vad
+ .start()
+ .then(() => {
+ this._report(onStatus, 'listening');
+ if (!this.continuous) session.noSpeechTimer = setTimeout(finish, NO_SPEECH_TIMEOUT_MS);
+ })
+ .catch(async (err) => {
+ session.finished = true;
+ await teardown(session);
+ if (this._session === session) this._session = null;
+ this._report(onStatus, 'idle');
+ reject(err);
+ });
+ });
+ }
+
+ stop() {
+ this._session?.finish?.();
+ }
+}
+
+async function teardown(session) {
+ try {
+ await session.vad?.destroy();
+ } catch (err) {
+ log.warn('[whisper-stt] VAD teardown failed', err?.message || err);
+ }
+ for (const track of session.stream.getTracks()) track.stop();
+ if (session.audioContext.state !== 'closed') await session.audioContext.close().catch(() => {});
+}
diff --git a/src/voice/vad.js b/src/voice/vad.js
index 056f1de54..e3b73a9eb 100644
--- a/src/voice/vad.js
+++ b/src/voice/vad.js
@@ -93,6 +93,9 @@ export class VoiceActivityDetector {
* The finished utterance, pre-roll included, at 16 kHz.
* @param {() => void} [opts.onMisfire] Speech started but was too short.
* @param {number} [opts.redemptionMs] Trailing silence before end-of-speech.
+ * @param {string} [opts.assetPath] Where the model and worklet are served.
+ * Root-relative by default; an embed on someone else's page passes the
+ * absolute three.ws path, since the host page does not serve these files.
*/
constructor({
stream,
@@ -103,6 +106,7 @@ export class VoiceActivityDetector {
onSpeechEnd,
onMisfire,
redemptionMs = DEFAULT_REDEMPTION_MS,
+ assetPath = VAD_ASSET_PATH,
} = {}) {
if (!stream) throw new Error('VoiceActivityDetector requires a MediaStream');
if (!audioContext) throw new Error('VoiceActivityDetector requires an AudioContext');
@@ -114,6 +118,7 @@ export class VoiceActivityDetector {
this.onSpeechEnd = onSpeechEnd || (() => {});
this.onMisfire = onMisfire || (() => {});
this.redemptionMs = redemptionMs;
+ this.assetPath = assetPath;
this._vad = null;
this._running = false;
this._destroyed = false;
@@ -138,8 +143,8 @@ export class VoiceActivityDetector {
this._vad = await MicVAD.new({
model: 'v5',
- baseAssetPath: VAD_ASSET_PATH,
- onnxWASMBasePath: VAD_ASSET_PATH,
+ baseAssetPath: this.assetPath,
+ onnxWASMBasePath: this.assetPath,
// The caller owns the mic. Handing MicVAD the existing stream and
// context is what keeps this to one getUserMedia for the whole loop.
getStream: async () => this.stream,
diff --git a/tests/whisper-stt.test.js b/tests/whisper-stt.test.js
new file mode 100644
index 000000000..63d0e2cca
--- /dev/null
+++ b/tests/whisper-stt.test.js
@@ -0,0 +1,105 @@
+import { describe, it, expect } from 'vitest';
+import { createSTT, BrowserSTT } from '../src/runtime/speech.js';
+import {
+ ON_DEVICE_MODELS,
+ MAX_UTTERANCE_SAMPLES,
+ WhisperSTT,
+ concatFrames,
+ onDeviceModelFor,
+ onDeviceSupported,
+ primaryLanguage,
+ transcribeWithFallback,
+} from '../src/runtime/whisper-stt.js';
+
+describe('createSTT("whisper")', () => {
+ it('no longer throws for the provider the manifest spec has always listed', () => {
+ expect(() => createSTT({ provider: 'whisper' })).not.toThrow();
+ });
+
+ it('falls back to the browser recognizer where audio capture is impossible', () => {
+ // Node has no window, mic or AudioContext: the honest answer is BrowserSTT.
+ expect(onDeviceSupported()).toBe(false);
+ expect(createSTT({ provider: 'whisper' })).toBeInstanceOf(BrowserSTT);
+ });
+});
+
+describe('language routing', () => {
+ it('reduces a BCP-47 tag to its primary subtag', () => {
+ expect(primaryLanguage('pt-BR')).toBe('pt');
+ expect(primaryLanguage('zh_Hans')).toBe('zh');
+ expect(primaryLanguage('')).toBe('en');
+ expect(primaryLanguage(undefined)).toBe('en');
+ });
+
+ it('sends English to Moonshine with no language hint', () => {
+ expect(onDeviceModelFor('en-US')).toEqual({ model: ON_DEVICE_MODELS.english, options: {} });
+ expect(onDeviceModelFor('en-GB').model).toBe(ON_DEVICE_MODELS.english);
+ });
+
+ it('sends every other language to Whisper pinned to that language', () => {
+ expect(onDeviceModelFor('fr-FR')).toEqual({
+ model: ON_DEVICE_MODELS.multilingual,
+ options: { language: 'fr', task: 'transcribe' },
+ });
+ expect(onDeviceModelFor('ja').options.language).toBe('ja');
+ });
+
+ it('keeps an utterance inside the 30 s Whisper window', () => {
+ expect(MAX_UTTERANCE_SAMPLES).toBeLessThan(30 * 16000);
+ });
+});
+
+describe('transcribeWithFallback', () => {
+ const audio = new Float32Array(16000);
+
+ it('returns the first engine that answers, trimmed, with its id', async () => {
+ const out = await transcribeWithFallback(audio, [
+ { id: 'on-device', transcribe: async () => ' hello there ' },
+ { id: 'server', transcribe: async () => 'never reached' },
+ ]);
+ expect(out).toEqual({ text: 'hello there', engine: 'on-device' });
+ });
+
+ it('hands over to the next engine when one throws', async () => {
+ const out = await transcribeWithFallback(audio, [
+ { id: 'on-device', transcribe: async () => { throw new Error('wasm failed'); } },
+ { id: 'server', transcribe: async () => 'from the server' },
+ ]);
+ expect(out).toEqual({ text: 'from the server', engine: 'server' });
+ });
+
+ it('surfaces the last error only when every engine fails', async () => {
+ await expect(
+ transcribeWithFallback(audio, [
+ { id: 'on-device', transcribe: async () => { throw new Error('first'); } },
+ { id: 'server', transcribe: async () => { throw new Error('/api/asr answered 503'); } },
+ ]),
+ ).rejects.toThrow('/api/asr answered 503');
+ });
+
+ it('treats silence as an empty transcript, not a failure', async () => {
+ const out = await transcribeWithFallback(audio, [{ id: 'on-device', transcribe: async () => null }]);
+ expect(out.text).toBe('');
+ });
+});
+
+describe('WhisperSTT', () => {
+ it('tries on-device first and the /api/asr lane second', () => {
+ expect(new WhisperSTT().engines().map((e) => e.id)).toEqual(['on-device', 'server']);
+ });
+
+ it('normalizes the API origin so the fallback URL never doubles a slash', () => {
+ expect(new WhisperSTT({ apiOrigin: 'https://three.ws/' }).apiOrigin).toBe('https://three.ws');
+ });
+
+ it('is idle until listen() is called', () => {
+ expect(new WhisperSTT().listening).toBe(false);
+ });
+});
+
+describe('concatFrames', () => {
+ it('joins frames in order', () => {
+ const out = concatFrames([Float32Array.of(1, 2), Float32Array.of(3), new Float32Array(0)]);
+ expect(Array.from(out)).toEqual([1, 2, 3]);
+ });
+});