Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/**
* WhisperSTT: the `whisper` speech-to-text provider of the agent manifest.
*
* Speech is recognized on the visitor's own device. No audio leaves the browser
* on the primary path, nothing is billed, and unlike `BrowserSTT` it works in
* every browser with WebAssembly, Firefox and Safari included, where
* window.SpeechRecognition is missing or ships the audio to a vendor.
*
* Pipeline:
*
* mic ─▶ silero VAD (src/voice/vad.js) ─▶ one utterance, 16 kHz Float32
* ─▶ on-device transformers.js ASR ─▶ text
* └▶ (on-device failed) POST /api/asr, the NVIDIA Riva lane ─▶ text
*
* Models (both MIT, ONNX q8, fetched once from the Hugging Face CDN and kept in
* the browser Cache API, so the second visit loads from disk):
*
* English onnx-community/moonshine-base-ONNX about 63 MB
* Other onnx-community/whisper-base about 77 MB, 99 languages
*
* Measured on the JFK reference clip (11 s, CPU, 4 threads): Moonshine 2.0 s and
* Whisper 3.0 s per utterance, both transcribing it word-perfect.
*
* The model download starts the moment listening starts, so it overlaps the
* user speaking instead of following it. Progress is reported through
* `onStatus` so a caller can say what is happening on a first, slow load.
*
* Same contract as BrowserSTT: listen({ onInterim, onFinal }) resolves with the
* final transcript, stop() ends listening and still transcribes whatever was
* said. `continuous: true` keeps listening across utterances until stop(),
* firing onFinal per utterance.
*/
import { log } from '../shared/log.js';
import { loadModule } from '../shared/load-module.js';
import { VoiceActivityDetector, VAD_ASSET_PATH, VAD_SAMPLE_RATE, float32ToWav } from '../voice/vad.js';
// Loaded from the CDN, never bundled: transformers.js drags in onnxruntime-web's
// WebGPU runtime, tens of MB. A bare `import('@huggingface/transformers')` is
// statically analyzable, so Vite/Rollup bundles it anyway wherever the build
// doesn't split dynamic imports into their own chunk (the single-file CDN embed
// in vite.config.js's libConfig sets inlineDynamicImports for exactly that
// reason), blowing that embed from ~600KB to 70+MB and tripping Cloud Run's
// response-size limit. loadModule()'s import() specifier is a runtime string,
// which no bundler can resolve at build time, so this stays lazy everywhere.
const TRANSFORMERS_CDN_URL = 'https://esm.sh/@huggingface/[email protected]';
export const ON_DEVICE_MODELS = Object.freeze({
english: 'onnx-community/moonshine-base-ONNX',
multilingual: 'onnx-community/whisper-base',
});
/** Whisper reads a 30 s window; cut a little short so padding never truncates speech. */
export const MAX_UTTERANCE_SAMPLES = 28 * VAD_SAMPLE_RATE;
/**
* Trailing silence that ends an utterance. The home voice loop runs at 352 ms
* because hands-free turn-taking has to feel instant; push-to-talk does not, and
* at 352 ms an ordinary mid-sentence pause ends the turn. The JFK reference clip
* pauses 1.0 to 1.2 s after "my fellow Americans": 352 ms and 1200 ms both cut
* there, 1500 ms keeps the sentence whole. Clicking the mic again ends the turn
* at once and still transcribes it, so the longer window never traps anyone.
*/
export const END_OF_SPEECH_SILENCE_MS = 1500;
/** Give up on a single-shot listen when nobody speaks for this long. */
export const NO_SPEECH_TIMEOUT_MS = 12_000;
/** BCP-47 tag to its primary language subtag: 'pt-BR' to 'pt'. */
export function primaryLanguage(tag) {
return String(tag || 'en').toLowerCase().split(/[-_]/)[0] || 'en';
}
/**
* Which on-device model serves a language, and the generate options it needs.
* English gets Moonshine (smaller, faster, English-only); everything else gets
* multilingual Whisper pinned to the requested language so it never guesses.
*/
export function onDeviceModelFor(language) {
const lang = primaryLanguage(language);
if (lang === 'en') return { model: ON_DEVICE_MODELS.english, options: {} };
return { model: ON_DEVICE_MODELS.multilingual, options: { language: lang, task: 'transcribe' } };
}
/** Can this browser capture audio and run the on-device recognizer at all? */
export function onDeviceSupported() {
if (typeof window === 'undefined' || typeof WebAssembly !== 'object') return false;
if (!navigator?.mediaDevices?.getUserMedia) return false;
return Boolean(window.AudioContext || window.webkitAudioContext);
}
/**
* Run each engine in order and return the first transcript. An engine that
* throws hands over to the next; the last error surfaces only when all fail.
*
* @param {Float32Array} audio 16 kHz mono
* @param {{id: string, transcribe: (audio: Float32Array) => Promise<string>}[]} engines
* @returns {Promise<{text: string, engine: string}>}
*/
export async function transcribeWithFallback(audio, engines) {
let lastError = null;
for (const engine of engines) {
try {
const text = await engine.transcribe(audio);
return { text: String(text || '').trim(), engine: engine.id };
} catch (err) {
lastError = err;
log.warn(`[whisper-stt] ${engine.id} recognizer failed, trying the next`, err?.message || err);
}
}
throw lastError || new Error('No speech recognizer is available');
}
/** Join utterance frames into one buffer. */
export function concatFrames(frames) {
let length = 0;
for (const f of frames) length += f.length;
const out = new Float32Array(length);
let offset = 0;
for (const f of frames) {
out.set(f, offset);
offset += f.length;
}
return out;
}
// One transformers.js import and one pipeline per model for the whole page, so
// two agents on the same page share a single download and a single session.
// Every caller still hears the download progress, not only the first.
let transformersModule = null;
const transcribers = new Map();
function loadTranscriber(model, onProgress) {
let entry = transcribers.get(model);
if (entry) {
if (onProgress && !entry.settled) entry.listeners.add(onProgress);
return entry.promise;
}
entry = { promise: null, listeners: new Set(onProgress ? [onProgress] : []), settled: false };
const fanOut = (event) => {
for (const listener of entry.listeners) listener(event);
};
entry.promise = (async () => {
transformersModule ||= loadModule(TRANSFORMERS_CDN_URL);
const { pipeline, env } = await transformersModule;
// Models come from the Hugging Face CDN only. Without this the library first
// probes /models/<id>/ on the current origin, which on a host page that
// embeds an agent is someone else's server.
env.allowLocalModels = false;
return pipeline('automatic-speech-recognition', model, {
dtype: 'q8',
device: 'wasm',
progress_callback: fanOut,
});
})();
const settle = () => {
entry.settled = true;
entry.listeners.clear();
};
entry.promise.then(settle, () => {
settle();
transcribers.delete(model);
});
transcribers.set(model, entry);
return entry.promise;
}
/** Download progress events summed across a model's files into one 0..1 figure. */
function progressTracker(report) {
const files = new Map();
return (event) => {
if (event?.status !== 'progress' || !event.file) return;
files.set(event.file, { loaded: event.loaded || 0, total: event.total || 0 });
let loaded = 0;
let total = 0;
for (const f of files.values()) {
loaded += f.loaded;
total += f.total;
}
if (total > 0) report(Math.min(1, loaded / total));
};
}
export class WhisperSTT {
/**
* @param {object} [opts]
* @param {string} [opts.language] BCP-47 tag, default en-US.
* @param {boolean} [opts.continuous] Keep listening across utterances until stop().
* @param {string} [opts.apiOrigin] three.ws origin for the server fallback and
* the VAD assets. Empty means same-origin; an embed passes its API origin.
* @param {number} [opts.endSilenceMs] Trailing silence that ends an utterance.
* @param {(status: SttStatus) => void} [opts.onStatus]
*
* @typedef {object} SttStatus
* @property {'idle'|'listening'|'transcribing'|'loading-model'} phase
* 'loading-model' means an utterance is waiting on the model download.
* @property {number} [progress] 0..1, with 'loading-model'.
* @property {number} [modelProgress] 0..1, while the model downloads in the
* background of another phase.
*/
constructor({
language = 'en-US',
continuous = false,
apiOrigin = '',
endSilenceMs = END_OF_SPEECH_SILENCE_MS,
onStatus = null,
} = {}) {
this.language = language;
this.continuous = continuous;
this.apiOrigin = String(apiOrigin || '').replace(/\/+$/, '');
this.endSilenceMs = endSilenceMs;
this.onStatus = onStatus;
this._session = null;
this._phase = 'idle';
/** null until a download is seen, then 0..1; 1 once the model is ready. */
this._modelProgress = null;
}
get listening() {
return Boolean(this._session);
}
/** The recognizers tried for each utterance, best first. */
engines() {
const { model, options } = onDeviceModelFor(this.language);
return [
{
id: 'on-device',
transcribe: async (audio) => {
const asr = await loadTranscriber(model, () => {});
try {
return (await asr(audio, options)).text;
} catch (err) {
// A language Whisper does not know throws before decoding; let it
// detect the language itself rather than failing the utterance.
if (!options.language) throw err;
return (await asr(audio, { task: 'transcribe' })).text;
}
},
},
{
id: 'server',
transcribe: async (audio) => {
const lang = encodeURIComponent(this.language);
const res = await fetch(`${this.apiOrigin}/api/asr?language=${lang}`, {
method: 'POST',
headers: { 'content-type': 'audio/wav' },
body: float32ToWav(audio, VAD_SAMPLE_RATE),
});
if (!res.ok) throw new Error(`/api/asr answered ${res.status}`);
return (await res.json()).text;
},
},
];
}
/** Start the model download without listening, e.g. ahead of a first turn. */
warm(onStatus) {
const { model } = onDeviceModelFor(this.language);
const pending = loadTranscriber(
model,
progressTracker((progress) => {
this._modelProgress = progress;
this._report(onStatus);
}),
);
pending.then(
() => {
const changed = this._modelProgress !== 1;
this._modelProgress = 1;
if (changed && this._phase !== 'idle') this._report(onStatus);
},
() => {
this._modelProgress = null;
},
);
return pending;
}
/**
* Report what the session is doing, folded together with the model download.
* An utterance waiting on the download reads as 'loading-model' (that is what
* the user is actually waiting for); otherwise the download rides along as
* `modelProgress` so "listening" is never hidden behind a progress figure.
*/
_report(onStatus, phase = this._phase) {
this._phase = phase;
const downloading = this._modelProgress != null && this._modelProgress < 1;
let status;
if (phase === 'transcribing' && downloading) status = { phase: 'loading-model', progress: this._modelProgress };
else status = downloading ? { phase, modelProgress: this._modelProgress } : { phase };
onStatus?.(status);
this.onStatus?.(status);
}
// The on-device models decode whole utterances, so there is no partial text
// and `onInterim` from the shared contract is never called.
async listen({ onFinal, onStatus } = {}) {
if (this._session) throw new Error('Already listening');
const stream = await navigator.mediaDevices.getUserMedia({
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
});
const Ctx = window.AudioContext || window.webkitAudioContext;
const audioContext = new Ctx();
// Overlap the model download with the user speaking. A failure here is not
// fatal: the server rung still answers, so it is only logged.
this.warm(onStatus).catch((err) => log.warn('[whisper-stt] on-device model unavailable', err?.message || err));
return new Promise((resolve, reject) => {
const session = {
stream,
audioContext,
vad: null,
frames: [],
samples: 0,
speaking: false,
// Set once a long utterance has been cut at the model window: the VAD's
// own end-of-speech audio would then repeat what was already sent.
cut: false,
texts: [],
errors: [],
work: Promise.resolve(),
finished: false,
noSpeechTimer: null,
};
this._session = session;
const transcribe = (audio) => {
if (!audio?.length) return;
session.work = session.work.then(async () => {
this._report(onStatus, 'transcribing');
try {
const { text } = await transcribeWithFallback(audio, this.engines());
if (text) {
session.texts.push(text);
onFinal?.(text);
}
} catch (err) {
session.errors.push(err);
}
if (this.continuous && !session.finished) this._report(onStatus, 'listening');
});
};
const finish = async () => {
if (session.finished) return;
session.finished = true;
clearTimeout(session.noSpeechTimer);
if (session.speaking && session.frames.length) transcribe(concatFrames(session.frames));
session.frames = [];
await teardown(session);
await session.work;
if (this._session === session) this._session = null;
this._report(onStatus, 'idle');
const text = session.texts.join(' ').trim();
if (!text && session.errors.length) reject(session.errors[session.errors.length - 1]);
else resolve(text);
};
session.finish = finish;
const resetUtterance = () => {
session.speaking = false;
session.cut = false;
session.frames = [];
session.samples = 0;
};
const endUtterance = (audio) => {
resetUtterance();
transcribe(audio);
if (!this.continuous) finish();
else this._report(onStatus, 'listening');
};
// Longer than the model's window: send what is buffered and keep listening
// to the rest of the same utterance.
const cutUtterance = () => {
transcribe(concatFrames(session.frames));
session.frames = [];
session.samples = 0;
session.cut = true;
if (!this.continuous) finish();
};
session.vad = new VoiceActivityDetector({
stream,
audioContext,
assetPath: `${this.apiOrigin}${VAD_ASSET_PATH}`,
redemptionMs: this.endSilenceMs,
onSpeechStart: () => {
clearTimeout(session.noSpeechTimer);
resetUtterance();
session.speaking = true;
},
onFrame: (frame) => {
if (!session.speaking || session.finished) return;
session.frames.push(frame);
session.samples += frame.length;
if (session.samples >= MAX_UTTERANCE_SAMPLES) cutUtterance();
},
onSpeechEnd: (audio) => {
if (session.finished || !session.speaking) return;
endUtterance(session.cut ? concatFrames(session.frames) : audio);
},
onMisfire: () => resetUtterance(),
});
session.vad
.start()
.then(() => {
this._report(onStatus, 'listening');
if (!this.continuous) session.noSpeechTimer = setTimeout(finish, NO_SPEECH_TIMEOUT_MS);
})
.catch(async (err) => {
session.finished = true;
await teardown(session);
if (this._session === session) this._session = null;
this._report(onStatus, 'idle');
reject(err);
});
});
}
stop() {
this._session?.finish?.();
}
}
async function teardown(session) {
try {
await session.vad?.destroy();
} catch (err) {
log.warn('[whisper-stt] VAD teardown failed', err?.message || err);
}
for (const track of session.stream.getTracks()) track.stop();
if (session.audioContext.state !== 'closed') await session.audioContext.close().catch(() => {});
}