Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/**
* Voice activity detection for the browser voice loop.
*
* A thin, lazily loaded wrapper over silero-vad (MIT,
* https://github.com/snakers4/silero-vad) through @ricky0123/vad-web. We do not
* write a detector: silero is a trained model with years of tuning behind it and
* an energy threshold is not a substitute.
*
* Two things this wrapper adds over calling MicVAD directly, and both are
* load-bearing for the home voice loop:
*
* 1. It never opens the microphone itself. The caller passes the MediaStream
* and the AudioContext it already owns, so the VAD, the wake word and the
* utterance capture all read ONE mic. Three getUserMedia calls would mean
* three indicator dots in the browser chrome and three tracks to remember to
* stop, and "mute" has to stop every track that exists.
* 2. It surfaces every 512-sample, 16 kHz frame to the caller. The wake word
* needs exactly that feed, and running a second resampler for it would
* double the audio work for no gain.
*
* Everything here loads on demand. Nothing in this module's dependency graph is
* fetched until start() is called, which is what makes "nothing about listening
* loads before you opt in" a true statement rather than a promise.
*
* Model: silero v5, 512-sample frames at 16 kHz, so 32 ms per frame. The v5
* frame is the reason we do not use the legacy model: its 1536-sample frame is
* 96 ms, which is a quarter of the entire end-of-speech budget in granularity
* alone.
*/
import { log } from '../shared/log.js';
/** Where copy-voice-models.mjs stages the VAD model and its worklet. */
export const VAD_ASSET_PATH = '/models/voice/runtime/';
/** Silero v5 reads 512 samples at a time; the loop's timing math depends on it. */
export const VAD_FRAME_SAMPLES = 512;
export const VAD_SAMPLE_RATE = 16000;
export const VAD_MS_PER_FRAME = (VAD_FRAME_SAMPLES / VAD_SAMPLE_RATE) * 1000;
/**
* Trailing silence before an utterance is called finished.
*
* The budget is "under 400 ms": longer and the agent feels broken, shorter and it
* clips people mid-sentence. 352 ms is 11 whole frames, and the frame count is
* what matters because the deadline is checked once per frame. It is not 12
* frames (384 ms) because the wall-clock gap a user actually experiences runs
* from the last frame silero rated as speech to the moment the utterance closes,
* and that is one frame period longer than the configured window: measured at
* 412 to 424 ms with 12 frames, which misses the budget. Eleven frames measures
* in the high 380s. Widening the budget was the alternative, and it was the wrong
* one.
*/
export const DEFAULT_REDEMPTION_MS = 352;
/**
* Audio kept from BEFORE speech was detected. Silero fires a frame or two into a
* word, so without this the transcript loses the first consonant.
*/
const PRE_SPEECH_PAD_MS = 480;
/** Shorter than a syllable; anything under this is a cough, a door, a keyboard. */
const MIN_SPEECH_MS = 250;
/**
* Silero's own defaults, restated here rather than inherited invisibly, because
* the endpoint measurement depends on knowing which of the two governs the
* countdown. Speech is declared above the positive threshold and the redemption
* clock starts below the negative one; the band between them is hysteresis, and
* it is what stops a breath inside a word from ending the utterance.
*/
const POSITIVE_SPEECH_THRESHOLD = 0.3;
const NEGATIVE_SPEECH_THRESHOLD = 0.25;
let vadModulePromise = null;
/** Load @ricky0123/vad-web once, on demand. */
function loadVadModule() {
if (!vadModulePromise) vadModulePromise = import('@ricky0123/vad-web');
return vadModulePromise;
}
export class VoiceActivityDetector {
/**
* @param {object} opts
* @param {MediaStream} opts.stream The caller's single mic stream.
* @param {AudioContext} opts.audioContext The caller's single audio context.
* @param {(frame: Float32Array, probability: number) => void} [opts.onFrame]
* Every 512-sample 16 kHz frame, with silero's speech probability.
* @param {() => void} [opts.onSpeechStart] Speech probably started.
* @param {() => void} [opts.onSpeechRealStart] Speech passed the minimum length.
* @param {(audio: Float32Array) => void} [opts.onSpeechEnd]
* The finished utterance, pre-roll included, at 16 kHz.
* @param {() => void} [opts.onMisfire] Speech started but was too short.
* @param {number} [opts.redemptionMs] Trailing silence before end-of-speech.
* @param {string} [opts.assetPath] Where the model and worklet are served.
* Root-relative by default; an embed on someone else's page passes the
* absolute three.ws path, since the host page does not serve these files.
*/
constructor({
stream,
audioContext,
onFrame,
onSpeechStart,
onSpeechRealStart,
onSpeechEnd,
onMisfire,
redemptionMs = DEFAULT_REDEMPTION_MS,
assetPath = VAD_ASSET_PATH,
} = {}) {
if (!stream) throw new Error('VoiceActivityDetector requires a MediaStream');
if (!audioContext) throw new Error('VoiceActivityDetector requires an AudioContext');
this.stream = stream;
this.audioContext = audioContext;
this.onFrame = onFrame || (() => {});
this.onSpeechStart = onSpeechStart || (() => {});
this.onSpeechRealStart = onSpeechRealStart || (() => {});
this.onSpeechEnd = onSpeechEnd || (() => {});
this.onMisfire = onMisfire || (() => {});
this.redemptionMs = redemptionMs;
this.assetPath = assetPath;
this._vad = null;
this._running = false;
this._destroyed = false;
/** Wall clock of the last frame that carried speech, for the end-of-speech measurement. */
this.lastSpeechFrameAt = 0;
}
get running() {
return this._running;
}
/** Load the model and begin reading the caller's stream. */
async start() {
if (this._destroyed) throw new Error('VoiceActivityDetector was destroyed');
if (this._vad) {
await this._vad.start();
this._running = true;
return;
}
const { MicVAD } = await loadVadModule();
this._vad = await MicVAD.new({
model: 'v5',
baseAssetPath: this.assetPath,
onnxWASMBasePath: this.assetPath,
// The caller owns the mic. Handing MicVAD the existing stream and
// context is what keeps this to one getUserMedia for the whole loop.
getStream: async () => this.stream,
resumeStream: async () => this.stream,
// Never stop the caller's tracks on pause: mute owns that, and a
// paused VAD must not silently release a mic the capture still needs.
pauseStream: async () => {},
audioContext: this.audioContext,
startOnLoad: false,
redemptionMs: this.redemptionMs,
preSpeechPadMs: PRE_SPEECH_PAD_MS,
minSpeechMs: MIN_SPEECH_MS,
// Silero's own defaults, restated so a future tuning pass has one place
// to argue with rather than an invisible import.
positiveSpeechThreshold: POSITIVE_SPEECH_THRESHOLD,
negativeSpeechThreshold: NEGATIVE_SPEECH_THRESHOLD,
onFrameProcessed: ({ isSpeech }, frame) => {
// Marked at the NEGATIVE threshold, because that is the one the
// redemption countdown actually starts from. Marking at the positive
// threshold instead would count the frames between the two as part of
// the trailing silence and report a gap the user never waited through.
if (isSpeech >= NEGATIVE_SPEECH_THRESHOLD) this.lastSpeechFrameAt = now();
this.onFrame(frame, isSpeech);
},
onSpeechStart: () => this.onSpeechStart(),
onSpeechRealStart: () => this.onSpeechRealStart(),
onSpeechEnd: (audio) => this.onSpeechEnd(audio),
onVADMisfire: () => this.onMisfire(),
ortConfig: (ort) => {
ort.env.logLevel = 'error';
// One thread: the loop shares the machine with a WebGL scene, and
// cross-origin isolation (which multi-threaded wasm needs) is not
// something a page embedding third-party avatars can assume.
ort.env.wasm.numThreads = 1;
},
});
await this._vad.start();
this._running = true;
}
/** Stop reading frames. The caller's mic tracks are left alone. */
async pause() {
if (!this._vad || !this._running) return;
this._running = false;
try {
await this._vad.pause();
} catch (err) {
log.warn('[vad] pause failed', err?.message);
}
}
/** Release the model and the worklet. Idempotent. */
async destroy() {
this._destroyed = true;
this._running = false;
if (!this._vad) return;
const vad = this._vad;
this._vad = null;
try {
await vad.destroy();
} catch (err) {
log.warn('[vad] destroy failed', err?.message);
}
}
}
/** Monotonic where available so a clock change cannot corrupt a latency number. */
export function now() {
return typeof performance !== 'undefined' && performance.now ? performance.now() : Date.now();
}
/**
* Float32 [-1, 1] at 16 kHz to a 16-bit mono RIFF/WAVE buffer, the one format
* /api/asr documents as accepted without a sample-rate query parameter.
*/
export function float32ToWav(samples, sampleRate = VAD_SAMPLE_RATE) {
const bytesPerSample = 2;
const dataBytes = samples.length * bytesPerSample;
const buffer = new ArrayBuffer(44 + dataBytes);
const view = new DataView(buffer);
const ascii = (offset, str) => {
for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i));
};
ascii(0, 'RIFF');
view.setUint32(4, 36 + dataBytes, true);
ascii(8, 'WAVE');
ascii(12, 'fmt ');
view.setUint32(16, 16, true);
view.setUint16(20, 1, true); // PCM
view.setUint16(22, 1, true); // mono
view.setUint32(24, sampleRate, true);
view.setUint32(28, sampleRate * bytesPerSample, true);
view.setUint16(32, bytesPerSample, true);
view.setUint16(34, 16, true);
ascii(36, 'data');
view.setUint32(40, dataBytes, true);
let offset = 44;
for (let i = 0; i < samples.length; i++, offset += 2) {
const s = Math.max(-1, Math.min(1, samples[i]));
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
}
return buffer;
}