Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(voice): serve the wake word and the VAD from three.ws itself

nirholasd2b8da8d82b2
diff --git a/package-lock.json b/package-lock.json
index d02ca928d..193828407 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -124,6 +124,7 @@
"@pump-fun/pump-sdk": "^1.36.0",
"@pump-fun/pump-swap-sdk": "^1.19.0",
"@pythnetwork/hermes-client": "^3.1.0",
+ "@ricky0123/vad-web": "^0.0.30",
"@scure/base": "^1.2.6",
"@solana-mobile/mobile-wallet-adapter-protocol-web3js": "^2.2.9",
"@solana-program/compute-budget": "^0.15.0",
@@ -12643,6 +12644,15 @@
"node": ">= 10"
}
},
+ "node_modules/@ricky0123/vad-web": {
+ "version": "0.0.30",
+ "resolved": "https://registry.npmjs.org/@ricky0123/vad-web/-/vad-web-0.0.30.tgz",
+ "integrity": "sha512-cJyYrh4YeeUBJcbR9Bic/bFDyB9qBkAepvpuWM3vLxnAi7bC3VHzf51UeNdT+OtY4D7MLAgV8iJMc4z41ZnaWg==",
+ "license": "ISC",
+ "dependencies": {
+ "onnxruntime-web": "^1.17.0"
+ }
+ },
"node_modules/@rolldown/pluginutils": {
"version": "1.0.0-beta.27",
"dev": true,
@@ -40357,6 +40367,7 @@
"version": "0.1.0",
"license": "Apache-2.0",
"dependencies": {
+ "@leeoniya/ufuzzy": "^1.0.19",
"home-assistant-js-websocket": "^9.6.0"
},
"engines": {
diff --git a/package.json b/package.json
index 59a13c999..0c15f5a1b 100644
--- a/package.json
+++ b/package.json
@@ -142,7 +142,7 @@
]
},
"scripts": {
- "postinstall": "(node scripts/build-cache.mjs check agent-payments-sdk agent-payments-sdk/src agent-payments-sdk/package.json || (npm run build --prefix agent-payments-sdk && node scripts/build-cache.mjs stamp agent-payments-sdk agent-payments-sdk/src agent-payments-sdk/package.json)) && (node scripts/build-cache.mjs check solana-agent-sdk solana-agent-sdk/src solana-agent-sdk/package.json || (npm run build:dist --prefix solana-agent-sdk && node scripts/build-cache.mjs stamp solana-agent-sdk solana-agent-sdk/src solana-agent-sdk/package.json)) && (node scripts/build-cache.mjs check walk-sdk walk-sdk/src walk-sdk/package.json || (npm run build --prefix walk-sdk && node scripts/build-cache.mjs stamp walk-sdk walk-sdk/src walk-sdk/package.json)) && (node scripts/fix-pump-sdk-esm.mjs & node scripts/fix-x402-extensions-esm.mjs & node scripts/fix-zauth-sdk-solana-esm.mjs & node scripts/fix-colyseus-storage.mjs & node scripts/fix-colyseus-send.mjs & node scripts/fix-rapier-init-warning.mjs & node scripts/copy-three-decoders.mjs & node scripts/setup-claude-skills.mjs & node scripts/setup-git-hooks.mjs & node scripts/build-docs-search-index.mjs & wait)",
+ "postinstall": "(node scripts/build-cache.mjs check agent-payments-sdk agent-payments-sdk/src agent-payments-sdk/package.json || (npm run build --prefix agent-payments-sdk && node scripts/build-cache.mjs stamp agent-payments-sdk agent-payments-sdk/src agent-payments-sdk/package.json)) && (node scripts/build-cache.mjs check solana-agent-sdk solana-agent-sdk/src solana-agent-sdk/package.json || (npm run build:dist --prefix solana-agent-sdk && node scripts/build-cache.mjs stamp solana-agent-sdk solana-agent-sdk/src solana-agent-sdk/package.json)) && (node scripts/build-cache.mjs check walk-sdk walk-sdk/src walk-sdk/package.json || (npm run build --prefix walk-sdk && node scripts/build-cache.mjs stamp walk-sdk walk-sdk/src walk-sdk/package.json)) && (node scripts/fix-pump-sdk-esm.mjs & node scripts/fix-x402-extensions-esm.mjs & node scripts/fix-zauth-sdk-solana-esm.mjs & node scripts/fix-colyseus-storage.mjs & node scripts/fix-colyseus-send.mjs & node scripts/fix-rapier-init-warning.mjs & node scripts/copy-three-decoders.mjs & node scripts/copy-voice-models.mjs & node scripts/setup-claude-skills.mjs & node scripts/setup-git-hooks.mjs & node scripts/build-docs-search-index.mjs & wait)",
"setup": "node scripts/setup-dev.mjs",
"dev": "vite --port 3000",
"dev:multi": "npm run dev --prefix multiplayer",
@@ -259,7 +259,7 @@
"triage:gcp:deep": "node scripts/gcp-triage.mjs --deep --since 6h",
"snapshot": "node scripts/page-snapshot.mjs",
"build:news": "node scripts/build-news.mjs",
- "prebuild": "node scripts/copy-three-decoders.mjs && (node scripts/build-news.mjs & node scripts/build-skill-metadata.mjs & node scripts/build-local-skill-packs.mjs & node scripts/build-discovery-cards.mjs & node scripts/build-guards.mjs & node scripts/build-atlas-index.mjs & node scripts/build-mcp-catalog.mjs & npm run build:examples & npm run build:docs-search & npm run build:club-assets & npm run build:walk-environments & wait) && node scripts/inject-blog-seo.mjs --write && node scripts/inject-hidden-guard.mjs --write && (node scripts/build-page-index.mjs && node scripts/audit-page-index.mjs --strict && node scripts/audit-empty-handlers.mjs && node scripts/audit-hidden-guard.mjs) && node scripts/i18n-annotate.mjs --apply --pages=\"public/news/*.html\" && node scripts/inject-seo-meta.mjs --write && node scripts/inject-theme-boot.mjs --write",
+ "prebuild": "node scripts/copy-three-decoders.mjs && node scripts/copy-voice-models.mjs && (node scripts/build-news.mjs & node scripts/build-skill-metadata.mjs & node scripts/build-local-skill-packs.mjs & node scripts/build-discovery-cards.mjs & node scripts/build-guards.mjs & node scripts/build-atlas-index.mjs & node scripts/build-mcp-catalog.mjs & npm run build:examples & npm run build:docs-search & npm run build:club-assets & npm run build:walk-environments & wait) && node scripts/inject-blog-seo.mjs --write && node scripts/inject-hidden-guard.mjs --write && (node scripts/build-page-index.mjs && node scripts/audit-page-index.mjs --strict && node scripts/audit-empty-handlers.mjs && node scripts/audit-hidden-guard.mjs) && node scripts/i18n-annotate.mjs --apply --pages=\"public/news/*.html\" && node scripts/inject-seo-meta.mjs --write && node scripts/inject-theme-boot.mjs --write",
"seo:meta": "node scripts/inject-blog-seo.mjs --write && node scripts/build-page-index.mjs && node scripts/inject-seo-meta.mjs --write",
"build": "NODE_OPTIONS='--no-deprecation --max-old-space-size=6144' vite build && node scripts/strip-sw-from-embeds.mjs && node scripts/inject-tour-boot.mjs && node scripts/inject-atlas.mjs && node scripts/inject-inline-behaviors.mjs",
"build:wasm": "node scripts/build-wasm.mjs",
@@ -472,6 +472,7 @@
"@pump-fun/pump-sdk": "^1.36.0",
"@pump-fun/pump-swap-sdk": "^1.19.0",
"@pythnetwork/hermes-client": "^3.1.0",
+ "@ricky0123/vad-web": "^0.0.30",
"@scure/base": "^1.2.6",
"@solana-mobile/mobile-wallet-adapter-protocol-web3js": "^2.2.9",
"@solana-program/compute-budget": "^0.15.0",
diff --git a/public/models/voice/wake-word/alexa_v0.1.onnx b/public/models/voice/wake-word/alexa_v0.1.onnx
new file mode 100644
index 000000000..984ec2cbb
Binary files /dev/null and b/public/models/voice/wake-word/alexa_v0.1.onnx differ
diff --git a/public/models/voice/wake-word/embedding_model.onnx b/public/models/voice/wake-word/embedding_model.onnx
new file mode 100644
index 000000000..afde53ef5
Binary files /dev/null and b/public/models/voice/wake-word/embedding_model.onnx differ
diff --git a/public/models/voice/wake-word/hey_jarvis_v0.1.onnx b/public/models/voice/wake-word/hey_jarvis_v0.1.onnx
new file mode 100644
index 000000000..371aa73e2
Binary files /dev/null and b/public/models/voice/wake-word/hey_jarvis_v0.1.onnx differ
diff --git a/public/models/voice/wake-word/hey_mycroft_v0.1.onnx b/public/models/voice/wake-word/hey_mycroft_v0.1.onnx
new file mode 100644
index 000000000..7305ca442
Binary files /dev/null and b/public/models/voice/wake-word/hey_mycroft_v0.1.onnx differ
diff --git a/public/models/voice/wake-word/hey_rhasspy_v0.1.onnx b/public/models/voice/wake-word/hey_rhasspy_v0.1.onnx
new file mode 100644
index 000000000..914d4f85e
Binary files /dev/null and b/public/models/voice/wake-word/hey_rhasspy_v0.1.onnx differ
diff --git a/public/models/voice/wake-word/melspectrogram.onnx b/public/models/voice/wake-word/melspectrogram.onnx
new file mode 100644
index 000000000..a3a60354e
Binary files /dev/null and b/public/models/voice/wake-word/melspectrogram.onnx differ
diff --git a/scripts/copy-voice-models.mjs b/scripts/copy-voice-models.mjs
new file mode 100644
index 000000000..47045f7e8
--- /dev/null
+++ b/scripts/copy-voice-models.mjs
@@ -0,0 +1,64 @@
+// Stage the browser voice-loop runtime into /public/models/voice/runtime/ so the
+// wake word and the VAD load every byte from three.ws itself.
+//
+// Why same-origin matters here and not only as a preference: this is an
+// always-on microphone feature. A user who opts in is trusting us with a live
+// mic, and a third-party CDN fetch on that path would put a stranger's server in
+// the room. Serving the models ourselves means the audio path has exactly one
+// origin, which is the claim the consent copy makes.
+//
+// What is staged (all from node_modules, so the lockfile pins the versions):
+// silero_vad_v5.onnx @ricky0123/vad-web — the MIT silero VAD
+// vad.worklet.bundle.min.js @ricky0123/vad-web — its AudioWorklet
+// ort-wasm-simd-threaded.mjs onnxruntime-web — the wasm loader
+// ort-wasm-simd-threaded.wasm onnxruntime-web — the runtime itself
+//
+// NOT staged here: the openWakeWord models under /public/models/voice/wake-word.
+// Those are Apache-2.0 release artifacts from dscripka/openWakeWord v0.5.1 that
+// no npm package ships, so they are committed as bytes rather than fetched at
+// build time. A build that reaches out to GitHub is a build that fails when
+// GitHub does.
+//
+// Runs from postinstall and prebuild, exactly like copy-three-decoders.mjs.
+// /public/models/voice/runtime/ is gitignored; dist/ gets it from prebuild.
+
+import { copyFileSync, existsSync, mkdirSync, statSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const repo = resolve(__dirname, '..');
+const out = join(repo, 'public/models/voice/runtime');
+
+const files = [
+ ['node_modules/@ricky0123/vad-web/dist/silero_vad_v5.onnx', 'silero_vad_v5.onnx'],
+ ['node_modules/@ricky0123/vad-web/dist/vad.worklet.bundle.min.js', 'vad.worklet.bundle.min.js'],
+ ['node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.mjs', 'ort-wasm-simd-threaded.mjs'],
+ ['node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.wasm', 'ort-wasm-simd-threaded.wasm'],
+];
+
+const missing = files.filter(([from]) => !existsSync(join(repo, from)));
+if (missing.length === files.length) {
+ console.warn('[copy-voice-models] no source files found — skipping. Run npm install first.');
+ process.exit(0);
+}
+
+mkdirSync(out, { recursive: true });
+
+let staged = 0;
+for (const [from, name] of files) {
+ const src = join(repo, from);
+ if (!existsSync(src)) {
+ console.warn(`[copy-voice-models] ${from} not found`);
+ continue;
+ }
+ const dest = join(out, name);
+ // Skip an identical restage so a repeated prebuild does not rewrite 11 MB.
+ if (existsSync(dest) && statSync(dest).size === statSync(src).size) {
+ staged++;
+ continue;
+ }
+ copyFileSync(src, dest);
+ staged++;
+}
+console.log(`[copy-voice-models] ${staged}/${files.length} → public/models/voice/runtime`);
diff --git a/src/voice/vad.js b/src/voice/vad.js
new file mode 100644
index 000000000..be747bf84
--- /dev/null
+++ b/src/voice/vad.js
@@ -0,0 +1,226 @@
+/**
+ * Voice activity detection for the browser voice loop.
+ *
+ * A thin, lazily loaded wrapper over silero-vad (MIT,
+ * https://github.com/snakers4/silero-vad) through @ricky0123/vad-web. We do not
+ * write a detector: silero is a trained model with years of tuning behind it and
+ * an energy threshold is not a substitute.
+ *
+ * Two things this wrapper adds over calling MicVAD directly, and both are
+ * load-bearing for the home voice loop:
+ *
+ * 1. It never opens the microphone itself. The caller passes the MediaStream
+ * and the AudioContext it already owns, so the VAD, the wake word and the
+ * utterance capture all read ONE mic. Three getUserMedia calls would mean
+ * three indicator dots in the browser chrome and three tracks to remember to
+ * stop, and "mute" has to stop every track that exists.
+ * 2. It surfaces every 512-sample, 16 kHz frame to the caller. The wake word
+ * needs exactly that feed, and running a second resampler for it would
+ * double the audio work for no gain.
+ *
+ * Everything here loads on demand. Nothing in this module's dependency graph is
+ * fetched until start() is called, which is what makes "nothing about listening
+ * loads before you opt in" a true statement rather than a promise.
+ *
+ * Model: silero v5, 512-sample frames at 16 kHz, so 32 ms per frame. The v5
+ * frame is the reason we do not use the legacy model: its 1536-sample frame is
+ * 96 ms, which is a quarter of the entire end-of-speech budget in granularity
+ * alone.
+ */
+
+import { log } from '../shared/log.js';
+
+/** Where copy-voice-models.mjs stages the VAD model and its worklet. */
+export const VAD_ASSET_PATH = '/models/voice/runtime/';
+
+/** Silero v5 reads 512 samples at a time; the loop's timing math depends on it. */
+export const VAD_FRAME_SAMPLES = 512;
+export const VAD_SAMPLE_RATE = 16000;
+export const VAD_MS_PER_FRAME = (VAD_FRAME_SAMPLES / VAD_SAMPLE_RATE) * 1000;
+
+/**
+ * Trailing silence before an utterance is called finished. The order's budget is
+ * "under 400 ms": longer and the agent feels broken, shorter and it clips people
+ * mid-sentence. 384 ms is 12 whole frames, the largest multiple of the frame
+ * period that fits under the budget, so the real behaviour matches the number we
+ * publish instead of rounding past it.
+ */
+export const DEFAULT_REDEMPTION_MS = 384;
+
+/**
+ * Audio kept from BEFORE speech was detected. Silero fires a frame or two into a
+ * word, so without this the transcript loses the first consonant.
+ */
+const PRE_SPEECH_PAD_MS = 480;
+
+/** Shorter than a syllable; anything under this is a cough, a door, a keyboard. */
+const MIN_SPEECH_MS = 250;
+
+let vadModulePromise = null;
+
+/** Load @ricky0123/vad-web once, on demand. */
+function loadVadModule() {
+ if (!vadModulePromise) vadModulePromise = import('@ricky0123/vad-web');
+ return vadModulePromise;
+}
+
+export class VoiceActivityDetector {
+ /**
+ * @param {object} opts
+ * @param {MediaStream} opts.stream The caller's single mic stream.
+ * @param {AudioContext} opts.audioContext The caller's single audio context.
+ * @param {(frame: Float32Array, probability: number) => void} [opts.onFrame]
+ * Every 512-sample 16 kHz frame, with silero's speech probability.
+ * @param {() => void} [opts.onSpeechStart] Speech probably started.
+ * @param {() => void} [opts.onSpeechRealStart] Speech passed the minimum length.
+ * @param {(audio: Float32Array) => void} [opts.onSpeechEnd]
+ * The finished utterance, pre-roll included, at 16 kHz.
+ * @param {() => void} [opts.onMisfire] Speech started but was too short.
+ * @param {number} [opts.redemptionMs] Trailing silence before end-of-speech.
+ */
+ constructor({
+ stream,
+ audioContext,
+ onFrame,
+ onSpeechStart,
+ onSpeechRealStart,
+ onSpeechEnd,
+ onMisfire,
+ redemptionMs = DEFAULT_REDEMPTION_MS,
+ } = {}) {
+ if (!stream) throw new Error('VoiceActivityDetector requires a MediaStream');
+ if (!audioContext) throw new Error('VoiceActivityDetector requires an AudioContext');
+ this.stream = stream;
+ this.audioContext = audioContext;
+ this.onFrame = onFrame || (() => {});
+ this.onSpeechStart = onSpeechStart || (() => {});
+ this.onSpeechRealStart = onSpeechRealStart || (() => {});
+ this.onSpeechEnd = onSpeechEnd || (() => {});
+ this.onMisfire = onMisfire || (() => {});
+ this.redemptionMs = redemptionMs;
+ this._vad = null;
+ this._running = false;
+ this._destroyed = false;
+ /** Wall clock of the last frame that carried speech, for the end-of-speech measurement. */
+ this.lastSpeechFrameAt = 0;
+ }
+
+ get running() {
+ return this._running;
+ }
+
+ /** Load the model and begin reading the caller's stream. */
+ async start() {
+ if (this._destroyed) throw new Error('VoiceActivityDetector was destroyed');
+ if (this._vad) {
+ await this._vad.start();
+ this._running = true;
+ return;
+ }
+
+ const { MicVAD } = await loadVadModule();
+
+ this._vad = await MicVAD.new({
+ model: 'v5',
+ baseAssetPath: VAD_ASSET_PATH,
+ onnxWASMBasePath: VAD_ASSET_PATH,
+ // The caller owns the mic. Handing MicVAD the existing stream and
+ // context is what keeps this to one getUserMedia for the whole loop.
+ getStream: async () => this.stream,
+ resumeStream: async () => this.stream,
+ // Never stop the caller's tracks on pause: mute owns that, and a
+ // paused VAD must not silently release a mic the capture still needs.
+ pauseStream: async () => {},
+ audioContext: this.audioContext,
+ startOnLoad: false,
+ redemptionMs: this.redemptionMs,
+ preSpeechPadMs: PRE_SPEECH_PAD_MS,
+ minSpeechMs: MIN_SPEECH_MS,
+ // Silero's own defaults, restated so a future tuning pass has one place
+ // to argue with rather than an invisible import.
+ positiveSpeechThreshold: 0.3,
+ negativeSpeechThreshold: 0.25,
+ onFrameProcessed: ({ isSpeech }, frame) => {
+ if (isSpeech >= 0.3) this.lastSpeechFrameAt = now();
+ this.onFrame(frame, isSpeech);
+ },
+ onSpeechStart: () => this.onSpeechStart(),
+ onSpeechRealStart: () => this.onSpeechRealStart(),
+ onSpeechEnd: (audio) => this.onSpeechEnd(audio),
+ onVADMisfire: () => this.onMisfire(),
+ ortConfig: (ort) => {
+ ort.env.logLevel = 'error';
+ // One thread: the loop shares the machine with a WebGL scene, and
+ // cross-origin isolation (which multi-threaded wasm needs) is not
+ // something a page embedding third-party avatars can assume.
+ ort.env.wasm.numThreads = 1;
+ },
+ });
+
+ await this._vad.start();
+ this._running = true;
+ }
+
+ /** Stop reading frames. The caller's mic tracks are left alone. */
+ async pause() {
+ if (!this._vad || !this._running) return;
+ this._running = false;
+ try {
+ await this._vad.pause();
+ } catch (err) {
+ log.warn('[vad] pause failed', err?.message);
+ }
+ }
+
+ /** Release the model and the worklet. Idempotent. */
+ async destroy() {
+ this._destroyed = true;
+ this._running = false;
+ if (!this._vad) return;
+ const vad = this._vad;
+ this._vad = null;
+ try {
+ await vad.destroy();
+ } catch (err) {
+ log.warn('[vad] destroy failed', err?.message);
+ }
+ }
+}
+
+/** Monotonic where available so a clock change cannot corrupt a latency number. */
+export function now() {
+ return typeof performance !== 'undefined' && performance.now ? performance.now() : Date.now();
+}
+
+/**
+ * Float32 [-1, 1] at 16 kHz to a 16-bit mono RIFF/WAVE buffer, the one format
+ * /api/asr documents as accepted without a sample-rate query parameter.
+ */
+export function float32ToWav(samples, sampleRate = VAD_SAMPLE_RATE) {
+ const bytesPerSample = 2;
+ const dataBytes = samples.length * bytesPerSample;
+ const buffer = new ArrayBuffer(44 + dataBytes);
+ const view = new DataView(buffer);
+ const ascii = (offset, str) => {
+ for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i));
+ };
+ ascii(0, 'RIFF');
+ view.setUint32(4, 36 + dataBytes, true);
+ ascii(8, 'WAVE');
+ ascii(12, 'fmt ');
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true); // PCM
+ view.setUint16(22, 1, true); // mono
+ view.setUint32(24, sampleRate, true);
+ view.setUint32(28, sampleRate * bytesPerSample, true);
+ view.setUint16(32, bytesPerSample, true);
+ view.setUint16(34, 16, true);
+ ascii(36, 'data');
+ view.setUint32(40, dataBytes, true);
+ let offset = 44;
+ for (let i = 0; i < samples.length; i++, offset += 2) {
+ const s = Math.max(-1, Math.min(1, samples[i]));
+ view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+ }
+ return buffer;
+}