Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(voice): serve the wake word and the VAD from three.ws itself
nirholasd2b8da8d82b2
diff --git a/package-lock.json b/package-lock.jsonindex d02ca928d..193828407 100644--- a/package-lock.json+++ b/package-lock.json@@ -124,6 +124,7 @@"@pump-fun/pump-sdk": "^1.36.0","@pump-fun/pump-swap-sdk": "^1.19.0","@pythnetwork/hermes-client": "^3.1.0",+ "@ricky0123/vad-web": "^0.0.30","@scure/base": "^1.2.6","@solana-mobile/mobile-wallet-adapter-protocol-web3js": "^2.2.9","@solana-program/compute-budget": "^0.15.0",@@ -12643,6 +12644,15 @@"node": ">= 10"}},+ "node_modules/@ricky0123/vad-web": {+ "version": "0.0.30",+ "resolved": "https://registry.npmjs.org/@ricky0123/vad-web/-/vad-web-0.0.30.tgz",+ "integrity": "sha512-cJyYrh4YeeUBJcbR9Bic/bFDyB9qBkAepvpuWM3vLxnAi7bC3VHzf51UeNdT+OtY4D7MLAgV8iJMc4z41ZnaWg==",+ "license": "ISC",+ "dependencies": {+ "onnxruntime-web": "^1.17.0"+ }+ },"node_modules/@rolldown/pluginutils": {"version": "1.0.0-beta.27","dev": true,@@ -40357,6 +40367,7 @@"version": "0.1.0","license": "Apache-2.0","dependencies": {+ "@leeoniya/ufuzzy": "^1.0.19","home-assistant-js-websocket": "^9.6.0"},"engines": {diff --git a/package.json b/package.jsonindex 59a13c999..0c15f5a1b 100644--- a/package.json+++ b/package.json@@ -142,7 +142,7 @@]},"scripts": {- "postinstall": "(node scripts/build-cache.mjs check agent-payments-sdk agent-payments-sdk/src agent-payments-sdk/package.json || (npm run build --prefix agent-payments-sdk && node scripts/build-cache.mjs stamp agent-payments-sdk agent-payments-sdk/src agent-payments-sdk/package.json)) && (node scripts/build-cache.mjs check solana-agent-sdk solana-agent-sdk/src solana-agent-sdk/package.json || (npm run build:dist --prefix solana-agent-sdk && node scripts/build-cache.mjs stamp solana-agent-sdk solana-agent-sdk/src solana-agent-sdk/package.json)) && (node scripts/build-cache.mjs check walk-sdk walk-sdk/src walk-sdk/package.json || (npm run build --prefix walk-sdk && node scripts/build-cache.mjs stamp walk-sdk walk-sdk/src walk-sdk/package.json)) && (node scripts/fix-pump-sdk-esm.mjs & node scripts/fix-x402-extensions-esm.mjs & node scripts/fix-zauth-sdk-solana-esm.mjs & node scripts/fix-colyseus-storage.mjs & node scripts/fix-colyseus-send.mjs & node scripts/fix-rapier-init-warning.mjs & node scripts/copy-three-decoders.mjs & node scripts/setup-claude-skills.mjs & node scripts/setup-git-hooks.mjs & node scripts/build-docs-search-index.mjs & wait)",+ "postinstall": "(node scripts/build-cache.mjs check agent-payments-sdk agent-payments-sdk/src agent-payments-sdk/package.json || (npm run build --prefix agent-payments-sdk && node scripts/build-cache.mjs stamp agent-payments-sdk agent-payments-sdk/src agent-payments-sdk/package.json)) && (node scripts/build-cache.mjs check solana-agent-sdk solana-agent-sdk/src solana-agent-sdk/package.json || (npm run build:dist --prefix solana-agent-sdk && node scripts/build-cache.mjs stamp solana-agent-sdk solana-agent-sdk/src solana-agent-sdk/package.json)) && (node scripts/build-cache.mjs check walk-sdk walk-sdk/src walk-sdk/package.json || (npm run build --prefix walk-sdk && node scripts/build-cache.mjs stamp walk-sdk walk-sdk/src walk-sdk/package.json)) && (node scripts/fix-pump-sdk-esm.mjs & node scripts/fix-x402-extensions-esm.mjs & node scripts/fix-zauth-sdk-solana-esm.mjs & node scripts/fix-colyseus-storage.mjs & node scripts/fix-colyseus-send.mjs & node scripts/fix-rapier-init-warning.mjs & node scripts/copy-three-decoders.mjs & node scripts/copy-voice-models.mjs & node scripts/setup-claude-skills.mjs & node scripts/setup-git-hooks.mjs & node scripts/build-docs-search-index.mjs & wait)","setup": "node scripts/setup-dev.mjs","dev": "vite --port 3000","dev:multi": "npm run dev --prefix multiplayer",@@ -259,7 +259,7 @@"triage:gcp:deep": "node scripts/gcp-triage.mjs --deep --since 6h","snapshot": "node scripts/page-snapshot.mjs","build:news": "node scripts/build-news.mjs",- "prebuild": "node scripts/copy-three-decoders.mjs && (node scripts/build-news.mjs & node scripts/build-skill-metadata.mjs & node scripts/build-local-skill-packs.mjs & node scripts/build-discovery-cards.mjs & node scripts/build-guards.mjs & node scripts/build-atlas-index.mjs & node scripts/build-mcp-catalog.mjs & npm run build:examples & npm run build:docs-search & npm run build:club-assets & npm run build:walk-environments & wait) && node scripts/inject-blog-seo.mjs --write && node scripts/inject-hidden-guard.mjs --write && (node scripts/build-page-index.mjs && node scripts/audit-page-index.mjs --strict && node scripts/audit-empty-handlers.mjs && node scripts/audit-hidden-guard.mjs) && node scripts/i18n-annotate.mjs --apply --pages=\"public/news/*.html\" && node scripts/inject-seo-meta.mjs --write && node scripts/inject-theme-boot.mjs --write",+ "prebuild": "node scripts/copy-three-decoders.mjs && node scripts/copy-voice-models.mjs && (node scripts/build-news.mjs & node scripts/build-skill-metadata.mjs & node scripts/build-local-skill-packs.mjs & node scripts/build-discovery-cards.mjs & node scripts/build-guards.mjs & node scripts/build-atlas-index.mjs & node scripts/build-mcp-catalog.mjs & npm run build:examples & npm run build:docs-search & npm run build:club-assets & npm run build:walk-environments & wait) && node scripts/inject-blog-seo.mjs --write && node scripts/inject-hidden-guard.mjs --write && (node scripts/build-page-index.mjs && node scripts/audit-page-index.mjs --strict && node scripts/audit-empty-handlers.mjs && node scripts/audit-hidden-guard.mjs) && node scripts/i18n-annotate.mjs --apply --pages=\"public/news/*.html\" && node scripts/inject-seo-meta.mjs --write && node scripts/inject-theme-boot.mjs --write","seo:meta": "node scripts/inject-blog-seo.mjs --write && node scripts/build-page-index.mjs && node scripts/inject-seo-meta.mjs --write","build": "NODE_OPTIONS='--no-deprecation --max-old-space-size=6144' vite build && node scripts/strip-sw-from-embeds.mjs && node scripts/inject-tour-boot.mjs && node scripts/inject-atlas.mjs && node scripts/inject-inline-behaviors.mjs","build:wasm": "node scripts/build-wasm.mjs",@@ -472,6 +472,7 @@"@pump-fun/pump-sdk": "^1.36.0","@pump-fun/pump-swap-sdk": "^1.19.0","@pythnetwork/hermes-client": "^3.1.0",+ "@ricky0123/vad-web": "^0.0.30","@scure/base": "^1.2.6","@solana-mobile/mobile-wallet-adapter-protocol-web3js": "^2.2.9","@solana-program/compute-budget": "^0.15.0",diff --git a/public/models/voice/wake-word/alexa_v0.1.onnx b/public/models/voice/wake-word/alexa_v0.1.onnxnew file mode 100644index 000000000..984ec2cbbBinary files /dev/null and b/public/models/voice/wake-word/alexa_v0.1.onnx differdiff --git a/public/models/voice/wake-word/embedding_model.onnx b/public/models/voice/wake-word/embedding_model.onnxnew file mode 100644index 000000000..afde53ef5Binary files /dev/null and b/public/models/voice/wake-word/embedding_model.onnx differdiff --git a/public/models/voice/wake-word/hey_jarvis_v0.1.onnx b/public/models/voice/wake-word/hey_jarvis_v0.1.onnxnew file mode 100644index 000000000..371aa73e2Binary files /dev/null and b/public/models/voice/wake-word/hey_jarvis_v0.1.onnx differdiff --git a/public/models/voice/wake-word/hey_mycroft_v0.1.onnx b/public/models/voice/wake-word/hey_mycroft_v0.1.onnxnew file mode 100644index 000000000..7305ca442Binary files /dev/null and b/public/models/voice/wake-word/hey_mycroft_v0.1.onnx differdiff --git a/public/models/voice/wake-word/hey_rhasspy_v0.1.onnx b/public/models/voice/wake-word/hey_rhasspy_v0.1.onnxnew file mode 100644index 000000000..914d4f85eBinary files /dev/null and b/public/models/voice/wake-word/hey_rhasspy_v0.1.onnx differdiff --git a/public/models/voice/wake-word/melspectrogram.onnx b/public/models/voice/wake-word/melspectrogram.onnxnew file mode 100644index 000000000..a3a60354eBinary files /dev/null and b/public/models/voice/wake-word/melspectrogram.onnx differdiff --git a/scripts/copy-voice-models.mjs b/scripts/copy-voice-models.mjsnew file mode 100644index 000000000..47045f7e8--- /dev/null+++ b/scripts/copy-voice-models.mjs@@ -0,0 +1,64 @@+// Stage the browser voice-loop runtime into /public/models/voice/runtime/ so the+// wake word and the VAD load every byte from three.ws itself.+//+// Why same-origin matters here and not only as a preference: this is an+// always-on microphone feature. A user who opts in is trusting us with a live+// mic, and a third-party CDN fetch on that path would put a stranger's server in+// the room. Serving the models ourselves means the audio path has exactly one+// origin, which is the claim the consent copy makes.+//+// What is staged (all from node_modules, so the lockfile pins the versions):+// silero_vad_v5.onnx @ricky0123/vad-web — the MIT silero VAD+// vad.worklet.bundle.min.js @ricky0123/vad-web — its AudioWorklet+// ort-wasm-simd-threaded.mjs onnxruntime-web — the wasm loader+// ort-wasm-simd-threaded.wasm onnxruntime-web — the runtime itself+//+// NOT staged here: the openWakeWord models under /public/models/voice/wake-word.+// Those are Apache-2.0 release artifacts from dscripka/openWakeWord v0.5.1 that+// no npm package ships, so they are committed as bytes rather than fetched at+// build time. A build that reaches out to GitHub is a build that fails when+// GitHub does.+//+// Runs from postinstall and prebuild, exactly like copy-three-decoders.mjs.+// /public/models/voice/runtime/ is gitignored; dist/ gets it from prebuild.++import { copyFileSync, existsSync, mkdirSync, statSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { fileURLToPath } from 'node:url';++const __dirname = dirname(fileURLToPath(import.meta.url));+const repo = resolve(__dirname, '..');+const out = join(repo, 'public/models/voice/runtime');++const files = [+ ['node_modules/@ricky0123/vad-web/dist/silero_vad_v5.onnx', 'silero_vad_v5.onnx'],+ ['node_modules/@ricky0123/vad-web/dist/vad.worklet.bundle.min.js', 'vad.worklet.bundle.min.js'],+ ['node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.mjs', 'ort-wasm-simd-threaded.mjs'],+ ['node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.wasm', 'ort-wasm-simd-threaded.wasm'],+];++const missing = files.filter(([from]) => !existsSync(join(repo, from)));+if (missing.length === files.length) {+ console.warn('[copy-voice-models] no source files found — skipping. Run npm install first.');+ process.exit(0);+}++mkdirSync(out, { recursive: true });++let staged = 0;+for (const [from, name] of files) {+ const src = join(repo, from);+ if (!existsSync(src)) {+ console.warn(`[copy-voice-models] ${from} not found`);+ continue;+ }+ const dest = join(out, name);+ // Skip an identical restage so a repeated prebuild does not rewrite 11 MB.+ if (existsSync(dest) && statSync(dest).size === statSync(src).size) {+ staged++;+ continue;+ }+ copyFileSync(src, dest);+ staged++;+}+console.log(`[copy-voice-models] ${staged}/${files.length} → public/models/voice/runtime`);diff --git a/src/voice/vad.js b/src/voice/vad.jsnew file mode 100644index 000000000..be747bf84--- /dev/null+++ b/src/voice/vad.js@@ -0,0 +1,226 @@+/**+ * Voice activity detection for the browser voice loop.+ *+ * A thin, lazily loaded wrapper over silero-vad (MIT,+ * https://github.com/snakers4/silero-vad) through @ricky0123/vad-web. We do not+ * write a detector: silero is a trained model with years of tuning behind it and+ * an energy threshold is not a substitute.+ *+ * Two things this wrapper adds over calling MicVAD directly, and both are+ * load-bearing for the home voice loop:+ *+ * 1. It never opens the microphone itself. The caller passes the MediaStream+ * and the AudioContext it already owns, so the VAD, the wake word and the+ * utterance capture all read ONE mic. Three getUserMedia calls would mean+ * three indicator dots in the browser chrome and three tracks to remember to+ * stop, and "mute" has to stop every track that exists.+ * 2. It surfaces every 512-sample, 16 kHz frame to the caller. The wake word+ * needs exactly that feed, and running a second resampler for it would+ * double the audio work for no gain.+ *+ * Everything here loads on demand. Nothing in this module's dependency graph is+ * fetched until start() is called, which is what makes "nothing about listening+ * loads before you opt in" a true statement rather than a promise.+ *+ * Model: silero v5, 512-sample frames at 16 kHz, so 32 ms per frame. The v5+ * frame is the reason we do not use the legacy model: its 1536-sample frame is+ * 96 ms, which is a quarter of the entire end-of-speech budget in granularity+ * alone.+ */++import { log } from '../shared/log.js';++/** Where copy-voice-models.mjs stages the VAD model and its worklet. */+export const VAD_ASSET_PATH = '/models/voice/runtime/';++/** Silero v5 reads 512 samples at a time; the loop's timing math depends on it. */+export const VAD_FRAME_SAMPLES = 512;+export const VAD_SAMPLE_RATE = 16000;+export const VAD_MS_PER_FRAME = (VAD_FRAME_SAMPLES / VAD_SAMPLE_RATE) * 1000;++/**+ * Trailing silence before an utterance is called finished. The order's budget is+ * "under 400 ms": longer and the agent feels broken, shorter and it clips people+ * mid-sentence. 384 ms is 12 whole frames, the largest multiple of the frame+ * period that fits under the budget, so the real behaviour matches the number we+ * publish instead of rounding past it.+ */+export const DEFAULT_REDEMPTION_MS = 384;++/**+ * Audio kept from BEFORE speech was detected. Silero fires a frame or two into a+ * word, so without this the transcript loses the first consonant.+ */+const PRE_SPEECH_PAD_MS = 480;++/** Shorter than a syllable; anything under this is a cough, a door, a keyboard. */+const MIN_SPEECH_MS = 250;++let vadModulePromise = null;++/** Load @ricky0123/vad-web once, on demand. */+function loadVadModule() {+ if (!vadModulePromise) vadModulePromise = import('@ricky0123/vad-web');+ return vadModulePromise;+}++export class VoiceActivityDetector {+ /**+ * @param {object} opts+ * @param {MediaStream} opts.stream The caller's single mic stream.+ * @param {AudioContext} opts.audioContext The caller's single audio context.+ * @param {(frame: Float32Array, probability: number) => void} [opts.onFrame]+ * Every 512-sample 16 kHz frame, with silero's speech probability.+ * @param {() => void} [opts.onSpeechStart] Speech probably started.+ * @param {() => void} [opts.onSpeechRealStart] Speech passed the minimum length.+ * @param {(audio: Float32Array) => void} [opts.onSpeechEnd]+ * The finished utterance, pre-roll included, at 16 kHz.+ * @param {() => void} [opts.onMisfire] Speech started but was too short.+ * @param {number} [opts.redemptionMs] Trailing silence before end-of-speech.+ */+ constructor({+ stream,+ audioContext,+ onFrame,+ onSpeechStart,+ onSpeechRealStart,+ onSpeechEnd,+ onMisfire,+ redemptionMs = DEFAULT_REDEMPTION_MS,+ } = {}) {+ if (!stream) throw new Error('VoiceActivityDetector requires a MediaStream');+ if (!audioContext) throw new Error('VoiceActivityDetector requires an AudioContext');+ this.stream = stream;+ this.audioContext = audioContext;+ this.onFrame = onFrame || (() => {});+ this.onSpeechStart = onSpeechStart || (() => {});+ this.onSpeechRealStart = onSpeechRealStart || (() => {});+ this.onSpeechEnd = onSpeechEnd || (() => {});+ this.onMisfire = onMisfire || (() => {});+ this.redemptionMs = redemptionMs;+ this._vad = null;+ this._running = false;+ this._destroyed = false;+ /** Wall clock of the last frame that carried speech, for the end-of-speech measurement. */+ this.lastSpeechFrameAt = 0;+ }++ get running() {+ return this._running;+ }++ /** Load the model and begin reading the caller's stream. */+ async start() {+ if (this._destroyed) throw new Error('VoiceActivityDetector was destroyed');+ if (this._vad) {+ await this._vad.start();+ this._running = true;+ return;+ }++ const { MicVAD } = await loadVadModule();++ this._vad = await MicVAD.new({+ model: 'v5',+ baseAssetPath: VAD_ASSET_PATH,+ onnxWASMBasePath: VAD_ASSET_PATH,+ // The caller owns the mic. Handing MicVAD the existing stream and+ // context is what keeps this to one getUserMedia for the whole loop.+ getStream: async () => this.stream,+ resumeStream: async () => this.stream,+ // Never stop the caller's tracks on pause: mute owns that, and a+ // paused VAD must not silently release a mic the capture still needs.+ pauseStream: async () => {},+ audioContext: this.audioContext,+ startOnLoad: false,+ redemptionMs: this.redemptionMs,+ preSpeechPadMs: PRE_SPEECH_PAD_MS,+ minSpeechMs: MIN_SPEECH_MS,+ // Silero's own defaults, restated so a future tuning pass has one place+ // to argue with rather than an invisible import.+ positiveSpeechThreshold: 0.3,+ negativeSpeechThreshold: 0.25,+ onFrameProcessed: ({ isSpeech }, frame) => {+ if (isSpeech >= 0.3) this.lastSpeechFrameAt = now();+ this.onFrame(frame, isSpeech);+ },+ onSpeechStart: () => this.onSpeechStart(),+ onSpeechRealStart: () => this.onSpeechRealStart(),+ onSpeechEnd: (audio) => this.onSpeechEnd(audio),+ onVADMisfire: () => this.onMisfire(),+ ortConfig: (ort) => {+ ort.env.logLevel = 'error';+ // One thread: the loop shares the machine with a WebGL scene, and+ // cross-origin isolation (which multi-threaded wasm needs) is not+ // something a page embedding third-party avatars can assume.+ ort.env.wasm.numThreads = 1;+ },+ });++ await this._vad.start();+ this._running = true;+ }++ /** Stop reading frames. The caller's mic tracks are left alone. */+ async pause() {+ if (!this._vad || !this._running) return;+ this._running = false;+ try {+ await this._vad.pause();+ } catch (err) {+ log.warn('[vad] pause failed', err?.message);+ }+ }++ /** Release the model and the worklet. Idempotent. */+ async destroy() {+ this._destroyed = true;+ this._running = false;+ if (!this._vad) return;+ const vad = this._vad;+ this._vad = null;+ try {+ await vad.destroy();+ } catch (err) {+ log.warn('[vad] destroy failed', err?.message);+ }+ }+}++/** Monotonic where available so a clock change cannot corrupt a latency number. */+export function now() {+ return typeof performance !== 'undefined' && performance.now ? performance.now() : Date.now();+}++/**+ * Float32 [-1, 1] at 16 kHz to a 16-bit mono RIFF/WAVE buffer, the one format+ * /api/asr documents as accepted without a sample-rate query parameter.+ */+export function float32ToWav(samples, sampleRate = VAD_SAMPLE_RATE) {+ const bytesPerSample = 2;+ const dataBytes = samples.length * bytesPerSample;+ const buffer = new ArrayBuffer(44 + dataBytes);+ const view = new DataView(buffer);+ const ascii = (offset, str) => {+ for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i));+ };+ ascii(0, 'RIFF');+ view.setUint32(4, 36 + dataBytes, true);+ ascii(8, 'WAVE');+ ascii(12, 'fmt ');+ view.setUint32(16, 16, true);+ view.setUint16(20, 1, true); // PCM+ view.setUint16(22, 1, true); // mono+ view.setUint32(24, sampleRate, true);+ view.setUint32(28, sampleRate * bytesPerSample, true);+ view.setUint16(32, bytesPerSample, true);+ view.setUint16(34, 16, true);+ ascii(36, 'data');+ view.setUint32(40, dataBytes, true);+ let offset = 44;+ for (let i = 0; i < samples.length; i++, offset += 2) {+ const s = Math.max(-1, Math.min(1, samples[i]));+ view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);+ }+ return buffer;+}