Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
refactor: keep hands-free VAD detector state on its capture attempt (#12404)
[._.]/ Adam Eivy01073d905f40
diff --git a/client/src/services/voiceClient.js b/client/src/services/voiceClient.jsindex f394068708..59b20b86fe 100644--- a/client/src/services/voiceClient.js+++ b/client/src/services/voiceClient.js@@ -780,28 +780,38 @@ const VAD = {debug: false, // window.__portosVadDebug = true to enable logging};-// Live thresholds (set by calibration, then static for the session).-let onRms = VAD.minOnRms;-let offRms = VAD.minOffRms;-let calibrating = false;-let calibrationSamples = [];-let calibrationUntil = 0;-+// Module-level on purpose: a display reading for the widget's level meter and+// a log throttle — neither is detection state, so neither belongs to an attempt.let lastRmsValue = 0;let lastDebugLogAt = 0;let continuousAttempt = null;-let vadState = 'idle';-let speechChunks = [];-// Fixed-size ring buffer of the last `preRollLimit` frames; avoids the-// O(n) Array.shift() that would otherwise run ~375 times/second.-let preRoll = null;-let preRollLimit = 0;-let preRollIdx = 0;-let preRollFilled = 0;-let silenceStartedAt = 0;-let speechStartedAt = 0;-let onsetFrames = 0;++// Detector state for ONE capture attempt (`attempt.vad`). This is the single+// list of starting values; a new attempt gets a fresh record and a detached one+// drops it, so nothing carries over between hands-free sessions.+const createVadState = (sampleRate) => {+ const preRollLimit = Math.max(1, Math.ceil((VAD.preRollMs / 1000) * sampleRate / 128));+ return {+ // Live thresholds (set by calibration, then static for the session).+ onRms: VAD.minOnRms,+ offRms: VAD.minOffRms,+ calibrating: true,+ calibrationSamples: [],+ calibrationUntil: performance.now() + VAD.calibrationMs,+ vadState: 'idle',+ speechChunks: [],+ // Fixed-size ring buffer of the last `preRollLimit` frames; avoids the+ // O(n) Array.shift() that would otherwise run ~375 times/second.+ preRoll: new Array(preRollLimit),+ preRollLimit,+ preRollIdx: 0,+ preRollFilled: 0,+ silenceStartedAt: 0,+ speechStartedAt: 0,+ onsetFrames: 0,+ };+};const WORKLET_SOURCE = `class VADProcessor extends AudioWorkletProcessor {@@ -824,17 +834,6 @@ const settleContinuousStart = (attempt) => {attempt.pending = false;};-const resetContinuousVad = () => {- speechChunks = [];- preRoll = null;- preRollIdx = 0;- preRollFilled = 0;- vadState = 'idle';- onsetFrames = 0;- calibrating = false;- calibrationSamples = [];-};-// Detach before any asynchronous close. A stale startup may still receive a// stream later, but it can release only resources on its own attempt.const detachContinuousAttempt = (attempt) => {@@ -843,7 +842,7 @@ const detachContinuousAttempt = (attempt) => {settleContinuousStart(attempt);if (continuousAttempt === attempt) {continuousAttempt = null;- resetContinuousVad();+ attempt.vad = null;}return attempt;};@@ -866,9 +865,10 @@ const releaseContinuousResources = (attempt) => {const submitUtterance = async (capture) => {if (!isCurrentContinuousAttempt(capture)) return;- if (!speechChunks.length) return;- const chunksToSubmit = speechChunks;- speechChunks = [];+ const { vad } = capture;+ if (!vad.speechChunks.length) return;+ const chunksToSubmit = vad.speechChunks;+ vad.speechChunks = [];const total = chunksToSubmit.reduce((n, c) => n + c.length, 0);const samples = new Float32Array(total);@@ -891,21 +891,21 @@ const submitUtterance = async (capture) => {capture.callbacks?.onSubmit?.({ submitted: true, peak, size: wav.byteLength });};-const finishCalibration = () => {- calibrating = false;- const samples = calibrationSamples;- calibrationSamples = [];+const finishCalibration = (vad) => {+ vad.calibrating = false;+ const samples = vad.calibrationSamples;+ vad.calibrationSamples = [];if (!samples.length) return;samples.sort((a, b) => a - b);// Median is robust if the user accidentally speaks during calibration —// up to ~50% of frames can be speech without poisoning the floor estimate.const floor = samples[Math.floor(samples.length * 0.5)] ?? samples[samples.length - 1];- offRms = Math.max(VAD.minOffRms, floor * 2.0);- onRms = Math.min(VAD.maxOnRms, Math.max(VAD.minOnRms, floor * 4.0, offRms * 2));- console.log(`🎙️ [vad] calibrated floor=${floor.toFixed(4)} → on=${onRms.toFixed(4)} off=${offRms.toFixed(4)}`);+ vad.offRms = Math.max(VAD.minOffRms, floor * 2.0);+ vad.onRms = Math.min(VAD.maxOnRms, Math.max(VAD.minOnRms, floor * 4.0, vad.offRms * 2));+ console.log(`🎙️ [vad] calibrated floor=${floor.toFixed(4)} → on=${vad.onRms.toFixed(4)} off=${vad.offRms.toFixed(4)}`);};-const snapshotPreRoll = () => {+const snapshotPreRoll = ({ preRoll, preRollLimit, preRollIdx, preRollFilled }) => {if (!preRollFilled) return [];const out = new Array(preRollFilled);const start = preRollFilled < preRollLimit ? 0 : preRollIdx;@@ -913,6 +913,13 @@ const snapshotPreRoll = () => {return out;};+// Speech -> idle transition shared by the watchdog and end-of-speech paths.+const returnToIdle = (vad) => {+ vad.vadState = 'idle';+ vad.silenceStartedAt = 0;+ vad.onsetFrames = 0;+};+const handleFrame = (frame, capture) => {if (!isCurrentContinuousAttempt(capture)) return;let sum = 0;@@ -920,64 +927,63 @@ const handleFrame = (frame, capture) => {const rms = Math.sqrt(sum / frame.length);lastRmsValue = rms;- preRoll[preRollIdx] = frame;- preRollIdx = (preRollIdx + 1) % preRollLimit;- if (preRollFilled < preRollLimit) preRollFilled += 1;+ const { vad } = capture;+ vad.preRoll[vad.preRollIdx] = frame;+ vad.preRollIdx = (vad.preRollIdx + 1) % vad.preRollLimit;+ if (vad.preRollFilled < vad.preRollLimit) vad.preRollFilled += 1;const now = performance.now();- if (calibrating) {- calibrationSamples.push(rms);- if (now >= calibrationUntil) finishCalibration();+ if (vad.calibrating) {+ vad.calibrationSamples.push(rms);+ if (now >= vad.calibrationUntil) finishCalibration(vad);else return;}if (VAD.debug && now - lastDebugLogAt > 250) {lastDebugLogAt = now;- console.log(`[vad] state=${vadState} rms=${rms.toFixed(4)} on=${onRms.toFixed(4)} off=${offRms.toFixed(4)} tts=${isTtsActive()}`);+ console.log(`[vad] state=${vad.vadState} rms=${rms.toFixed(4)} on=${vad.onRms.toFixed(4)} off=${vad.offRms.toFixed(4)} tts=${isTtsActive()}`);}- if (vadState === 'idle') {+ if (vad.vadState === 'idle') {// Raise the bar while TTS plays AND during the echo tail so neither the// bot's live audio nor reverb through the mic trigger a false barge-in.// Skip the bump entirely on a confirmed headset — there's no acoustic// path from speaker to mic, so the user can interrupt at normal volume.const inEcho = isInTtsEchoWindow() && !audioRoute.likelyHeadset;- const effectiveOnRms = inEcho ? onRms * VAD.bargeInMul : onRms;+ const effectiveOnRms = inEcho ? vad.onRms * VAD.bargeInMul : vad.onRms;const effectiveConfirmMs = inEcho ? VAD.bargeInOnsetConfirmMs : VAD.onsetConfirmMs;if (rms > effectiveOnRms) {- onsetFrames += 1;+ vad.onsetFrames += 1;const frameMs = (frame.length / (capture.context?.sampleRate || 48000)) * 1000;- if (onsetFrames * frameMs >= effectiveConfirmMs) {+ if (vad.onsetFrames * frameMs >= effectiveConfirmMs) {// Confirmed speech onset — barge-in + start capturing- vadState = 'speaking';- speechStartedAt = now;- silenceStartedAt = 0;- onsetFrames = 0;+ vad.vadState = 'speaking';+ vad.speechStartedAt = now;+ vad.silenceStartedAt = 0;+ vad.onsetFrames = 0;if (isTtsActive()) {socket.emit('voice:interrupt');cancelVoicePlayback();}- speechChunks = snapshotPreRoll();+ vad.speechChunks = snapshotPreRoll(vad);capture.callbacks?.onSpeechStart?.();}} else {- onsetFrames = 0;+ vad.onsetFrames = 0;}return;}// state === 'speaking'- speechChunks.push(frame);+ vad.speechChunks.push(frame);// Watchdog: if silence never crosses offRms (e.g. noisy env, echo tail,// stuck mic), force the turn to end so VAD doesn't jam here forever.- if (now - speechStartedAt >= VAD.maxSpeechMs) {- vadState = 'idle';- silenceStartedAt = 0;- onsetFrames = 0;+ if (now - vad.speechStartedAt >= VAD.maxSpeechMs) {+ returnToIdle(vad);capture.callbacks?.onSpeechEnd?.();- if (speechChunks.length) {+ if (vad.speechChunks.length) {submitUtterance(capture).catch((err) => console.warn('[voice] watchdog submit failed:', err));} else {capture.callbacks?.onSubmit?.({ submitted: false, peak: 0, discarded: true });@@ -985,25 +991,23 @@ const handleFrame = (frame, capture) => {return;}- if (rms < offRms) {- if (!silenceStartedAt) silenceStartedAt = now;- if (now - silenceStartedAt >= VAD.silenceMs) {- const speechMs = silenceStartedAt - speechStartedAt;- vadState = 'idle';- silenceStartedAt = 0;- onsetFrames = 0;+ if (rms < vad.offRms) {+ if (!vad.silenceStartedAt) vad.silenceStartedAt = now;+ if (now - vad.silenceStartedAt >= VAD.silenceMs) {+ const speechMs = vad.silenceStartedAt - vad.speechStartedAt;+ returnToIdle(vad);capture.callbacks?.onSpeechEnd?.();if (speechMs >= VAD.minSpeechMs) {submitUtterance(capture).catch((err) => console.warn('[voice] submit failed:', err));} else {// Too short to submit — notify the widget so it leaves 'thinking'// instead of waiting for a server response that won't arrive.- speechChunks = [];+ vad.speechChunks = [];capture.callbacks?.onSubmit?.({ submitted: false, peak: 0, discarded: true });}}} else {- silenceStartedAt = 0;+ vad.silenceStartedAt = 0;}};@@ -1019,6 +1023,7 @@ export const startContinuous = async (callbacks = {}) => {context: null,source: null,workletNode: null,+ vad: null,};continuousAttempt = attempt;@@ -1064,21 +1069,7 @@ export const startContinuous = async (callbacks = {}) => {attempt.source = attempt.context.createMediaStreamSource(attempt.stream);attempt.workletNode = new AudioWorkletNode(attempt.context, 'vad-processor');- const sampleRate = attempt.context.sampleRate;- preRollLimit = Math.max(1, Math.ceil((VAD.preRollMs / 1000) * sampleRate / 128));- preRoll = new Array(preRollLimit);- preRollIdx = 0;- preRollFilled = 0;- speechChunks = [];- vadState = 'idle';- silenceStartedAt = 0;- speechStartedAt = 0;- onsetFrames = 0;- calibrating = true;- calibrationSamples = [];- calibrationUntil = performance.now() + VAD.calibrationMs;- onRms = VAD.minOnRms;- offRms = VAD.minOffRms;+ attempt.vad = createVadState(attempt.context.sampleRate);attempt.workletNode.port.onmessage = (e) => handleFrame(e.data, attempt);attempt.source.connect(attempt.workletNode);diff --git a/client/src/services/voiceClient.test.js b/client/src/services/voiceClient.test.jsindex 015afd4a12..fe96f42860 100644--- a/client/src/services/voiceClient.test.js+++ b/client/src/services/voiceClient.test.js@@ -717,6 +717,33 @@ describe('voice playback cancellation', () => {},);+ it('starts a restarted capture calibrating and idle after the previous one stopped mid-speech', async () => {+ let now = 1000;+ vi.spyOn(performance, 'now').mockImplementation(() => now);+ const loud = () => new Float32Array(128).fill(0.5);+ const onFirstSpeech = vi.fn();+ await voiceClient.startContinuous({ onSpeechStart: onFirstSpeech });+ const firstFrame = FakeAudioWorkletNode.last.port.onmessage;+ now = 1700;+ firstFrame({ data: new Float32Array(128) });+ for (let i = 0; i < 40; i++) firstFrame({ data: loud() });+ expect(onFirstSpeech).toHaveBeenCalledOnce();+ await voiceClient.stopContinuous();++ const onSpeechStart = vi.fn();+ await voiceClient.startContinuous({ onSpeechStart });+ const frame = FakeAudioWorkletNode.last.port.onmessage;+ // Still inside the new capture's calibration window: loud frames only+ // feed the noise-floor estimate, never a speech onset.+ for (let i = 0; i < 40; i++) frame({ data: loud() });+ expect(onSpeechStart).not.toHaveBeenCalled();+ // Past the window the detector is live again, starting from idle.+ now = 1700 + 700;+ frame({ data: new Float32Array(128) });+ for (let i = 0; i < 40; i++) frame({ data: loud() });+ expect(onSpeechStart).toHaveBeenCalledOnce();+ });+it.each(['resolve', 'reject'])('does not stop a retry when a disposed startup later %ss', async (outcome) => {let resolveModule;let rejectModule;