Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(transcription): stricter local whisper speech gate

Gabriel Stein2a441fd7551b
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index 457b0e244..d50d56012 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -5,6 +5,11 @@ import { isBuiltInMicrophone } from "../utils/audioDeviceUtils";
import { isSecureEndpoint } from "../utils/urlUtils";
import { withSessionRefresh } from "../lib/neonAuth";
import { getBaseLanguageCode, validateLanguageForModel } from "../utils/languageSupport";
+import {
+ createLocalSpeechGateState,
+ getLocalSpeechGateDecision,
+ recordLocalSpeechWindow,
+} from "./localSpeechGate";
import {
getSettings,
getEffectiveReasoningModel,
@@ -110,6 +115,7 @@ class AudioManager {
this.sttConfig = null;
this.lastAudioBlob = null;
this.lastAudioMetadata = null;
+ this._localSpeechGateState = null;
}
getWorkletBlobUrl() {
@@ -301,28 +307,30 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
);
}
- // Silence detection: observe audio energy via AnalyserNode
try {
this._silenceCtx = new AudioContext();
this._silenceAnalyser = this._silenceCtx.createAnalyser();
this._silenceAnalyser.fftSize = 2048;
const sourceNode = this._silenceCtx.createMediaStreamSource(micStream);
sourceNode.connect(this._silenceAnalyser);
- this._peakRms = 0;
+ this._localSpeechGateState = createLocalSpeechGateState();
const dataArray = new Uint8Array(this._silenceAnalyser.fftSize);
this._silenceInterval = setInterval(() => {
this._silenceAnalyser.getByteTimeDomainData(dataArray);
let sum = 0;
+ let peak = 0;
for (let i = 0; i < dataArray.length; i++) {
const v = (dataArray[i] - 128) / 128;
sum += v * v;
+ const abs = Math.abs(v);
+ if (abs > peak) peak = abs;
}
const rms = Math.sqrt(sum / dataArray.length);
- if (rms > this._peakRms) this._peakRms = rms;
+ recordLocalSpeechWindow(this._localSpeechGateState, rms, peak);
}, 100);
} catch (e) {
- logger.warn("Silence detection setup failed, skipping", { error: e.message }, "audio");
- this._peakRms = 1; // assume speech if detection fails
+ logger.warn("Audio level gate setup failed, skipping", { error: e.message }, "audio");
+ this._localSpeechGateState = null;
}
this.mediaRecorder = new MediaRecorder(micStream);
@@ -335,7 +343,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
};
this.mediaRecorder.onstop = async () => {
- // Clean up silence detection
if (this._silenceInterval) {
clearInterval(this._silenceInterval);
this._silenceInterval = null;
@@ -473,32 +480,43 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
async processAudio(audioBlob, metadata = {}) {
const pipelineStart = performance.now();
+ const settings = getSettings();
+ const speechGateDecision = getLocalSpeechGateDecision(this._localSpeechGateState);
+ this._localSpeechGateState = null;
- // Skip transcription if recording was silence
- const SILENCE_THRESHOLD = 0.002;
- if (this._peakRms != null && this._peakRms < SILENCE_THRESHOLD) {
+ const shouldUseStrongLocalWhisperGate =
+ settings.useLocalWhisper && settings.localTranscriptionProvider === "whisper";
+ if (
+ speechGateDecision.skip &&
+ (speechGateDecision.reason === "silence" || shouldUseStrongLocalWhisperGate)
+ ) {
logger.info(
- "Silence detected, skipping transcription",
- { peakRms: this._peakRms.toFixed(4), threshold: SILENCE_THRESHOLD },
+ "Speech gate skipped transcription",
+ {
+ reason: speechGateDecision.reason,
+ useLocalWhisper: settings.useLocalWhisper,
+ localProvider: settings.localTranscriptionProvider,
+ peakRms: speechGateDecision.peakRms?.toFixed(4),
+ peakAmplitude: speechGateDecision.peakAmplitude?.toFixed(4),
+ speechWindowCount: speechGateDecision.speechWindowCount,
+ maxConsecutiveSpeechWindows: speechGateDecision.maxConsecutiveSpeechWindows,
+ },
"audio"
);
- this._peakRms = null;
this.isProcessing = false;
this.onStateChange?.({ isRecording: false, isProcessing: false });
this.onTranscriptionComplete?.({ success: true, text: "" });
return;
}
- this._peakRms = null;
try {
- const s = getSettings();
- const useLocalWhisper = s.useLocalWhisper;
- const localProvider = s.localTranscriptionProvider;
- const whisperModel = s.whisperModel;
- const parakeetModel = s.parakeetModel || "parakeet-tdt-0.6b-v3";
+ const useLocalWhisper = settings.useLocalWhisper;
+ const localProvider = settings.localTranscriptionProvider;
+ const whisperModel = settings.whisperModel;
+ const parakeetModel = settings.parakeetModel || "parakeet-tdt-0.6b-v3";
- const cloudTranscriptionMode = s.cloudTranscriptionMode;
- const isSignedIn = s.isSignedIn;
+ const cloudTranscriptionMode = settings.cloudTranscriptionMode;
+ const isSignedIn = settings.isSignedIn;
const isOpenWhisprCloudMode = !useLocalWhisper && cloudTranscriptionMode === "openwhispr";
const useCloud = isOpenWhisprCloudMode && isSignedIn;
diff --git a/src/helpers/localSpeechGate.js b/src/helpers/localSpeechGate.js
new file mode 100644
index 000000000..e999f494d
--- /dev/null
+++ b/src/helpers/localSpeechGate.js
@@ -0,0 +1,68 @@
+const SILENCE_RMS_THRESHOLD = 0.002;
+const SPEECH_WINDOW_RMS_THRESHOLD = 0.0045;
+const SPEECH_WINDOW_PEAK_THRESHOLD = 0.04;
+const STRONG_SPEECH_RMS_THRESHOLD = 0.008;
+const MIN_SPEECH_WINDOWS = 2;
+const MIN_CONSECUTIVE_SPEECH_WINDOWS = 2;
+
+export const createLocalSpeechGateState = () => ({
+ peakRms: 0,
+ peakAmplitude: 0,
+ windowCount: 0,
+ speechWindowCount: 0,
+ consecutiveSpeechWindows: 0,
+ maxConsecutiveSpeechWindows: 0,
+});
+
+export const recordLocalSpeechWindow = (state, rms, peak) => {
+ if (!state) {
+ return null;
+ }
+
+ state.windowCount += 1;
+ state.peakRms = Math.max(state.peakRms, rms);
+ state.peakAmplitude = Math.max(state.peakAmplitude, peak);
+
+ const isSpeechWindow = rms >= SPEECH_WINDOW_RMS_THRESHOLD && peak >= SPEECH_WINDOW_PEAK_THRESHOLD;
+ if (!isSpeechWindow) {
+ state.consecutiveSpeechWindows = 0;
+ return state;
+ }
+
+ state.speechWindowCount += 1;
+ state.consecutiveSpeechWindows += 1;
+ state.maxConsecutiveSpeechWindows = Math.max(
+ state.maxConsecutiveSpeechWindows,
+ state.consecutiveSpeechWindows
+ );
+ return state;
+};
+
+export const getLocalSpeechGateDecision = (state) => {
+ if (!state?.windowCount) {
+ return { skip: false, reason: "unavailable" };
+ }
+
+ const metrics = {
+ peakRms: state.peakRms,
+ peakAmplitude: state.peakAmplitude,
+ windowCount: state.windowCount,
+ speechWindowCount: state.speechWindowCount,
+ maxConsecutiveSpeechWindows: state.maxConsecutiveSpeechWindows,
+ };
+
+ if (state.peakRms < SILENCE_RMS_THRESHOLD) {
+ return { skip: true, reason: "silence", ...metrics };
+ }
+
+ const hasSpeech =
+ state.speechWindowCount >= MIN_SPEECH_WINDOWS ||
+ state.maxConsecutiveSpeechWindows >= MIN_CONSECUTIVE_SPEECH_WINDOWS ||
+ (state.speechWindowCount >= 1 && state.peakRms >= STRONG_SPEECH_RMS_THRESHOLD);
+
+ if (!hasSpeech) {
+ return { skip: true, reason: "insufficient_speech", ...metrics };
+ }
+
+ return { skip: false, reason: "speech_detected", ...metrics };
+};
diff --git a/test/helpers/localSpeechGate.test.js b/test/helpers/localSpeechGate.test.js
new file mode 100644
index 000000000..05abbeb54
--- /dev/null
+++ b/test/helpers/localSpeechGate.test.js
@@ -0,0 +1,71 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+test("treats near silence as skippable", async () => {
+ const {
+ createLocalSpeechGateState,
+ recordLocalSpeechWindow,
+ getLocalSpeechGateDecision,
+ } = await import("../../src/helpers/localSpeechGate.js");
+
+ const state = createLocalSpeechGateState();
+ recordLocalSpeechWindow(state, 0.0012, 0.01);
+ recordLocalSpeechWindow(state, 0.0016, 0.015);
+ recordLocalSpeechWindow(state, 0.0014, 0.012);
+
+ assert.deepEqual(getLocalSpeechGateDecision(state), {
+ skip: true,
+ reason: "silence",
+ peakRms: 0.0016,
+ peakAmplitude: 0.015,
+ windowCount: 3,
+ speechWindowCount: 0,
+ maxConsecutiveSpeechWindows: 0,
+ });
+});
+
+test("rejects isolated noise bursts without sustained speech", async () => {
+ const {
+ createLocalSpeechGateState,
+ recordLocalSpeechWindow,
+ getLocalSpeechGateDecision,
+ } = await import("../../src/helpers/localSpeechGate.js");
+
+ const state = createLocalSpeechGateState();
+ recordLocalSpeechWindow(state, 0.0025, 0.03);
+ recordLocalSpeechWindow(state, 0.0052, 0.045);
+ recordLocalSpeechWindow(state, 0.0028, 0.028);
+
+ const decision = getLocalSpeechGateDecision(state);
+
+ assert.equal(decision.skip, true);
+ assert.equal(decision.reason, "insufficient_speech");
+ assert.equal(decision.peakRms, 0.0052);
+ assert.equal(decision.peakAmplitude, 0.045);
+ assert.equal(decision.windowCount, 3);
+ assert.equal(decision.speechWindowCount, 1);
+ assert.equal(decision.maxConsecutiveSpeechWindows, 1);
+});
+
+test("allows sustained speech-like energy through", async () => {
+ const {
+ createLocalSpeechGateState,
+ recordLocalSpeechWindow,
+ getLocalSpeechGateDecision,
+ } = await import("../../src/helpers/localSpeechGate.js");
+
+ const state = createLocalSpeechGateState();
+ recordLocalSpeechWindow(state, 0.003, 0.025);
+ recordLocalSpeechWindow(state, 0.0056, 0.06);
+ recordLocalSpeechWindow(state, 0.0061, 0.065);
+
+ assert.deepEqual(getLocalSpeechGateDecision(state), {
+ skip: false,
+ reason: "speech_detected",
+ peakRms: 0.0061,
+ peakAmplitude: 0.065,
+ windowCount: 3,
+ speechWindowCount: 2,
+ maxConsecutiveSpeechWindows: 2,
+ });
+});