Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(transcription): stricter local whisper speech gate
Gabriel Stein2a441fd7551b
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex 457b0e244..d50d56012 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -5,6 +5,11 @@ import { isBuiltInMicrophone } from "../utils/audioDeviceUtils";import { isSecureEndpoint } from "../utils/urlUtils";import { withSessionRefresh } from "../lib/neonAuth";import { getBaseLanguageCode, validateLanguageForModel } from "../utils/languageSupport";+import {+ createLocalSpeechGateState,+ getLocalSpeechGateDecision,+ recordLocalSpeechWindow,+} from "./localSpeechGate";import {getSettings,getEffectiveReasoningModel,@@ -110,6 +115,7 @@ class AudioManager {this.sttConfig = null;this.lastAudioBlob = null;this.lastAudioMetadata = null;+ this._localSpeechGateState = null;}getWorkletBlobUrl() {@@ -301,28 +307,30 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor););}- // Silence detection: observe audio energy via AnalyserNodetry {this._silenceCtx = new AudioContext();this._silenceAnalyser = this._silenceCtx.createAnalyser();this._silenceAnalyser.fftSize = 2048;const sourceNode = this._silenceCtx.createMediaStreamSource(micStream);sourceNode.connect(this._silenceAnalyser);- this._peakRms = 0;+ this._localSpeechGateState = createLocalSpeechGateState();const dataArray = new Uint8Array(this._silenceAnalyser.fftSize);this._silenceInterval = setInterval(() => {this._silenceAnalyser.getByteTimeDomainData(dataArray);let sum = 0;+ let peak = 0;for (let i = 0; i < dataArray.length; i++) {const v = (dataArray[i] - 128) / 128;sum += v * v;+ const abs = Math.abs(v);+ if (abs > peak) peak = abs;}const rms = Math.sqrt(sum / dataArray.length);- if (rms > this._peakRms) this._peakRms = rms;+ recordLocalSpeechWindow(this._localSpeechGateState, rms, peak);}, 100);} catch (e) {- logger.warn("Silence detection setup failed, skipping", { error: e.message }, "audio");- this._peakRms = 1; // assume speech if detection fails+ logger.warn("Audio level gate setup failed, skipping", { error: e.message }, "audio");+ this._localSpeechGateState = null;}this.mediaRecorder = new MediaRecorder(micStream);@@ -335,7 +343,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);};this.mediaRecorder.onstop = async () => {- // Clean up silence detectionif (this._silenceInterval) {clearInterval(this._silenceInterval);this._silenceInterval = null;@@ -473,32 +480,43 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);async processAudio(audioBlob, metadata = {}) {const pipelineStart = performance.now();+ const settings = getSettings();+ const speechGateDecision = getLocalSpeechGateDecision(this._localSpeechGateState);+ this._localSpeechGateState = null;- // Skip transcription if recording was silence- const SILENCE_THRESHOLD = 0.002;- if (this._peakRms != null && this._peakRms < SILENCE_THRESHOLD) {+ const shouldUseStrongLocalWhisperGate =+ settings.useLocalWhisper && settings.localTranscriptionProvider === "whisper";+ if (+ speechGateDecision.skip &&+ (speechGateDecision.reason === "silence" || shouldUseStrongLocalWhisperGate)+ ) {logger.info(- "Silence detected, skipping transcription",- { peakRms: this._peakRms.toFixed(4), threshold: SILENCE_THRESHOLD },+ "Speech gate skipped transcription",+ {+ reason: speechGateDecision.reason,+ useLocalWhisper: settings.useLocalWhisper,+ localProvider: settings.localTranscriptionProvider,+ peakRms: speechGateDecision.peakRms?.toFixed(4),+ peakAmplitude: speechGateDecision.peakAmplitude?.toFixed(4),+ speechWindowCount: speechGateDecision.speechWindowCount,+ maxConsecutiveSpeechWindows: speechGateDecision.maxConsecutiveSpeechWindows,+ },"audio");- this._peakRms = null;this.isProcessing = false;this.onStateChange?.({ isRecording: false, isProcessing: false });this.onTranscriptionComplete?.({ success: true, text: "" });return;}- this._peakRms = null;try {- const s = getSettings();- const useLocalWhisper = s.useLocalWhisper;- const localProvider = s.localTranscriptionProvider;- const whisperModel = s.whisperModel;- const parakeetModel = s.parakeetModel || "parakeet-tdt-0.6b-v3";+ const useLocalWhisper = settings.useLocalWhisper;+ const localProvider = settings.localTranscriptionProvider;+ const whisperModel = settings.whisperModel;+ const parakeetModel = settings.parakeetModel || "parakeet-tdt-0.6b-v3";- const cloudTranscriptionMode = s.cloudTranscriptionMode;- const isSignedIn = s.isSignedIn;+ const cloudTranscriptionMode = settings.cloudTranscriptionMode;+ const isSignedIn = settings.isSignedIn;const isOpenWhisprCloudMode = !useLocalWhisper && cloudTranscriptionMode === "openwhispr";const useCloud = isOpenWhisprCloudMode && isSignedIn;diff --git a/src/helpers/localSpeechGate.js b/src/helpers/localSpeechGate.jsnew file mode 100644index 000000000..e999f494d--- /dev/null+++ b/src/helpers/localSpeechGate.js@@ -0,0 +1,68 @@+const SILENCE_RMS_THRESHOLD = 0.002;+const SPEECH_WINDOW_RMS_THRESHOLD = 0.0045;+const SPEECH_WINDOW_PEAK_THRESHOLD = 0.04;+const STRONG_SPEECH_RMS_THRESHOLD = 0.008;+const MIN_SPEECH_WINDOWS = 2;+const MIN_CONSECUTIVE_SPEECH_WINDOWS = 2;++export const createLocalSpeechGateState = () => ({+ peakRms: 0,+ peakAmplitude: 0,+ windowCount: 0,+ speechWindowCount: 0,+ consecutiveSpeechWindows: 0,+ maxConsecutiveSpeechWindows: 0,+});++export const recordLocalSpeechWindow = (state, rms, peak) => {+ if (!state) {+ return null;+ }++ state.windowCount += 1;+ state.peakRms = Math.max(state.peakRms, rms);+ state.peakAmplitude = Math.max(state.peakAmplitude, peak);++ const isSpeechWindow = rms >= SPEECH_WINDOW_RMS_THRESHOLD && peak >= SPEECH_WINDOW_PEAK_THRESHOLD;+ if (!isSpeechWindow) {+ state.consecutiveSpeechWindows = 0;+ return state;+ }++ state.speechWindowCount += 1;+ state.consecutiveSpeechWindows += 1;+ state.maxConsecutiveSpeechWindows = Math.max(+ state.maxConsecutiveSpeechWindows,+ state.consecutiveSpeechWindows+ );+ return state;+};++export const getLocalSpeechGateDecision = (state) => {+ if (!state?.windowCount) {+ return { skip: false, reason: "unavailable" };+ }++ const metrics = {+ peakRms: state.peakRms,+ peakAmplitude: state.peakAmplitude,+ windowCount: state.windowCount,+ speechWindowCount: state.speechWindowCount,+ maxConsecutiveSpeechWindows: state.maxConsecutiveSpeechWindows,+ };++ if (state.peakRms < SILENCE_RMS_THRESHOLD) {+ return { skip: true, reason: "silence", ...metrics };+ }++ const hasSpeech =+ state.speechWindowCount >= MIN_SPEECH_WINDOWS ||+ state.maxConsecutiveSpeechWindows >= MIN_CONSECUTIVE_SPEECH_WINDOWS ||+ (state.speechWindowCount >= 1 && state.peakRms >= STRONG_SPEECH_RMS_THRESHOLD);++ if (!hasSpeech) {+ return { skip: true, reason: "insufficient_speech", ...metrics };+ }++ return { skip: false, reason: "speech_detected", ...metrics };+};diff --git a/test/helpers/localSpeechGate.test.js b/test/helpers/localSpeechGate.test.jsnew file mode 100644index 000000000..05abbeb54--- /dev/null+++ b/test/helpers/localSpeechGate.test.js@@ -0,0 +1,71 @@+const test = require("node:test");+const assert = require("node:assert/strict");++test("treats near silence as skippable", async () => {+ const {+ createLocalSpeechGateState,+ recordLocalSpeechWindow,+ getLocalSpeechGateDecision,+ } = await import("../../src/helpers/localSpeechGate.js");++ const state = createLocalSpeechGateState();+ recordLocalSpeechWindow(state, 0.0012, 0.01);+ recordLocalSpeechWindow(state, 0.0016, 0.015);+ recordLocalSpeechWindow(state, 0.0014, 0.012);++ assert.deepEqual(getLocalSpeechGateDecision(state), {+ skip: true,+ reason: "silence",+ peakRms: 0.0016,+ peakAmplitude: 0.015,+ windowCount: 3,+ speechWindowCount: 0,+ maxConsecutiveSpeechWindows: 0,+ });+});++test("rejects isolated noise bursts without sustained speech", async () => {+ const {+ createLocalSpeechGateState,+ recordLocalSpeechWindow,+ getLocalSpeechGateDecision,+ } = await import("../../src/helpers/localSpeechGate.js");++ const state = createLocalSpeechGateState();+ recordLocalSpeechWindow(state, 0.0025, 0.03);+ recordLocalSpeechWindow(state, 0.0052, 0.045);+ recordLocalSpeechWindow(state, 0.0028, 0.028);++ const decision = getLocalSpeechGateDecision(state);++ assert.equal(decision.skip, true);+ assert.equal(decision.reason, "insufficient_speech");+ assert.equal(decision.peakRms, 0.0052);+ assert.equal(decision.peakAmplitude, 0.045);+ assert.equal(decision.windowCount, 3);+ assert.equal(decision.speechWindowCount, 1);+ assert.equal(decision.maxConsecutiveSpeechWindows, 1);+});++test("allows sustained speech-like energy through", async () => {+ const {+ createLocalSpeechGateState,+ recordLocalSpeechWindow,+ getLocalSpeechGateDecision,+ } = await import("../../src/helpers/localSpeechGate.js");++ const state = createLocalSpeechGateState();+ recordLocalSpeechWindow(state, 0.003, 0.025);+ recordLocalSpeechWindow(state, 0.0056, 0.06);+ recordLocalSpeechWindow(state, 0.0061, 0.065);++ assert.deepEqual(getLocalSpeechGateDecision(state), {+ skip: false,+ reason: "speech_detected",+ peakRms: 0.0061,+ peakAmplitude: 0.065,+ windowCount: 3,+ speechWindowCount: 2,+ maxConsecutiveSpeechWindows: 2,+ });+});