Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
perf: optimize transcription pipeline with caching, batched reads, and non-blocking operations
Gabriel Stein03baf024592b
diff --git a/src/App.jsx b/src/App.jsxindex f4d267094..0f65c743d 100644--- a/src/App.jsx+++ b/src/App.jsx@@ -181,23 +181,12 @@ export default function App() {if (result.success && result.text) {setTranscript(result.text);- // Paste immediately - don't wait for database save- const pastePromise = safePaste(result.text);-- // Save to database in parallel- const savePromise = window.electronAPI- .saveTranscription(result.text)- .catch((err) => {- // Failed to save transcription- });-- // Wait for paste to complete, but don't block on database save- await pastePromise;+ safePaste(result.text);+ window.electronAPI.saveTranscription(result.text).catch(() => {});}},});- // Process the audio using our enhanced AudioManagerawait audioManager.processAudio(audioBlob);} catch (err) {toast({diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex 29e96abc1..ccb4570b9 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -1,18 +1,18 @@import ReasoningService from "../services/ReasoningService";import { API_ENDPOINTS, buildApiUrl, normalizeBaseUrl } from "../config/constants";-// Debug logger for renderer process+const isDebugMode = typeof process !== 'undefined' && (process.env.OPENWHISPR_DEBUG === 'true' || process.env.NODE_ENV === 'development');+const debugLogger = {logReasoning: async (stage, details) => {+ if (!isDebugMode) return;+if (window.electronAPI?.logReasoning) {try {await window.electronAPI.logReasoning(stage, details);} catch (error) {- console.error('Failed to log reasoning:', error);+ // Silent fail}- } else {- // Fallback to console if IPC not available- console.log(`🤖 [REASONING ${stage}]`, details);}}};@@ -27,10 +27,10 @@ class AudioManager {this.onStateChange = null;this.onError = null;this.onTranscriptionComplete = null;- this.cachedApiKey = null; // Cache API key+ this.cachedApiKey = null;+ this.cachedTranscriptionEndpoint = null;}- // Set callback functionssetCallbacks({ onStateChange, onError, onTranscriptionComplete }) {this.onStateChange = onStateChange;this.onError = onError;@@ -110,11 +110,8 @@ class AudioManager {async processAudio(audioBlob) {try {- // Get user preferences- const useLocalWhisper =- localStorage.getItem("useLocalWhisper") === "true";+ const useLocalWhisper = localStorage.getItem("useLocalWhisper") === "true";const whisperModel = localStorage.getItem("whisperModel") || "base";-let result;if (useLocalWhisper) {@@ -124,7 +121,6 @@ class AudioManager {}this.onTranscriptionComplete?.(result);} catch (error) {- // Don't show error here if it's "No audio detected" - already shown elsewhereif (error.message !== "No audio detected") {this.onError?.({title: "Transcription Error",@@ -138,22 +134,8 @@ class AudioManager {}async processWithLocalWhisper(audioBlob, model = "base") {-- // Analyze audio levels first- const audioAnalysis = await this.analyzeAudioLevels(audioBlob);- if (audioAnalysis && audioAnalysis.isSilent) {- // Show error to user immediately- this.onError?.({- title: "No Audio Detected",- description: "The recording appears to be silent. Please check that your microphone is working and not muted.",- });- // Still continue to try transcription in case analysis was wrong- }-try {const arrayBuffer = await audioBlob.arrayBuffer();-- // Get language preference for local Whisperconst language = localStorage.getItem("preferredLanguage");const options = { model };if (language && language !== "auto") {@@ -164,21 +146,15 @@ class AudioManager {arrayBuffer,options);-if (result.success && result.text) {const text = await this.processTranscription(result.text, "local");- // Allow empty strings as valid responses (reasoning service might return cleaned empty text)if (text !== null && text !== undefined) {return { success: true, text: text || result.text, source: "local" };} else {throw new Error("No text transcribed");}- } else if (- result.success === false &&- result.message === "No audio detected"- ) {- // Show specific error to user with more details+ } else if (result.success === false && result.message === "No audio detected") {this.onError?.({title: "No Audio Detected",description: "The recording contained no detectable audio. Please check your microphone settings.",@@ -236,38 +212,6 @@ class AudioManager {return apiKey;}- async analyzeAudioLevels(audioBlob) {- try {- const audioContext = new (window.AudioContext || window.webkitAudioContext)();- const arrayBuffer = await audioBlob.arrayBuffer();- const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);-- const channelData = audioBuffer.getChannelData(0);- let sum = 0;- let max = 0;-- for (let i = 0; i < channelData.length; i++) {- const sample = Math.abs(channelData[i]);- sum += sample;- max = Math.max(max, sample);- }-- const average = sum / channelData.length;- const duration = audioBuffer.duration;--- return {- duration,- averageLevel: average,- maxLevel: max,- isSilent: max < 0.01- };- } catch (error) {- return null;- }- }-- // Convert audio to optimal format for API (reduces upload time)async optimizeAudio(audioBlob) {return new Promise((resolve) => {const audioContext = new (window.AudioContext ||@@ -295,8 +239,6 @@ class AudioManager {source.start();const renderedBuffer = await offlineContext.startRendering();-- // Convert to WAV blobconst wavBlob = this.audioBufferToWav(renderedBuffer);resolve(wavBlob);} catch (error) {@@ -310,7 +252,6 @@ class AudioManager {});}- // Convert AudioBuffer to WAV formataudioBufferToWav(buffer) {const length = buffer.length;const arrayBuffer = new ArrayBuffer(44 + length * 2);@@ -318,7 +259,6 @@ class AudioManager {const sampleRate = buffer.sampleRate;const channelData = buffer.getChannelData(0);- // WAV headerconst writeString = (offset, string) => {for (let i = 0; i < string.length; i++) {view.setUint8(offset + i, string.charCodeAt(i));@@ -339,7 +279,6 @@ class AudioManager {writeString(36, "data");view.setUint32(40, length * 2, true);- // Convert samples to 16-bit PCMlet offset = 44;for (let i = 0; i < length; i++) {const sample = Math.max(-1, Math.min(1, channelData[i]));@@ -354,14 +293,7 @@ class AudioManager {return new Blob([arrayBuffer], { type: "audio/wav" });}- async processWithReasoningModel(text) {- const model = (typeof window !== 'undefined' && window.localStorage)- ? (localStorage.getItem("reasoningModel") || "gpt-4o-mini")- : "gpt-4o-mini";- const agentName = (typeof window !== 'undefined' && window.localStorage)- ? (localStorage.getItem("agentName") || null)- : null;-+ async processWithReasoningModel(text, model, agentName) {debugLogger.logReasoning("CALLING_REASONING_SERVICE", {model,agentName,@@ -439,21 +371,16 @@ class AudioManager {async processTranscription(text, source) {const normalizedText = typeof text === "string" ? text.trim() : "";-- // Log incoming transcription+debugLogger.logReasoning("TRANSCRIPTION_RECEIVED", {source,textLength: normalizedText.length,textPreview: normalizedText.substring(0, 100) + (normalizedText.length > 100 ? "..." : ""),timestamp: new Date().toISOString()});-- // Check if reasoning should handle cleanup- const useReasoning = await this.isReasoningAvailable();-- // Safe localStorage access- const reasoningModel = (typeof window !== 'undefined' && window.localStorage)- ? (localStorage.getItem("reasoningModel") || "gpt-4o-mini")++ const reasoningModel = (typeof window !== 'undefined' && window.localStorage)+ ? (localStorage.getItem("reasoningModel") || "gpt-4o-mini"): "gpt-4o-mini";const reasoningProvider = (typeof window !== 'undefined' && window.localStorage)? (localStorage.getItem("reasoningProvider") || "auto")@@ -461,26 +388,26 @@ class AudioManager {const agentName = (typeof window !== 'undefined' && window.localStorage)? (localStorage.getItem("agentName") || null): null;-+ const useReasoning = await this.isReasoningAvailable();+debugLogger.logReasoning("REASONING_CHECK", {useReasoning,reasoningModel,reasoningProvider,agentName});-+if (useReasoning) {try {- // Minimal cleanup for reasoning modelsconst preparedText = normalizedText;-+debugLogger.logReasoning("SENDING_TO_REASONING", {preparedTextLength: preparedText.length,model: reasoningModel,provider: reasoningProvider});-- const result = await this.processWithReasoningModel(preparedText);++ const result = await this.processWithReasoningModel(preparedText, reasoningModel, agentName);debugLogger.logReasoning("REASONING_SUCCESS", {resultLength: result.length,@@ -496,32 +423,32 @@ class AudioManager {fallbackToCleanup: true});console.error(`Reasoning failed (${source}):`, error.message);- // Fall back to standard cleanup}}-+debugLogger.logReasoning("USING_STANDARD_CLEANUP", {reason: useReasoning ? "Reasoning failed" : "Reasoning not enabled"});-- // Standard cleanup when reasoning is unavailable or fails+return normalizedText;}async processWithOpenAIAPI(audioBlob) {try {- // Parallel: get API key (cached) and optimize audio+ const language = localStorage.getItem("preferredLanguage");+ const allowLocalFallback = localStorage.getItem("allowLocalFallback") === "true";+ const fallbackModel = localStorage.getItem("fallbackWhisperModel") || "base";+ const shouldOptimize = audioBlob.size > 1024 * 1024;+const [apiKey, optimizedAudio] = await Promise.all([this.getAPIKey(),- this.optimizeAudio(audioBlob),+ shouldOptimize ? this.optimizeAudio(audioBlob) : Promise.resolve(audioBlob),]);const formData = new FormData();formData.append("file", optimizedAudio, "audio.wav");formData.append("model", "whisper-1");- // Add language hint if set (improves processing speed)- const language = localStorage.getItem("preferredLanguage");if (language && language !== "auto") {formData.append("language", language);}@@ -543,7 +470,7 @@ class AudioManager {}const result = await response.json();-+if (result.text) {const text = await this.processTranscription(result.text, "openai");const source = await this.isReasoningAvailable() ? "openai-reasoned" : "openai";@@ -552,20 +479,11 @@ class AudioManager {throw new Error("No text transcribed");}} catch (error) {-- // Try fallback to Local Whisper ONLY if enabled AND we're in OpenAI mode- const allowLocalFallback =- localStorage.getItem("allowLocalFallback") === "true";const isOpenAIMode = localStorage.getItem("useLocalWhisper") !== "true";if (allowLocalFallback && isOpenAIMode) {- const fallbackModel =- localStorage.getItem("fallbackWhisperModel") || "base";try {const arrayBuffer = await audioBlob.arrayBuffer();-- // Get language preference for fallback as well- const language = localStorage.getItem("preferredLanguage");const options = { model: fallbackModel };if (language && language !== "auto") {options.language = language;@@ -582,7 +500,6 @@ class AudioManager {return { success: true, text, source: "local-fallback" };}}- // If local fallback fails, throw the original OpenAI errorthrow error;} catch (fallbackError) {throw new Error(@@ -596,6 +513,10 @@ class AudioManager {}getTranscriptionEndpoint() {+ if (this.cachedTranscriptionEndpoint) {+ return this.cachedTranscriptionEndpoint;+ }+try {const stored = typeof localStorage !== "undefined"? localStorage.getItem("cloudTranscriptionBaseUrl") || ""@@ -605,23 +526,29 @@ class AudioManager {const normalizedBase = normalizeBaseUrl(base);if (!normalizedBase) {+ this.cachedTranscriptionEndpoint = API_ENDPOINTS.TRANSCRIPTION;return API_ENDPOINTS.TRANSCRIPTION;}- // Security: Only allow HTTPS endpoints (except localhost for development)const isLocalhost = normalizedBase.includes('://localhost') || normalizedBase.includes('://127.0.0.1');if (!normalizedBase.startsWith('https://') && !isLocalhost) {console.warn('Non-HTTPS endpoint rejected for security. Using default.');+ this.cachedTranscriptionEndpoint = API_ENDPOINTS.TRANSCRIPTION;return API_ENDPOINTS.TRANSCRIPTION;}+ let endpoint;if (/\/audio\/(transcriptions|translations)$/i.test(normalizedBase)) {- return normalizedBase;+ endpoint = normalizedBase;+ } else {+ endpoint = buildApiUrl(normalizedBase, '/audio/transcriptions');}- return buildApiUrl(normalizedBase, '/audio/transcriptions');+ this.cachedTranscriptionEndpoint = endpoint;+ return endpoint;} catch (error) {console.warn('Failed to resolve transcription endpoint:', error);+ this.cachedTranscriptionEndpoint = API_ENDPOINTS.TRANSCRIPTION;return API_ENDPOINTS.TRANSCRIPTION;}}diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 3a22ea01e..595902f8b 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -88,11 +88,12 @@ class IPCHandlers {}});- // Database handlersipcMain.handle("db-save-transcription", async (event, text) => {const result = this.databaseManager.saveTranscription(text);if (result?.success && result?.transcription) {- this.broadcastToWindows("transcription-added", result.transcription);+ setImmediate(() => {+ this.broadcastToWindows("transcription-added", result.transcription);+ });}return result;});@@ -104,8 +105,10 @@ class IPCHandlers {ipcMain.handle("db-clear-transcriptions", async (event) => {const result = this.databaseManager.clearTranscriptions();if (result?.success) {- this.broadcastToWindows("transcriptions-cleared", {- cleared: result.cleared,+ setImmediate(() => {+ this.broadcastToWindows("transcriptions-cleared", {+ cleared: result.cleared,+ });});}return result;@@ -114,7 +117,9 @@ class IPCHandlers {ipcMain.handle("db-delete-transcription", async (event, id) => {const result = this.databaseManager.deleteTranscription(id);if (result?.success) {- this.broadcastToWindows("transcription-deleted", { id });+ setImmediate(() => {+ this.broadcastToWindows("transcription-deleted", { id });+ });}return result;});diff --git a/src/helpers/whisper.js b/src/helpers/whisper.jsindex 4a804a606..9be9e0e75 100644--- a/src/helpers/whisper.js+++ b/src/helpers/whisper.js@@ -10,11 +10,12 @@ const debugLogger = require("./debugLogger");class WhisperManager {constructor() {- this.pythonCmd = null; // Cache Python executable path- this.whisperInstalled = null; // Cache installation status- this.isInitialized = false; // Track if startup init completed- this.currentDownloadProcess = null; // Track current download process for cancellation+ this.pythonCmd = null;+ this.whisperInstalled = null;+ this.isInitialized = false;+ this.currentDownloadProcess = null;this.pythonInstaller = new PythonInstaller();+ this.cachedFFmpegPath = null;}sanitizeErrorMessage(message = "") {@@ -162,88 +163,87 @@ class WhisperManager {return tempAudioPath;}- async runWhisperProcess(tempAudioPath, model, language) {- const pythonCmd = await this.findPythonExecutable();- const whisperScriptPath = this.getWhisperScriptPath();-- // Check if whisper script exists- if (!fs.existsSync(whisperScriptPath)) {- throw new Error(`Whisper script not found at: ${whisperScriptPath}`);- }-- const args = [whisperScriptPath, tempAudioPath, "--model", model];- if (language) {- args.push("--language", language);+ async getFFmpegPath() {+ if (this.cachedFFmpegPath) {+ return this.cachedFFmpegPath;}- args.push("--output-format", "json");+ let ffmpegPath;- return new Promise(async (resolve, reject) => {- // Get FFmpeg path with robust production/development handling- let ffmpegPath;+ try {+ ffmpegPath = require("ffmpeg-static");+ debugLogger.logFFmpegDebug('Initial ffmpeg-static path', ffmpegPath);- try {- ffmpegPath = require("ffmpeg-static");- debugLogger.logFFmpegDebug('Initial ffmpeg-static path', ffmpegPath);-- // Add Windows .exe extension if missing- if (process.platform === "win32" && !ffmpegPath.endsWith(".exe")) {- ffmpegPath += ".exe";- }+ if (process.platform === "win32" && !ffmpegPath.endsWith(".exe")) {+ ffmpegPath += ".exe";+ }- if (process.env.NODE_ENV !== "development" && !fs.existsSync(ffmpegPath)) {- const possiblePaths = [- ffmpegPath.replace("app.asar", "app.asar.unpacked"),- ffmpegPath.replace(/.*app\.asar/, path.join(__dirname, "..", "..", "app.asar.unpacked")),- path.join(process.resourcesPath, "app.asar.unpacked", "node_modules", "ffmpeg-static", process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg")- ];+ if (process.env.NODE_ENV !== "development" && !fs.existsSync(ffmpegPath)) {+ const possiblePaths = [+ ffmpegPath.replace("app.asar", "app.asar.unpacked"),+ ffmpegPath.replace(/.*app\.asar/, path.join(__dirname, "..", "..", "app.asar.unpacked")),+ path.join(process.resourcesPath, "app.asar.unpacked", "node_modules", "ffmpeg-static", process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg")+ ];- debugLogger.log('FFmpeg not found at primary path, checking alternatives');-- for (const possiblePath of possiblePaths) {- const exists = fs.existsSync(possiblePath);-- if (exists) {- ffmpegPath = possiblePath;- debugLogger.log('FFmpeg found at:', ffmpegPath);- break;- }+ debugLogger.log('FFmpeg not found at primary path, checking alternatives');++ for (const possiblePath of possiblePaths) {+ if (fs.existsSync(possiblePath)) {+ ffmpegPath = possiblePath;+ debugLogger.log('FFmpeg found at:', ffmpegPath);+ break;}}+ }- // Final validation of bundled FFmpeg- if (!fs.existsSync(ffmpegPath)) {- debugLogger.error('Bundled FFmpeg not found at:', ffmpegPath);- throw new Error(`Bundled FFmpeg not found at ${ffmpegPath}`);- }-- // Validate it's actually executable- try {- fs.accessSync(ffmpegPath, fs.constants.X_OK);- debugLogger.log('FFmpeg is executable');- } catch (e) {- debugLogger.error('FFmpeg exists but is not executable:', e.message);- throw new Error(`FFmpeg exists but is not executable: ${ffmpegPath}`);- }+ if (!fs.existsSync(ffmpegPath)) {+ debugLogger.error('Bundled FFmpeg not found at:', ffmpegPath);+ throw new Error(`Bundled FFmpeg not found at ${ffmpegPath}`);+ }+ try {+ fs.accessSync(ffmpegPath, fs.constants.X_OK);+ debugLogger.log('FFmpeg is executable');} catch (e) {- debugLogger.log('Bundled FFmpeg not available, trying system FFmpeg');-- // Try system FFmpeg with validation- const systemFFmpeg = process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg";-- try {- const versionResult = await runCommand(systemFFmpeg, ["--version"], { timeout: TIMEOUTS.QUICK_CHECK });- ffmpegPath = systemFFmpeg;- debugLogger.log('Using system FFmpeg');- } catch (systemError) {- debugLogger.error('System FFmpeg also unavailable:', systemError.message);- ffmpegPath = systemFFmpeg; // Last resort - let Python handle the error- }+ debugLogger.error('FFmpeg exists but is not executable:', e.message);+ throw new Error(`FFmpeg exists but is not executable: ${ffmpegPath}`);+ }++ } catch (e) {+ debugLogger.log('Bundled FFmpeg not available, trying system FFmpeg');++ const systemFFmpeg = process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg";++ try {+ const versionResult = await runCommand(systemFFmpeg, ["--version"], { timeout: TIMEOUTS.QUICK_CHECK });+ ffmpegPath = systemFFmpeg;+ debugLogger.log('Using system FFmpeg');+ } catch (systemError) {+ debugLogger.error('System FFmpeg also unavailable:', systemError.message);+ ffmpegPath = systemFFmpeg;}+ }++ this.cachedFFmpegPath = ffmpegPath;+ return ffmpegPath;+ }++ async runWhisperProcess(tempAudioPath, model, language) {+ const pythonCmd = await this.findPythonExecutable();+ const whisperScriptPath = this.getWhisperScriptPath();++ if (!fs.existsSync(whisperScriptPath)) {+ throw new Error(`Whisper script not found at: ${whisperScriptPath}`);+ }++ const args = [whisperScriptPath, tempAudioPath, "--model", model];+ if (language) {+ args.push("--language", language);+ }+ args.push("--output-format", "json");- // Enhanced environment setup - Python script checks multiple env vars- // Make sure to use absolute paths+ return new Promise(async (resolve, reject) => {+ const ffmpegPath = await this.getFFmpegPath();const absoluteFFmpegPath = path.resolve(ffmpegPath);const enhancedEnv = {...process.env,@@ -656,67 +656,13 @@ class WhisperManager {async checkFFmpegAvailability() {debugLogger.logWhisperPipeline('checkFFmpegAvailability - start', {});-+try {const pythonCmd = await this.findPythonExecutable();const whisperScriptPath = this.getWhisperScriptPath();-- // Get FFmpeg path to pass to Python script- let ffmpegPath;- try {- ffmpegPath = require("ffmpeg-static");- debugLogger.logFFmpegDebug('checkFFmpegAvailability', ffmpegPath, {- NODE_ENV: process.env.NODE_ENV,- resourcesPath: process.resourcesPath,- __dirname: __dirname- });-- // Always check if the path exists and handle ASAR unpacking- if (!fs.existsSync(ffmpegPath)) {- debugLogger.log('FFmpeg not found at initial path, checking alternatives');-- const possiblePaths = [- // Direct ASAR replacement- ffmpegPath.replace("app.asar", "app.asar.unpacked"),- // Alternative ASAR replacement patterns- ffmpegPath.replace(/app\.asar[\/\\]/, "app.asar.unpacked/"),- // Resources folder fallback- process.resourcesPath ? path.join(process.resourcesPath, "app.asar.unpacked", "node_modules", "ffmpeg-static", process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg") : null,- // Development fallback- path.join(__dirname, "..", "..", "node_modules", "ffmpeg-static", process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg")- ].filter(Boolean);-- // Check alternative paths- debugLogger.log('Checking alternative FFmpeg paths for availability check');-- for (const possiblePath of possiblePaths) {- const exists = fs.existsSync(possiblePath);- // Check path existence- debugLogger.logFFmpegDebug('Checking availability path', possiblePath);-- if (exists) {- ffmpegPath = possiblePath;- // Found FFmpeg- debugLogger.log('FFmpeg found for availability check at:', ffmpegPath);- break;- }- }- }-- // Final validation- if (!ffmpegPath || !fs.existsSync(ffmpegPath)) {- debugLogger.log('FFmpeg not found at any location');- ffmpegPath = null;- } else {- // Using bundled FFmpeg- }- } catch (e) {- debugLogger.log('ffmpeg-static error:', e.message);- ffmpegPath = null;- }+ const ffmpegPath = await this.getFFmpegPath();const result = await new Promise((resolve) => {- // Set up environment with FFmpeg pathconst env = {...process.env,FFMPEG_PATH: ffmpegPath || "",