Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
refactor: refactored end-to-end transcription performance
Gabriel Stein05ae1c2c88f2
diff --git a/main.js b/main.jsindex 9240eedbf..684460e01 100644--- a/main.js+++ b/main.js@@ -51,6 +51,11 @@ async function startApp() {await new Promise((resolve) => setTimeout(resolve, 2000));}+ // Initialize Whisper manager at startup (don't await to avoid blocking)+ whisperManager.initializeAtStartup().catch((err) => {+ console.log("Whisper initialization info:", err.message);+ });+// Create main windowtry {await windowManager.createMainWindow();diff --git a/src/App.jsx b/src/App.jsxindex a498a0f1b..1674bba9f 100644--- a/src/App.jsx+++ b/src/App.jsx@@ -81,30 +81,29 @@ export default function App() {const startRecording = async () => {try {- setError("");+ setError('');const stream = await navigator.mediaDevices.getUserMedia({ audio: true });-+mediaRecorderRef.current = new window.MediaRecorder(stream);audioChunksRef.current = [];-+mediaRecorderRef.current.ondataavailable = (event) => {audioChunksRef.current.push(event.data);};-+mediaRecorderRef.current.onstop = async () => {setIsProcessing(true);- const audioBlob = new Blob(audioChunksRef.current, {- type: "audio/wav",- });- await processAudio(audioBlob);- stream.getTracks().forEach((track) => track.stop());+ const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/wav' });+ // Start processing immediately without waiting+ processAudio(audioBlob);+ stream.getTracks().forEach(track => track.stop());};-+mediaRecorderRef.current.start();setIsRecording(true);} catch (err) {- console.error("Recording error:", err);- setError("Failed to access microphone: " + err.message);+ console.error("Recording error:", err)+ setError('Failed to access microphone: ' + err.message);}};@@ -126,25 +125,25 @@ export default function App() {const processAudio = async (audioBlob) => {try {- // Check if local Whisper is available and user preference- const useLocalWhisper =- localStorage.getItem("useLocalWhisper") === "true";- const whisperModel = localStorage.getItem("whisperModel") || "turbo"; // Default to turbo for speed-+ // Cache preferences to avoid repeated localStorage calls+ const useLocalWhisper = localStorage.getItem('useLocalWhisper') === 'true';+ const whisperModel = localStorage.getItem('whisperModel') || 'base';+if (useLocalWhisper) {await processWithLocalWhisper(audioBlob, whisperModel);} else {await processWithOpenAIAPI(audioBlob);}+} catch (err) {- console.error("Transcription error:", err);- setError("Transcription failed: " + err.message);+ console.error("Transcription error:", err)+ setError('Transcription failed: ' + err.message);} finally {setIsProcessing(false);}};- const processWithLocalWhisper = async (audioBlob, model = "turbo") => {+ const processWithLocalWhisper = async (audioBlob, model = "base") => {try {// Convert Blob to ArrayBuffer for IPC transfer - do this earlyconst arrayBuffer = await audioBlob.arrayBuffer();diff --git a/src/components/ControlPanel.tsx b/src/components/ControlPanel.tsxindex c8bfe34e6..403049c6f 100644--- a/src/components/ControlPanel.tsx+++ b/src/components/ControlPanel.tsx@@ -123,7 +123,9 @@ declare global {onUpdateAvailable: (callback: (event: any, info: any) => void) => void;onUpdateNotAvailable: (callback: (event: any, info: any) => void) => void;onUpdateDownloaded: (callback: (event: any, info: any) => void) => void;- onUpdateDownloadProgress: (callback: (event: any, progressObj: any) => void) => void;+ onUpdateDownloadProgress: (+ callback: (event: any, progressObj: any) => void+ ) => void;onUpdateError: (callback: (event: any, error: any) => void) => void;};}@@ -156,7 +158,7 @@ export default function ControlPanel() {setUpdateDownloaded(status.updateDownloaded);setIsDevelopment(status.isDevelopment);} catch (error) {- console.error('Error initializing update status:', error);+ console.error("Error initializing update status:", error);}};@@ -172,7 +174,7 @@ export default function ControlPanel() {});window.electronAPI.onUpdateError((event, error) => {- console.error('Update error:', error);+ console.error("Update error:", error);});}, []);@@ -190,7 +192,7 @@ export default function ControlPanel() {const copyToClipboard = async (text: string) => {try {await navigator.clipboard.writeText(text);- alert("Text copied to your scribal collection!");+ alert("Text copied to your clipboard!");} catch (err) {}};diff --git a/src/helpers/whisper.js b/src/helpers/whisper.jsindex ce6d0a1b9..8d4dd23e3 100644--- a/src/helpers/whisper.js+++ b/src/helpers/whisper.js@@ -6,7 +6,22 @@ const crypto = require("crypto");class WhisperManager {constructor() {- // Initialize whisper manager+ this.pythonCmd = null; // Cache Python executable path+ this.whisperInstalled = null; // Cache installation status+ this.isInitialized = false; // Track if startup init completed+ }++ async initializeAtStartup() {+ try {+ // Initialize Python path and Whisper installation status+ await this.findPythonExecutable();+ await this.checkWhisperInstallation();+ this.isInitialized = true;+ console.log("✅ Whisper manager initialized at startup");+ } catch (error) {+ console.log("⚠️ Whisper not available at startup:", error.message);+ this.isInitialized = true; // Mark as initialized even if Whisper isn't available+ }}async transcribeLocalWhisper(audioBlob, options = {}) {@@ -75,13 +90,13 @@ class WhisperManager {let stderr = "";let isResolved = false;- // Set timeout for faster failure detection+ // Set timeout for longer recordingsconst timeout = setTimeout(() => {if (!isResolved) {whisperProcess.kill("SIGTERM");- reject(new Error("Whisper transcription timed out (15 seconds)"));+ reject(new Error("Whisper transcription timed out (60 seconds)"));}- }, 15000);+ }, 60000);whisperProcess.stdout.on("data", (data) => {stdout += data.toString();@@ -154,13 +169,17 @@ class WhisperManager {cleanupTempFile(tempAudioPath) {try {fs.unlinkSync(tempAudioPath);- console.log("🗑️ Cleaned up temp audio file");} catch (cleanupError) {console.warn("⚠️ Could not clean up temp file:", cleanupError.message);}}async findPythonExecutable() {+ // Return cached result if available+ if (this.pythonCmd) {+ return this.pythonCmd;+ }+const possiblePaths = ["python3","python",@@ -184,7 +203,7 @@ class WhisperManager {});if (result) {- console.log("🐍 Found Python at:", pythonPath);+ this.pythonCmd = pythonPath; // Cache the resultreturn pythonPath;}} catch (error) {@@ -198,10 +217,15 @@ class WhisperManager {}async checkWhisperInstallation() {+ // Return cached result if available+ if (this.whisperInstalled !== null) {+ return this.whisperInstalled;+ }+try {const pythonCmd = await this.findPythonExecutable();- return new Promise((resolve) => {+ const result = await new Promise((resolve) => {const checkProcess = spawn(pythonCmd, ["-c",'import whisper; print("OK")',@@ -214,22 +238,27 @@ class WhisperManager {checkProcess.on("close", (code) => {if (code === 0 && output.includes("OK")) {- console.log("✅ Whisper is installed and working");resolve({ installed: true, working: true });} else {- console.log("❌ Whisper is not properly installed");resolve({ installed: false, working: false });}});checkProcess.on("error", (error) => {- console.log("❌ Error checking Whisper:", error.message);resolve({ installed: false, working: false, error: error.message });});});++ this.whisperInstalled = result; // Cache the result+ return result;} catch (error) {- console.log("❌ Error finding Python:", error.message);- return { installed: false, working: false, error: error.message };+ const errorResult = {+ installed: false,+ working: false,+ error: error.message,+ };+ this.whisperInstalled = errorResult;+ return errorResult;}}diff --git a/whisper_bridge.py b/whisper_bridge.pyindex 16fac68cc..1bfe0b023 100755--- a/whisper_bridge.py+++ b/whisper_bridge.py@@ -14,13 +14,32 @@ import whisperimport threadingimport timeimport requests+import gc++# Global model cache to avoid reloading+_model_cache = {}def load_model(model_name="base"):- """Load Whisper model with error handling"""+ """Load Whisper model with caching for performance"""+ global _model_cache++ # Return cached model if available+ if model_name in _model_cache:+ return _model_cache[model_name]+try:print(f"Loading Whisper model: {model_name}", file=sys.stderr)model = whisper.load_model(model_name)print(f"Model {model_name} loaded successfully", file=sys.stderr)++ # Cache the model but limit cache size+ if len(_model_cache) >= 2: # Keep max 2 models in memory+ # Remove oldest model+ oldest_key = next(iter(_model_cache))+ del _model_cache[oldest_key]+ gc.collect() # Force garbage collection++ _model_cache[model_name] = modelreturn modelexcept Exception as e:print(f"Error loading model: {e}", file=sys.stderr)@@ -288,19 +307,14 @@ def delete_model(model_name="base"):}def transcribe_audio(audio_path, model_name="base", language=None):- """Transcribe audio file using Whisper"""- print(f"Starting transcription with model: {model_name}", file=sys.stderr)-+ """Transcribe audio file using Whisper with optimizations"""try:- # Load model+ # Load model (uses cache for performance)model = load_model(model_name)if model is None:return {"error": "Failed to load Whisper model", "success": False}- # Transcribe with minimal logging to avoid stdout pollution- print(f"Transcribing audio file: {audio_path}", file=sys.stderr)-- # Set transcription options for performance+ # Set transcription options - keep it simple for reliabilityoptions = {"fp16": False, # Use FP32 for better compatibility"verbose": False, # Reduce logging overhead