Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
faet: improved openAI fallback to local whisper
Gabriel Stein514fc58c2cbb
diff --git a/src/App.jsx b/src/App.jsxindex ce11d8ae0..1cbb19df4 100644--- a/src/App.jsx+++ b/src/App.jsx@@ -82,7 +82,8 @@ export default function App() {const audioChunksRef = useRef([]);const { toast } = useToast();const { hotkey } = useHotkey();- const { isDragging, handleMouseDown, handleMouseUp, handleClick } = useWindowDrag();+ const { isDragging, handleMouseDown, handleMouseUp, handleClick } =+ useWindowDrag();const [dragStartPos, setDragStartPos] = useState(null);const [hasDragged, setHasDragged] = useState(false);@@ -215,10 +216,18 @@ export default function App() {}} catch (err) {console.error("Local Whisper error:", err);- const allowFallback =++ // Check if it's a "No audio detected" error and don't retry+ if (err.message === "No audio detected") {+ throw err;+ }++ // Try fallback to OpenAI API ONLY if enabled AND we're in local mode+ const allowOpenAIFallback =localStorage.getItem("allowOpenAIFallback") === "true";+ const isLocalMode = localStorage.getItem("useLocalWhisper") === "true";- if (allowFallback) {+ if (allowOpenAIFallback && isLocalMode) {toast({title: "Fallback Mode",description: "Local Whisper failed. Retrying with OpenAI API...",@@ -226,6 +235,7 @@ export default function App() {});await processWithOpenAIAPI(audioBlob);} else {+ // No fallback allowed - strict local modethrow new Error(`Local Whisper failed: ${err.message}`);}}@@ -312,6 +322,56 @@ export default function App() {}} catch (err) {console.error("OpenAI API error:", err);++ // Try fallback to Local Whisper ONLY if enabled AND we're in OpenAI mode+ const allowLocalFallback =+ localStorage.getItem("allowLocalFallback") === "true";+ const isOpenAIMode = localStorage.getItem("useLocalWhisper") !== "true";++ if (allowLocalFallback && isOpenAIMode) {+ toast({+ title: "Fallback Mode",+ description: "OpenAI API failed. Retrying with Local Whisper...",+ variant: "default",+ });+ const fallbackModel =+ localStorage.getItem("fallbackWhisperModel") || "base";+ try {+ const arrayBuffer = await audioBlob.arrayBuffer();+ const options = { model: fallbackModel };+ const result = await window.electronAPI.transcribeLocalWhisper(+ arrayBuffer,+ options+ );++ if (result.success && result.text) {+ let text = AudioManager.cleanTranscription(result.text);+ if (text) {+ setTranscript(text);+ // Paste immediately - don't wait for database save+ const pastePromise = safePaste(text);+ // Save to database in parallel+ window.electronAPI.saveTranscription(text).catch((saveErr) => {+ console.error("Failed to save transcription:", saveErr);+ });+ await pastePromise;+ return; // Success with fallback+ }+ }+ // If local fallback fails, throw the original OpenAI error+ throw err;+ } catch (fallbackError) {+ console.error("Local fallback also failed:", fallbackError);+ toast({+ title: "Both Methods Failed",+ description:+ "Both OpenAI API and Local Whisper failed. Please try again.",+ variant: "destructive",+ });+ throw err; // Throw original OpenAI error+ }+ }+throw err;}};@@ -419,10 +479,11 @@ export default function App() {onMouseMove={(e) => {if (dragStartPos && !hasDragged) {const distance = Math.sqrt(- Math.pow(e.clientX - dragStartPos.x, 2) +- Math.pow(e.clientY - dragStartPos.y, 2)+ Math.pow(e.clientX - dragStartPos.x, 2) ++ Math.pow(e.clientY - dragStartPos.y, 2));- if (distance > 5) { // 5px threshold for drag+ if (distance > 5) {+ // 5px threshold for dragsetHasDragged(true);}}diff --git a/src/components/SettingsPage.tsx b/src/components/SettingsPage.tsxindex c6f98007f..9c539c23c 100644--- a/src/components/SettingsPage.tsx+++ b/src/components/SettingsPage.tsx@@ -30,6 +30,8 @@ export default function SettingsPage({ onBack }: SettingsPageProps) {const [useLocalWhisper, setUseLocalWhisper] = useState(false);const [whisperModel, setWhisperModel] = useState("base");const [allowOpenAIFallback, setAllowOpenAIFallback] = useState(false);+ const [allowLocalFallback, setAllowLocalFallback] = useState(false);+ const [fallbackWhisperModel, setFallbackWhisperModel] = useState("base");// Update stateconst [currentVersion, setCurrentVersion] = useState<string>("");@@ -79,11 +81,17 @@ export default function SettingsPage({ onBack }: SettingsPageProps) {// Load Whisper settingsconst savedUseLocal = localStorage.getItem("useLocalWhisper") === "true";const savedModel = localStorage.getItem("whisperModel") || "base";- const savedAllowFallback =+ const savedAllowOpenAIFallback =localStorage.getItem("allowOpenAIFallback") === "true";+ const savedAllowLocalFallback =+ localStorage.getItem("allowLocalFallback") === "true";+ const savedFallbackModel =+ localStorage.getItem("fallbackWhisperModel") || "base";setUseLocalWhisper(savedUseLocal);setWhisperModel(savedModel);- setAllowOpenAIFallback(savedAllowFallback);+ setAllowOpenAIFallback(savedAllowOpenAIFallback);+ setAllowLocalFallback(savedAllowLocalFallback);+ setFallbackWhisperModel(savedFallbackModel);// Load API key from main process first, then fallback to localStorageconst loadApiKey = async () => {@@ -177,27 +185,33 @@ export default function SettingsPage({ onBack }: SettingsPageProps) {try {await window.electronAPI.saveOpenAIKey(apiKey);localStorage.setItem("openaiApiKey", apiKey);+ localStorage.setItem("allowLocalFallback", allowLocalFallback.toString());+ localStorage.setItem("fallbackWhisperModel", fallbackWhisperModel);try {await window.electronAPI.createProductionEnvFile(apiKey);setAlertDialog({open: true,title: "API Key Saved",- description:- "OpenAI API key inscribed successfully! Your credentials have been securely recorded for transcription services.",+ description: `OpenAI API key inscribed successfully! Your credentials have been securely recorded for transcription services.${+ allowLocalFallback ? " Local Whisper fallback is enabled." : ""+ }`,});} catch (envError) {console.log("Could not create production .env file:", envError);setAlertDialog({open: true,title: "API Key Saved",- description:- "OpenAI API key saved successfully and will be available for transcription",+ description: `OpenAI API key saved successfully and will be available for transcription${+ allowLocalFallback ? " with Local Whisper fallback enabled" : ""+ }`,});}} catch (error) {console.error("Failed to save API key:", error);localStorage.setItem("openaiApiKey", apiKey);+ localStorage.setItem("allowLocalFallback", allowLocalFallback.toString());+ localStorage.setItem("fallbackWhisperModel", fallbackWhisperModel);setAlertDialog({open: true,title: "API Key Saved",@@ -211,9 +225,13 @@ export default function SettingsPage({ onBack }: SettingsPageProps) {localStorage.setItem("whisperModel", whisperModel);localStorage.setItem("allowOpenAIFallback", allowOpenAIFallback.toString());- // Also save API key if fallback is enabled and key is provided+ // SECURITY: Only save API key if fallback is enabled and key is provided+ // This ensures we never store unused keysif (allowOpenAIFallback && apiKey.trim()) {localStorage.setItem("openaiApiKey", apiKey);+ } else if (!allowOpenAIFallback) {+ // Clear the key if fallback is disabled for security+ localStorage.removeItem("openaiApiKey");}setAlertDialog({@@ -338,6 +356,83 @@ export default function SettingsPage({ onBack }: SettingsPageProps) {className="text-blue-800"helpText="Get your API key from platform.openai.com"/>++ {/* Local Whisper Fallback Toggle */}+ {whisperHook.whisperInstalled && (+ <div className="space-y-3 mt-4">+ <div className="flex items-center justify-between">+ <label className="text-sm font-medium text-blue-800">+ Fall back to Local Whisper+ </label>+ <label className="relative inline-flex items-center cursor-pointer">+ <input+ type="checkbox"+ className="sr-only"+ checked={allowLocalFallback}+ onChange={(e) =>+ setAllowLocalFallback(e.target.checked)+ }+ />+ <div+ className={`w-11 h-6 bg-gray-200 rounded-full peer peer-focus:ring-2 peer-focus:ring-blue-300 ${+ allowLocalFallback+ ? "bg-blue-600"+ : "bg-gray-300"+ } transition-colors duration-200`}+ >+ <div+ className={`absolute top-0.5 left-0.5 bg-white border border-gray-300 rounded-full h-5 w-5 transition-transform duration-200 ${+ allowLocalFallback+ ? "translate-x-5"+ : "translate-x-0"+ }`}+ ></div>+ </div>+ </label>+ </div>+ <p className="text-xs text-blue-700">+ If OpenAI API fails, try Local Whisper as backup+ </p>++ {/* Fallback Model Selection - only show if fallback is enabled */}+ {allowLocalFallback && (+ <div className="p-3 bg-blue-50 border border-blue-200 rounded-lg">+ <label className="block text-sm font-medium text-blue-800 mb-2">+ Fallback Whisper Model+ </label>+ <select+ value={fallbackWhisperModel}+ onChange={(e) =>+ setFallbackWhisperModel(e.target.value)+ }+ className="w-full text-sm border border-blue-300 rounded-md p-2 bg-white focus:border-blue-500 focus:ring-1 focus:ring-blue-500"+ >+ <option value="tiny">+ Tiny - Fast, basic quality+ </option>+ <option value="base">+ Base - Balanced (Recommended)+ </option>+ <option value="small">+ Small - Better quality+ </option>+ <option value="medium">+ Medium - High quality+ </option>+ <option value="large">+ Large - Best quality+ </option>+ <option value="turbo">+ Turbo - Fast & high quality+ </option>+ </select>+ <p className="text-xs text-blue-600 mt-1">+ Only used if OpenAI API fails+ </p>+ </div>+ )}+ </div>+ )}</div>)}diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex 01c975844..c6e8439bc 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -1,4 +1,4 @@-import TextCleanup from '../utils/textCleanup';+import TextCleanup from "../utils/textCleanup";class AudioManager {constructor() {@@ -107,7 +107,7 @@ class AudioManager {removeRepetitions: true,capitalizeFirst: true,addPeriod: false, // Don't auto-add periods to preserve user intent- ...options+ ...options,});}@@ -143,13 +143,17 @@ class AudioManager {throw error;}- // Try fallback to OpenAI API if enabled- const allowFallback =+ // Try fallback to OpenAI API ONLY if enabled AND we're in local mode+ const allowOpenAIFallback =localStorage.getItem("allowOpenAIFallback") === "true";- if (allowFallback) {+ const isLocalMode = localStorage.getItem("useLocalWhisper") === "true";++ if (allowOpenAIFallback && isLocalMode) {console.log("Falling back to OpenAI API...");try {- return await this.processWithOpenAIAPI(audioBlob);+ const fallbackResult = await this.processWithOpenAIAPI(audioBlob);+ // Mark the result as fallback for user awareness+ return { ...fallbackResult, source: "openai-fallback" };} catch (fallbackError) {// If OpenAI fallback also fails, throw the original errorthrow new Error(@@ -157,6 +161,7 @@ class AudioManager {);}} else {+ // No fallback allowed - strict local modethrow new Error(`Local Whisper failed: ${error.message}`);}}@@ -214,6 +219,40 @@ class AudioManager {}} catch (error) {console.error("OpenAI API error:", error);++ // Try fallback to Local Whisper ONLY if enabled AND we're in OpenAI mode+ const allowLocalFallback =+ localStorage.getItem("allowLocalFallback") === "true";+ const isOpenAIMode = localStorage.getItem("useLocalWhisper") !== "true";++ if (allowLocalFallback && isOpenAIMode) {+ console.log("Falling back to Local Whisper...");+ const fallbackModel =+ localStorage.getItem("fallbackWhisperModel") || "base";+ try {+ const arrayBuffer = await audioBlob.arrayBuffer();+ const options = { model: fallbackModel };+ const result = await window.electronAPI.transcribeLocalWhisper(+ arrayBuffer,+ options+ );++ if (result.success && result.text) {+ let text = AudioManager.cleanTranscription(result.text);+ if (text) {+ return { success: true, text, source: "local-fallback" };+ }+ }+ // If local fallback fails, throw the original OpenAI error+ throw error;+ } catch (fallbackError) {+ console.error("Local fallback also failed:", fallbackError);+ throw new Error(+ `OpenAI API failed: ${error.message}. Local fallback also failed: ${fallbackError.message}`+ );+ }+ }+throw error;}}