Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

faet: improved openAI fallback to local whisper

Gabriel Stein514fc58c2cbb
diff --git a/src/App.jsx b/src/App.jsx
index ce11d8ae0..1cbb19df4 100644
--- a/src/App.jsx
+++ b/src/App.jsx
@@ -82,7 +82,8 @@ export default function App() {
const audioChunksRef = useRef([]);
const { toast } = useToast();
const { hotkey } = useHotkey();
- const { isDragging, handleMouseDown, handleMouseUp, handleClick } = useWindowDrag();
+ const { isDragging, handleMouseDown, handleMouseUp, handleClick } =
+ useWindowDrag();
const [dragStartPos, setDragStartPos] = useState(null);
const [hasDragged, setHasDragged] = useState(false);
@@ -215,10 +216,18 @@ export default function App() {
}
} catch (err) {
console.error("Local Whisper error:", err);
- const allowFallback =
+
+ // Check if it's a "No audio detected" error and don't retry
+ if (err.message === "No audio detected") {
+ throw err;
+ }
+
+ // Try fallback to OpenAI API ONLY if enabled AND we're in local mode
+ const allowOpenAIFallback =
localStorage.getItem("allowOpenAIFallback") === "true";
+ const isLocalMode = localStorage.getItem("useLocalWhisper") === "true";
- if (allowFallback) {
+ if (allowOpenAIFallback && isLocalMode) {
toast({
title: "Fallback Mode",
description: "Local Whisper failed. Retrying with OpenAI API...",
@@ -226,6 +235,7 @@ export default function App() {
});
await processWithOpenAIAPI(audioBlob);
} else {
+ // No fallback allowed - strict local mode
throw new Error(`Local Whisper failed: ${err.message}`);
}
}
@@ -312,6 +322,56 @@ export default function App() {
}
} catch (err) {
console.error("OpenAI API error:", err);
+
+ // Try fallback to Local Whisper ONLY if enabled AND we're in OpenAI mode
+ const allowLocalFallback =
+ localStorage.getItem("allowLocalFallback") === "true";
+ const isOpenAIMode = localStorage.getItem("useLocalWhisper") !== "true";
+
+ if (allowLocalFallback && isOpenAIMode) {
+ toast({
+ title: "Fallback Mode",
+ description: "OpenAI API failed. Retrying with Local Whisper...",
+ variant: "default",
+ });
+ const fallbackModel =
+ localStorage.getItem("fallbackWhisperModel") || "base";
+ try {
+ const arrayBuffer = await audioBlob.arrayBuffer();
+ const options = { model: fallbackModel };
+ const result = await window.electronAPI.transcribeLocalWhisper(
+ arrayBuffer,
+ options
+ );
+
+ if (result.success && result.text) {
+ let text = AudioManager.cleanTranscription(result.text);
+ if (text) {
+ setTranscript(text);
+ // Paste immediately - don't wait for database save
+ const pastePromise = safePaste(text);
+ // Save to database in parallel
+ window.electronAPI.saveTranscription(text).catch((saveErr) => {
+ console.error("Failed to save transcription:", saveErr);
+ });
+ await pastePromise;
+ return; // Success with fallback
+ }
+ }
+ // If local fallback fails, throw the original OpenAI error
+ throw err;
+ } catch (fallbackError) {
+ console.error("Local fallback also failed:", fallbackError);
+ toast({
+ title: "Both Methods Failed",
+ description:
+ "Both OpenAI API and Local Whisper failed. Please try again.",
+ variant: "destructive",
+ });
+ throw err; // Throw original OpenAI error
+ }
+ }
+
throw err;
}
};
@@ -419,10 +479,11 @@ export default function App() {
onMouseMove={(e) => {
if (dragStartPos && !hasDragged) {
const distance = Math.sqrt(
- Math.pow(e.clientX - dragStartPos.x, 2) +
- Math.pow(e.clientY - dragStartPos.y, 2)
+ Math.pow(e.clientX - dragStartPos.x, 2) +
+ Math.pow(e.clientY - dragStartPos.y, 2)
);
- if (distance > 5) { // 5px threshold for drag
+ if (distance > 5) {
+ // 5px threshold for drag
setHasDragged(true);
}
}
diff --git a/src/components/SettingsPage.tsx b/src/components/SettingsPage.tsx
index c6f98007f..9c539c23c 100644
--- a/src/components/SettingsPage.tsx
+++ b/src/components/SettingsPage.tsx
@@ -30,6 +30,8 @@ export default function SettingsPage({ onBack }: SettingsPageProps) {
const [useLocalWhisper, setUseLocalWhisper] = useState(false);
const [whisperModel, setWhisperModel] = useState("base");
const [allowOpenAIFallback, setAllowOpenAIFallback] = useState(false);
+ const [allowLocalFallback, setAllowLocalFallback] = useState(false);
+ const [fallbackWhisperModel, setFallbackWhisperModel] = useState("base");
// Update state
const [currentVersion, setCurrentVersion] = useState<string>("");
@@ -79,11 +81,17 @@ export default function SettingsPage({ onBack }: SettingsPageProps) {
// Load Whisper settings
const savedUseLocal = localStorage.getItem("useLocalWhisper") === "true";
const savedModel = localStorage.getItem("whisperModel") || "base";
- const savedAllowFallback =
+ const savedAllowOpenAIFallback =
localStorage.getItem("allowOpenAIFallback") === "true";
+ const savedAllowLocalFallback =
+ localStorage.getItem("allowLocalFallback") === "true";
+ const savedFallbackModel =
+ localStorage.getItem("fallbackWhisperModel") || "base";
setUseLocalWhisper(savedUseLocal);
setWhisperModel(savedModel);
- setAllowOpenAIFallback(savedAllowFallback);
+ setAllowOpenAIFallback(savedAllowOpenAIFallback);
+ setAllowLocalFallback(savedAllowLocalFallback);
+ setFallbackWhisperModel(savedFallbackModel);
// Load API key from main process first, then fallback to localStorage
const loadApiKey = async () => {
@@ -177,27 +185,33 @@ export default function SettingsPage({ onBack }: SettingsPageProps) {
try {
await window.electronAPI.saveOpenAIKey(apiKey);
localStorage.setItem("openaiApiKey", apiKey);
+ localStorage.setItem("allowLocalFallback", allowLocalFallback.toString());
+ localStorage.setItem("fallbackWhisperModel", fallbackWhisperModel);
try {
await window.electronAPI.createProductionEnvFile(apiKey);
setAlertDialog({
open: true,
title: "API Key Saved",
- description:
- "OpenAI API key inscribed successfully! Your credentials have been securely recorded for transcription services.",
+ description: `OpenAI API key inscribed successfully! Your credentials have been securely recorded for transcription services.${
+ allowLocalFallback ? " Local Whisper fallback is enabled." : ""
+ }`,
});
} catch (envError) {
console.log("Could not create production .env file:", envError);
setAlertDialog({
open: true,
title: "API Key Saved",
- description:
- "OpenAI API key saved successfully and will be available for transcription",
+ description: `OpenAI API key saved successfully and will be available for transcription${
+ allowLocalFallback ? " with Local Whisper fallback enabled" : ""
+ }`,
});
}
} catch (error) {
console.error("Failed to save API key:", error);
localStorage.setItem("openaiApiKey", apiKey);
+ localStorage.setItem("allowLocalFallback", allowLocalFallback.toString());
+ localStorage.setItem("fallbackWhisperModel", fallbackWhisperModel);
setAlertDialog({
open: true,
title: "API Key Saved",
@@ -211,9 +225,13 @@ export default function SettingsPage({ onBack }: SettingsPageProps) {
localStorage.setItem("whisperModel", whisperModel);
localStorage.setItem("allowOpenAIFallback", allowOpenAIFallback.toString());
- // Also save API key if fallback is enabled and key is provided
+ // SECURITY: Only save API key if fallback is enabled and key is provided
+ // This ensures we never store unused keys
if (allowOpenAIFallback && apiKey.trim()) {
localStorage.setItem("openaiApiKey", apiKey);
+ } else if (!allowOpenAIFallback) {
+ // Clear the key if fallback is disabled for security
+ localStorage.removeItem("openaiApiKey");
}
setAlertDialog({
@@ -338,6 +356,83 @@ export default function SettingsPage({ onBack }: SettingsPageProps) {
className="text-blue-800"
helpText="Get your API key from platform.openai.com"
/>
+
+ {/* Local Whisper Fallback Toggle */}
+ {whisperHook.whisperInstalled && (
+ <div className="space-y-3 mt-4">
+ <div className="flex items-center justify-between">
+ <label className="text-sm font-medium text-blue-800">
+ Fall back to Local Whisper
+ </label>
+ <label className="relative inline-flex items-center cursor-pointer">
+ <input
+ type="checkbox"
+ className="sr-only"
+ checked={allowLocalFallback}
+ onChange={(e) =>
+ setAllowLocalFallback(e.target.checked)
+ }
+ />
+ <div
+ className={`w-11 h-6 bg-gray-200 rounded-full peer peer-focus:ring-2 peer-focus:ring-blue-300 ${
+ allowLocalFallback
+ ? "bg-blue-600"
+ : "bg-gray-300"
+ } transition-colors duration-200`}
+ >
+ <div
+ className={`absolute top-0.5 left-0.5 bg-white border border-gray-300 rounded-full h-5 w-5 transition-transform duration-200 ${
+ allowLocalFallback
+ ? "translate-x-5"
+ : "translate-x-0"
+ }`}
+ ></div>
+ </div>
+ </label>
+ </div>
+ <p className="text-xs text-blue-700">
+ If OpenAI API fails, try Local Whisper as backup
+ </p>
+
+ {/* Fallback Model Selection - only show if fallback is enabled */}
+ {allowLocalFallback && (
+ <div className="p-3 bg-blue-50 border border-blue-200 rounded-lg">
+ <label className="block text-sm font-medium text-blue-800 mb-2">
+ Fallback Whisper Model
+ </label>
+ <select
+ value={fallbackWhisperModel}
+ onChange={(e) =>
+ setFallbackWhisperModel(e.target.value)
+ }
+ className="w-full text-sm border border-blue-300 rounded-md p-2 bg-white focus:border-blue-500 focus:ring-1 focus:ring-blue-500"
+ >
+ <option value="tiny">
+ Tiny - Fast, basic quality
+ </option>
+ <option value="base">
+ Base - Balanced (Recommended)
+ </option>
+ <option value="small">
+ Small - Better quality
+ </option>
+ <option value="medium">
+ Medium - High quality
+ </option>
+ <option value="large">
+ Large - Best quality
+ </option>
+ <option value="turbo">
+ Turbo - Fast & high quality
+ </option>
+ </select>
+ <p className="text-xs text-blue-600 mt-1">
+ Only used if OpenAI API fails
+ </p>
+ </div>
+ )}
+ </div>
+ )}
</div>
)}
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index 01c975844..c6e8439bc 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -1,4 +1,4 @@
-import TextCleanup from '../utils/textCleanup';
+import TextCleanup from "../utils/textCleanup";
class AudioManager {
constructor() {
@@ -107,7 +107,7 @@ class AudioManager {
removeRepetitions: true,
capitalizeFirst: true,
addPeriod: false, // Don't auto-add periods to preserve user intent
- ...options
+ ...options,
});
}
@@ -143,13 +143,17 @@ class AudioManager {
throw error;
}
- // Try fallback to OpenAI API if enabled
- const allowFallback =
+ // Try fallback to OpenAI API ONLY if enabled AND we're in local mode
+ const allowOpenAIFallback =
localStorage.getItem("allowOpenAIFallback") === "true";
- if (allowFallback) {
+ const isLocalMode = localStorage.getItem("useLocalWhisper") === "true";
+
+ if (allowOpenAIFallback && isLocalMode) {
console.log("Falling back to OpenAI API...");
try {
- return await this.processWithOpenAIAPI(audioBlob);
+ const fallbackResult = await this.processWithOpenAIAPI(audioBlob);
+ // Mark the result as fallback for user awareness
+ return { ...fallbackResult, source: "openai-fallback" };
} catch (fallbackError) {
// If OpenAI fallback also fails, throw the original error
throw new Error(
@@ -157,6 +161,7 @@ class AudioManager {
);
}
} else {
+ // No fallback allowed - strict local mode
throw new Error(`Local Whisper failed: ${error.message}`);
}
}
@@ -214,6 +219,40 @@ class AudioManager {
}
} catch (error) {
console.error("OpenAI API error:", error);
+
+ // Try fallback to Local Whisper ONLY if enabled AND we're in OpenAI mode
+ const allowLocalFallback =
+ localStorage.getItem("allowLocalFallback") === "true";
+ const isOpenAIMode = localStorage.getItem("useLocalWhisper") !== "true";
+
+ if (allowLocalFallback && isOpenAIMode) {
+ console.log("Falling back to Local Whisper...");
+ const fallbackModel =
+ localStorage.getItem("fallbackWhisperModel") || "base";
+ try {
+ const arrayBuffer = await audioBlob.arrayBuffer();
+ const options = { model: fallbackModel };
+ const result = await window.electronAPI.transcribeLocalWhisper(
+ arrayBuffer,
+ options
+ );
+
+ if (result.success && result.text) {
+ let text = AudioManager.cleanTranscription(result.text);
+ if (text) {
+ return { success: true, text, source: "local-fallback" };
+ }
+ }
+ // If local fallback fails, throw the original OpenAI error
+ throw error;
+ } catch (fallbackError) {
+ console.error("Local fallback also failed:", fallbackError);
+ throw new Error(
+ `OpenAI API failed: ${error.message}. Local fallback also failed: ${fallbackError.message}`
+ );
+ }
+ }
+
throw error;
}
}