Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

feat(transcription): add Gemini as a BYOK cloud transcription provider (#1899)

Gabriel Steinccf837c8fc8d
diff --git a/.github/workflows/stt-canary.yml b/.github/workflows/stt-canary.yml
index 01685d261..0009951a6 100644
--- a/.github/workflows/stt-canary.yml
+++ b/.github/workflows/stt-canary.yml
@@ -1,6 +1,7 @@
-# Weekly live check that every realtime STT provider still accepts the app's
-# credential flow and WebSocket handshake. Deliberately NOT part of PR CI: it
-# exercises external APIs and its failures are provider news, not code
+# Weekly live check that every STT provider still accepts the app's credential
+# flow — WebSocket handshake for realtime providers, a real transcription for
+# batch providers with a bespoke request shape. Deliberately NOT part of PR CI:
+# it exercises external APIs and its failures are provider news, not code
# regressions. Same pattern as llm-canary.yml.
name: stt-canary
@@ -30,6 +31,7 @@ jobs:
STT_CANARY_DEEPGRAM_KEY: ${{ secrets.STT_CANARY_DEEPGRAM_KEY }}
STT_CANARY_ASSEMBLYAI_KEY: ${{ secrets.STT_CANARY_ASSEMBLYAI_KEY }}
STT_CANARY_TINFOIL_KEY: ${{ secrets.STT_CANARY_TINFOIL_KEY }}
+ STT_CANARY_GEMINI_KEY: ${{ secrets.STT_CANARY_GEMINI_KEY }}
run: |
set -o pipefail
node scripts/stt-canary.mjs | tee canary-report.md
@@ -38,7 +40,7 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
run: |
- title="STT canary: realtime provider handshake failing"
+ title="STT canary: provider credential flow failing"
body="$(cat canary-report.md)
_Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}_"
diff --git a/preload.js b/preload.js
index 13feaa022..98ac8924c 100644
--- a/preload.js
+++ b/preload.js
@@ -98,8 +98,7 @@ contextBridge.exposeInMainWorld("electronAPI", {
micWarmHoldChanged: (active) => ipcRenderer.send("mic-warm-hold-changed", active),
dictationLifecycleStateChanged: (state, inputKind) =>
ipcRenderer.send("dictation-lifecycle-state-changed", state, inputKind),
- dictationAudioLevelChanged: (level) =>
- ipcRenderer.send("dictation-audio-level-changed", level),
+ dictationAudioLevelChanged: (level) => ipcRenderer.send("dictation-audio-level-changed", level),
toggleAgentPanelDictation: () => ipcRenderer.invoke("toggle-agent-panel-dictation"),
cancelAgentPanelDictation: () => ipcRenderer.invoke("cancel-agent-panel-dictation"),
getAgentDictationPillState: () => ipcRenderer.invoke("get-agent-dictation-pill-state"),
@@ -553,6 +552,7 @@ contextBridge.exposeInMainWorld("electronAPI", {
// xAI / Mistral transcription proxies (keys handled by the manifest bridge)
proxyXaiTranscription: (data) => ipcRenderer.invoke("proxy-xai-transcription", data),
proxyMistralTranscription: (data) => ipcRenderer.invoke("proxy-mistral-transcription", data),
+ proxyGeminiTranscription: (data) => ipcRenderer.invoke("proxy-gemini-transcription", data),
// Corti API
getCortiClientId: () => ipcRenderer.invoke("get-corti-client-id"),
diff --git a/scripts/stt-canary.mjs b/scripts/stt-canary.mjs
index 16ca52ff4..7fc34094a 100644
--- a/scripts/stt-canary.mjs
+++ b/scripts/stt-canary.mjs
@@ -1,10 +1,13 @@
/**
- * Live realtime-STT canary. Per keyed provider, acquires a session credential
- * through the app's REAL token registry (realtimeTokenProviders.js) and, where
- * the transport is a plain WebSocket, performs a live handshake against the
- * same endpoint the app dials — so a provider changing its auth, endpoint, or
+ * Live STT canary. Per keyed provider, acquires a session credential through
+ * the app's REAL token registry (realtimeTokenProviders.js) and, where the
+ * transport is a plain WebSocket, performs a live handshake against the same
+ * endpoint the app dials — so a provider changing its auth, endpoint, or
* handshake surfaces here on a schedule instead of in a customer report
* (#1624 sat in shipped builds for three days as a swallowed warmup warning).
+ * Batch providers whose request shape is theirs alone rather than the shared
+ * OpenAI-compatible multipart (Gemini) send a real transcription through the
+ * shipped module for the same reason.
*
* Run: node scripts/stt-canary.mjs
* Keys come from STT_CANARY_<PROVIDER>_KEY env vars; providers without a key
@@ -13,12 +16,24 @@
* not report green.
* Corti is not probed (needs tenant/environment credentials beyond a key).
*/
+import { spawnSync } from "child_process";
+import fs from "fs";
+import os from "os";
+import path from "path";
import WebSocket from "ws";
+import ffmpegPath from "ffmpeg-static";
import tokenProviders from "../src/helpers/realtimeTokenProviders.js";
+import audioUtils from "../src/utils/audioUtils.js";
+import geminiTranscription from "../src/helpers/geminiTranscription.js";
const { fetchRealtimeTokenForProvider } = tokenProviders;
+const { pcm16ToWav } = audioUtils;
+const { transcribeWithGemini } = geminiTranscription;
const HANDSHAKE_TIMEOUT_MS = 15000;
+// Half a second of 16 kHz mono silence: enough for a provider to accept and
+// decode the container, short enough to stay far inside every request cap.
+const SILENT_WAV = () => pcm16ToWav(Buffer.alloc(16000));
// Mirrors the app's dial: openaiRealtimeStreaming.js connects with a bare
// Bearer header; deepgramStreaming.js passes the key as the bearer token.
@@ -68,6 +83,33 @@ function probeWebSocket(url, token, { awaitServerEvent = false } = {}) {
});
}
+// Dictation records WebM/Opus, and Gemini's documented input formats do not
+// list it — so wav and webm are probed separately: a wav-only probe would stay
+// green while every BYOK dictation failed.
+function toWebmOpus(wav) {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "stt-canary-"));
+ const input = path.join(dir, "in.wav");
+ const output = path.join(dir, "out.webm");
+ try {
+ fs.writeFileSync(input, wav);
+ const result = spawnSync(ffmpegPath, ["-y", "-i", input, "-c:a", "libopus", output], {
+ stdio: "ignore",
+ });
+ if (result.status !== 0) return null;
+ return fs.readFileSync(output);
+ } finally {
+ fs.rmSync(dir, { recursive: true, force: true });
+ }
+}
+
+// A completed interaction proves auth, endpoint, and container decode; silence
+// legitimately transcribes to empty text, so text content is not asserted.
+async function probeGeminiBatch(key, audio, contentType) {
+ if (!audio) return { ok: false, detail: `could not build ${contentType} fixture` };
+ await transcribeWithGemini({ audioBuffer: audio, contentType, apiKey: key }, fetch);
+ return { ok: true };
+}
+
const tokenDeps = (key) => ({
environmentManager: {
getOpenAIKey: () => key,
@@ -131,6 +173,16 @@ const PROBES = [
: { ok: false, detail: "empty token" };
},
},
+ {
+ id: "gemini-batch-wav",
+ keyEnv: "STT_CANARY_GEMINI_KEY",
+ run: (key) => probeGeminiBatch(key, SILENT_WAV(), "audio/wav"),
+ },
+ {
+ id: "gemini-batch-webm",
+ keyEnv: "STT_CANARY_GEMINI_KEY",
+ run: (key) => probeGeminiBatch(key, toWebmOpus(SILENT_WAV()), "audio/webm"),
+ },
];
const report = [];
@@ -153,7 +205,7 @@ if (skipped.length === PROBES.length) {
failures.push("no canary secrets configured — every provider was skipped, nothing was probed");
}
-console.log("## Realtime STT canary\n");
+console.log("## STT canary\n");
console.log("| provider | result |\n|---|---|");
for (const line of report) console.log(line);
if (skipped.length) console.log(`\nSkipped (no key configured): ${skipped.join(", ")}`);
diff --git a/src/components/TranscriptionModelPicker.tsx b/src/components/TranscriptionModelPicker.tsx
index 651e27cde..9e6cfb2e5 100644
--- a/src/components/TranscriptionModelPicker.tsx
+++ b/src/components/TranscriptionModelPicker.tsx
@@ -221,6 +221,7 @@ const CLOUD_PROVIDER_TABS = [
{ id: "groq", name: "Groq" },
{ id: "xai", name: "xAI" },
{ id: "mistral", name: "Mistral" },
+ { id: "gemini", name: "Gemini" },
{ id: "corti", name: "Corti" },
{ id: "tinfoil", name: "Tinfoil" },
{ id: "custom", name: "Custom" },
@@ -232,6 +233,7 @@ interface ProviderCredentialField {
| "groqApiKey"
| "xaiApiKey"
| "mistralApiKey"
+ | "geminiApiKey"
| "cortiClientId"
| "cortiClientSecret"
| "cortiEnvironment"
@@ -263,6 +265,10 @@ const PROVIDER_CREDENTIALS: Record<
consoleUrl: "https://console.mistral.ai/api-keys",
fields: [{ key: "mistralApiKey", input: "secret" }],
},
+ gemini: {
+ consoleUrl: "https://aistudio.google.com/apikey",
+ fields: [{ key: "geminiApiKey", input: "secret" }],
+ },
corti: {
consoleUrl: "https://www.corti.ai/?utm_source=referral&utm_content=&utm_campaign=openwhispr",
fields: [
@@ -365,6 +371,8 @@ export default function TranscriptionModelPicker({
const setXaiApiKey = useSettingsStore((s) => s.setXaiApiKey);
const mistralApiKey = useSettingsStore((s) => s.mistralApiKey);
const setMistralApiKey = useSettingsStore((s) => s.setMistralApiKey);
+ const geminiApiKey = useSettingsStore((s) => s.geminiApiKey);
+ const setGeminiApiKey = useSettingsStore((s) => s.setGeminiApiKey);
const cortiClientId = useSettingsStore((s) => s.cortiClientId);
const setCortiClientId = useSettingsStore((s) => s.setCortiClientId);
const cortiClientSecret = useSettingsStore((s) => s.cortiClientSecret);
@@ -922,6 +930,7 @@ export default function TranscriptionModelPicker({
groqApiKey,
xaiApiKey,
mistralApiKey,
+ geminiApiKey,
cortiClientId,
cortiClientSecret,
cortiEnvironment,
@@ -933,6 +942,7 @@ export default function TranscriptionModelPicker({
groqApiKey: setGroqApiKey,
xaiApiKey: setXaiApiKey,
mistralApiKey: setMistralApiKey,
+ geminiApiKey: setGeminiApiKey,
cortiClientId: setCortiClientId,
cortiClientSecret: setCortiClientSecret,
cortiEnvironment: setCortiEnvironment,
diff --git a/src/components/notes/BatchQueueView.tsx b/src/components/notes/BatchQueueView.tsx
index 62ae07153..c5d26ae0c 100644
--- a/src/components/notes/BatchQueueView.tsx
+++ b/src/components/notes/BatchQueueView.tsx
@@ -6,6 +6,7 @@ import type { QueueItem } from "../../stores/batchQueueStore";
interface BatchQueueViewProps {
queue: QueueItem[];
+ byokMaxFileSizeMb: number;
completedCount: number;
failedCount: number;
totalCount: number;
@@ -31,6 +32,7 @@ function StatusIcon({ status }: { status: QueueItem["status"] }) {
export default function BatchQueueView({
queue,
+ byokMaxFileSizeMb,
completedCount,
failedCount,
totalCount,
@@ -127,9 +129,15 @@ export default function BatchQueueView({
{item.status === "error" && item.error && (
<span
className="text-[10px] text-destructive/50 truncate max-w-20"
- title={t(`notes.upload.${item.error}`, { defaultValue: item.error })}
+ title={t(`notes.upload.${item.error}`, {
+ defaultValue: item.error,
+ size: byokMaxFileSizeMb,
+ })}
>
- {t(`notes.upload.${item.error}`, { defaultValue: item.error })}
+ {t(`notes.upload.${item.error}`, {
+ defaultValue: item.error,
+ size: byokMaxFileSizeMb,
+ })}
</span>
)}
diff --git a/src/components/notes/UploadAudioView.tsx b/src/components/notes/UploadAudioView.tsx
index 46e723e2b..31147ec05 100644
--- a/src/components/notes/UploadAudioView.tsx
+++ b/src/components/notes/UploadAudioView.tsx
@@ -67,14 +67,13 @@ import { getBaseLanguageCode } from "../../utils/languageSupport";
import { isTranscriptionContextAllowed } from "../../stores/policyRules";
import { usePolicyStore } from "../../stores/policyStore";
import { usePolicySnapshot, useTranscriptionContextAllowed } from "../../hooks/usePolicy";
-import { resolveTranscriptionRoute } from "../../helpers/transcriptionRoute";
+import { byokFileSizeLimit, resolveTranscriptionRoute } from "../../helpers/transcriptionRoute";
import { saveUploadNote, uploadTitleFallback } from "../../services/uploadNotes";
type UploadState = "idle" | "selected" | "downloading" | "transcribing" | "complete" | "error";
const SUPPORTED_EXTENSIONS = ["mp3", "wav", "m4a", "webm", "ogg", "oga", "flac", "aac", "opus"];
-const BYOK_MAX_FILE_SIZE = 25 * 1024 * 1024; // 25 MB — hard limit for bring-your-own-key
const CLOUD_FREE_MAX_FILE_SIZE = 25 * 1024 * 1024; // 25 MB — free plan cloud limit
const CLOUD_PRO_MAX_FILE_SIZE = 500 * 1024 * 1024; // 500 MB — pro plan cloud limit
@@ -253,6 +252,7 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
groqApiKey,
xaiApiKey,
mistralApiKey,
+ geminiApiKey,
tinfoilApiKey,
customTranscriptionApiKey,
} = apiKeys;
@@ -307,9 +307,11 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
// Mode-aware file size validation
// Local: no limits at all
- // BYOK: 25 MB hard max regardless of plan
+ // BYOK: 25 MB hard max regardless of plan (14 MB for Gemini's inline cap)
// Cloud free: 25 MB max (upgrade to Pro for more)
// Cloud pro: 500 MB max
+ const byokMaxFileSize = byokFileSizeLimit(cloudTranscriptionProvider);
+ const byokMaxFileSizeMb = Math.floor(byokMaxFileSize / (1024 * 1024));
let fileTooLarge = false;
let requiresUpgrade = false;
let requiresAccount = false;
@@ -322,7 +324,7 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
} else if (isSelfHosted || cloudTranscriptionProvider === "custom") {
// Self-hosted / custom endpoints (e.g. local whisper.cpp): no file size restrictions
} else if (isByok) {
- byokTooLarge = file.sizeBytes > BYOK_MAX_FILE_SIZE;
+ byokTooLarge = file.sizeBytes > byokMaxFileSize;
if (byokTooLarge && !isSignedIn) {
requiresAccount = true;
}
@@ -406,6 +408,7 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
groqApiKey,
xaiApiKey,
mistralApiKey,
+ geminiApiKey,
tinfoilApiKey,
customTranscriptionApiKey,
})
@@ -443,6 +446,7 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
groqApiKey,
xaiApiKey,
mistralApiKey,
+ geminiApiKey,
tinfoilApiKey,
customTranscriptionApiKey,
cortiClientId,
@@ -490,7 +494,7 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
// Batch counterpart of the single-file size gating above; returns keys under notes.upload.*.
const getBatchSizeErrorKey = (sizeBytes: number): string | null => {
if (useLocalWhisper || isSelfHosted || cloudTranscriptionProvider === "custom") return null;
- if (isByok) return sizeBytes > BYOK_MAX_FILE_SIZE ? "byokTooLarge" : null;
+ if (isByok) return sizeBytes > byokMaxFileSize ? "byokTooLarge" : null;
if (sizeBytes > CLOUD_PRO_MAX_FILE_SIZE) return "fileTooLarge";
if (!isProUser && sizeBytes > CLOUD_FREE_MAX_FILE_SIZE) return "paidPlanRequired";
return null;
@@ -1042,6 +1046,7 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
<div className="mt-3">
<BatchQueueView
queue={batch.queue}
+ byokMaxFileSizeMb={byokMaxFileSizeMb}
completedCount={batch.completedCount}
failedCount={batch.failedCount}
totalCount={batch.totalCount}
@@ -1100,6 +1105,7 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
isLargeFile={isLargeFile}
isOpenWhisprCloud={isOpenWhisprCloud}
byokTooLarge={byokTooLarge}
+ byokMaxFileSizeMb={byokMaxFileSizeMb}
requiresAccount={requiresAccount}
isProUser={!!isProUser}
onUpgrade={() => usage?.openCheckout()}
@@ -1517,7 +1523,7 @@ function IdleView({
}
interface SelectedViewProps {
- t: (key: string) => string;
+ t: (key: string, options?: Record<string, unknown>) => string;
file: { name: string; path: string; size: string; sizeBytes: number };
getActiveModelLabel: () => string;
reset: () => void;
@@ -1528,6 +1534,7 @@ interface SelectedViewProps {
isLargeFile: boolean;
isOpenWhisprCloud: boolean;
byokTooLarge: boolean;
+ byokMaxFileSizeMb: number;
requiresAccount: boolean;
isProUser: boolean;
onUpgrade: () => void;
@@ -1547,6 +1554,7 @@ function SelectedView({
isLargeFile,
isOpenWhisprCloud,
byokTooLarge,
+ byokMaxFileSizeMb,
requiresAccount,
isProUser,
onUpgrade,
@@ -1589,10 +1597,10 @@ function SelectedView({
{byokTooLarge && (
<div className="rounded-lg border border-primary/12 dark:border-primary/15 bg-primary/[0.03] px-3 py-2.5 mb-3">
<p className="text-xs text-foreground/50 leading-relaxed">
- {t("notes.upload.byokTooLarge")}
+ {t("notes.upload.byokTooLarge", { size: byokMaxFileSizeMb })}
</p>
<p className="text-xs text-foreground/35 leading-relaxed mt-1.5">
- {t("notes.upload.byokTooLargeDetail")}
+ {t("notes.upload.byokTooLargeDetail", { size: byokMaxFileSizeMb })}
</p>
<p className="text-xs text-foreground/50 leading-relaxed mt-1.5 font-medium">
{requiresAccount
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index 32c6c20e9..b8f99104a 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -374,6 +374,16 @@ const PROXY_TRANSCRIPTION_PROVIDERS = {
return payload;
},
},
+ gemini: {
+ displayName: "Gemini",
+ ipc: () => window.electronAPI?.proxyGeminiTranscription,
+ buildPayload: ({ audioBuffer, model, language, keyterms }) => ({
+ audioBuffer,
+ model,
+ language,
+ keyterms: keyterms.length > 0 ? keyterms : undefined,
+ }),
+ },
xai: {
displayName: "xAI",
ipc: () => window.electronAPI?.proxyXaiTranscription,
@@ -2255,6 +2265,18 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
err.code = "API_KEY_MISSING";
throw err;
}
+ } else if (provider === "gemini") {
+ apiKey = s.geminiApiKey;
+ if (!apiKey?.trim()) {
+ apiKey = await window.electronAPI.getGeminiKey?.();
+ }
+ if (!apiKey?.trim()) {
+ const err = new Error(
+ "Gemini API key not found. Please set your API key in the Control Panel."
+ );
+ err.code = "API_KEY_MISSING";
+ throw err;
+ }
} else if (provider === "groq") {
// Prefer store value (user-entered via UI) over main process (.env)
apiKey = s.groqApiKey;
diff --git a/src/helpers/geminiTranscription.js b/src/helpers/geminiTranscription.js
new file mode 100644
index 000000000..81bb3a0b1
--- /dev/null
+++ b/src/helpers/geminiTranscription.js
@@ -0,0 +1,114 @@
+const { net } = require("electron");
+const debugLogger = require("./debugLogger");
+
+// gemini-3.5-transcribe is only served by the Interactions API; there is no
+// generateContent or OpenAI-compatible endpoint for it.
+const GEMINI_INTERACTIONS_URL = "https://generativelanguage.googleapis.com/v1beta/interactions";
+
+// Gemini documents audio/mp3 and audio/aac instead of the audio/mpeg and
+// audio/mp4 types the rest of the pipeline uses.
+const GEMINI_MIME_TYPES = {
+ "audio/mpeg": "audio/mp3",
+ "audio/mp4": "audio/aac",
+};
+
+// The full transcript lives in output_text, but the documented sample response
+// omits it — joining the steps' text content is the safe fallback.
+function extractText(data) {
+ const outputText = data?.output_text;
+ if (typeof outputText === "string" && outputText.trim()) {
+ return outputText;
+ }
+ return (data?.steps || [])
+ .flatMap((step) => step?.content || [])
+ .filter((part) => part?.type === "text" && typeof part.text === "string")
+ .map((part) => part.text)
+ .join(" ");
+}
+
+async function transcribeWithGemini(
+ { audioBuffer, model, contentType, language, keyterms, apiKey },
+ fetchImpl
+) {
+ if (!apiKey?.trim()) {
+ const error = new Error("Gemini API key not configured. Add your key in Settings.");
+ error.code = "API_KEY_MISSING";
+ throw error;
+ }
+
+ const resolvedModel = model || "gemini-3.5-transcribe";
+ const transcriptionConfig = {};
+ if (language && language !== "auto") {
+ transcriptionConfig.language_codes = [language];
+ }
+ if (keyterms && keyterms.length > 0) {
+ transcriptionConfig.custom_vocabulary = keyterms;
+ }
+ const requestBody = {
+ model: resolvedModel,
+ input: [
+ {
+ type: "audio",
+ data: (Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer)).toString(
+ "base64"
+ ),
+ mime_type: GEMINI_MIME_TYPES[contentType] || contentType,
+ },
+ ],
+ };
+ if (Object.keys(transcriptionConfig).length > 0) {
+ requestBody.generation_config = { transcription_config: transcriptionConfig };
+ }
+
+ debugLogger.debug(
+ "Gemini batch transcription starting",
+ { model: resolvedModel, language, audioBytes: audioBuffer.byteLength },
+ "transcription"
+ );
+
+ const doFetch = fetchImpl || ((url, init) => net.fetch(url, init));
+ const response = await doFetch(GEMINI_INTERACTIONS_URL, {
+ method: "POST",
+ headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey },
+ body: JSON.stringify(requestBody),
+ });
+
+ if (!response.ok) {
+ const errorText = await response.text().catch(() => "");
+ // Google rejects a bad key with 400 + reason API_KEY_INVALID rather than 401,
+ // so status alone would surface the raw JSON instead of a fixable message.
+ if (
+ response.status === 401 ||
+ response.status === 403 ||
+ errorText.includes("API_KEY_INVALID")
+ ) {
+ const error = new Error("Invalid Gemini API key. Check your key in Settings.");
+ error.code = "INVALID_KEY";
+ throw error;
+ }
+ const error = new Error(`Gemini API Error: ${response.status} ${errorText}`.trim());
+ if (response.status === 429) {
+ error.code = "PROVIDER_RATE_LIMITED";
+ error.messageKey = "hooks.audioRecording.errorDescriptions.providerRateLimited";
+ } else if (response.status >= 500) {
+ error.code = "SERVER_ERROR";
+ }
+ throw error;
+ }
+
+ const data = await response.json();
+ // Anything short of "completed" (failed, cancelled, budget_exceeded, ...) must
+ // not fall through to extractText, which would return empty or truncated text
+ // as a successful transcription.
+ if (data?.status && data.status !== "completed") {
+ const detail = data?.error?.message;
+ throw new Error(
+ detail
+ ? `Gemini transcription failed: ${detail}`
+ : `Gemini transcription did not complete (status: ${data.status})`
+ );
+ }
+ return { text: extractText(data), model: resolvedModel };
+}
+
+module.exports = { transcribeWithGemini };
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 2c7ce4a72..eedec23e4 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -68,6 +68,7 @@ const { createTinfoilRealtimeSocket } = require("./tinfoilSecureClient");
const { TINFOIL_REALTIME_MODEL } = require("./tinfoilRealtimeStreaming");
const { getTinfoilChatModels } = require("./tinfoilCatalog");
const { transcribeWithTinfoil } = require("./tinfoilTranscription");
+const { transcribeWithGemini } = require("./geminiTranscription");
const AudioStorageManager = require("./audioStorage");
const AgentStreamRequestRegistry = require("./agentStreamRequestRegistry");
const createMeetingTranscriptionLifecycle = require("./meetingTranscriptionLifecycle");
@@ -146,6 +147,11 @@ const XAI_STT_URL = "https://api.x.ai/v1/stt";
// Debounce delay: wait for user to stop typing before processing corrections
const AUTO_LEARN_DEBOUNCE_MS = 1500;
+// Route caps vary by provider (Gemini's inline-base64 limit is the lowest), so
+// the message reports the cap that actually applied.
+const byokSizeCapError = (sizeCapBytes) =>
+ `File too large. Maximum size for bring-your-own-key is ${Math.floor(sizeCapBytes / (1024 * 1024))} MB.`;
+
const AUDIO_MIME_TYPES = {
mp3: "audio/mpeg",
wav: "audio/wav",
@@ -4233,6 +4239,22 @@ class IPCHandlers {
})
);
+ // Gemini's Interactions API takes JSON with inline base64 audio, not
+ // OpenAI-compatible multipart, so batch transcription is proxied through main.
+ ipcMain.handle(
+ "proxy-gemini-transcription",
+ serializeIpcError(async (event, { audioBuffer, model, language, keyterms }) => {
+ return await transcribeWithGemini({
+ audioBuffer: Buffer.from(audioBuffer),
+ model,
+ contentType: "audio/webm",
+ language,
+ keyterms,
+ apiKey: this.environmentManager.getGeminiKey(),
+ });
+ })
+ );
+
ipcMain.handle("get-custom-transcription-key", async () => {
return this.environmentManager.getCustomTranscriptionKey();
});
@@ -5881,6 +5903,18 @@ class IPCHandlers {
language: route.language,
});
if (text) result = { text, source: "corti", model: route.model };
+ } else if (route.transport === "proxied" && route.provider === "gemini") {
+ if (route.sizeCapBytes && buffer.byteLength > route.sizeCapBytes) {
+ throw new Error(byokSizeCapError(route.sizeCapBytes));
+ }
+ const { text } = await transcribeWithGemini({
+ audioBuffer: buffer,
+ model: route.model,
+ contentType: "audio/webm",
+ language: route.language,
+ apiKey: this.environmentManager.getGeminiKey(),
+ });
+ if (text) result = { text, source: "gemini", model: route.model };
} else {
// mistral/xai have no OpenAI-compatible endpoint — main talks to them
// directly; everything else consumes the route endpoint as-is.
@@ -9119,10 +9153,7 @@ class IPCHandlers {
const fileSize = fs.statSync(realByok).size;
if (route.sizeCapBytes && fileSize > route.sizeCapBytes) {
- return {
- success: false,
- error: "File too large. Maximum size for bring-your-own-key is 25 MB.",
- };
+ return { success: false, error: byokSizeCapError(route.sizeCapBytes) };
}
if (route.transport === "proxied" && route.provider === "corti") {
@@ -9155,6 +9186,19 @@ class IPCHandlers {
return { success: true, text };
}
+ if (route.transport === "proxied" && route.provider === "gemini") {
+ const ext = path.extname(realByok).toLowerCase().replace(".", "");
+ // Deliberately no language hint — same rationale as the multipart
+ // branch below, and Gemini's language_codes is a hard constraint.
+ const { text } = await transcribeWithGemini({
+ audioBuffer: fs.readFileSync(realByok),
+ model: route.model,
+ contentType: AUDIO_MIME_TYPES[ext] || "audio/mpeg",
+ apiKey: apiKey || this.environmentManager.getGeminiKey(),
+ });
+ return { success: true, text };
+ }
+
if (!apiKey && route.provider !== "custom") {
throw new Error("No API key configured. Add your key in Settings.");
}
diff --git a/src/helpers/transcriptionRoute.ts b/src/helpers/transcriptionRoute.ts
index a73f08f60..623f5a94e 100644
--- a/src/helpers/transcriptionRoute.ts
+++ b/src/helpers/transcriptionRoute.ts
@@ -29,6 +29,14 @@ import {
const BYOK_FILE_SIZE_LIMIT = 25 * 1024 * 1024;
+// Gemini's Interactions API takes inline base64 audio inside a 20 MB total
+// request cap; base64 inflates 4/3, so cap the raw audio lower.
+const GEMINI_FILE_SIZE_LIMIT = 14 * 1024 * 1024;
+
+export function byokFileSizeLimit(provider: string): number {
+ return provider === "gemini" ? GEMINI_FILE_SIZE_LIMIT : BYOK_FILE_SIZE_LIMIT;
+}
+
const CUSTOM_ENDPOINT_INVALID_MESSAGE_KEY =
"hooks.audioRecording.errorDescriptions.customEndpointInvalid";
@@ -65,7 +73,7 @@ export type TranscriptionRoute =
| { transport: "local" }
| {
transport: "proxied";
- provider: "tinfoil" | "mistral" | "xai" | "corti";
+ provider: "tinfoil" | "mistral" | "xai" | "corti" | "gemini";
model: string | null;
language?: string;
sizeCapBytes: number;
@@ -124,13 +132,15 @@ export function resolveByokModel(provider: string, configuredModel?: string): st
(provider === "groq" && trimmed.startsWith("whisper-large-v3")) ||
(provider === "openai" && (trimmed.startsWith("gpt-4o") || trimmed === "whisper-1")) ||
(provider === "mistral" && trimmed.startsWith("voxtral-")) ||
- (provider === "corti" && trimmed.startsWith("corti-"));
+ (provider === "corti" && trimmed.startsWith("corti-")) ||
+ (provider === "gemini" && trimmed.startsWith("gemini-"));
if (matchesProvider) return trimmed;
}
if (provider === "groq") return "whisper-large-v3-turbo";
if (provider === "xai") return "grok-stt";
if (provider === "mistral") return "voxtral-mini-latest";
if (provider === "corti") return "corti-transcribe";
+ if (provider === "gemini") return "gemini-3.5-transcribe";
return "gpt-4o-mini-transcribe";
}
@@ -242,6 +252,15 @@ export function resolveTranscriptionRoute({
sizeCapBytes: BYOK_FILE_SIZE_LIMIT,
};
}
+ if (provider === "gemini") {
+ return {
+ transport: "proxied",
+ provider,
+ model,
+ language,
+ sizeCapBytes: GEMINI_FILE_SIZE_LIMIT,
+ };
+ }
if (provider === "corti") {
return {
transport: "proxied",
diff --git a/src/locales/de/translation.json b/src/locales/de/translation.json
index c8b71406f..3411ec303 100644
--- a/src/locales/de/translation.json
+++ b/src/locales/de/translation.json
@@ -2575,6 +2575,7 @@
"groq_whisper_large_v3": "Hochpräzise Spracherkennung",
"groq_whisper_large_v3_turbo": "216x Echtzeitgeschwindigkeit",
"mistral_voxtral_mini_latest": "Schnelle mehrsprachige Transkription",
+ "gemini_3_5_transcribe": "Googles Gemini-Spracherkennung",
"corti_transcribe": "Medizinische Transkription in klinischer Qualität",
"xai_grok_stt": "Hochpräzise Spracherkennung von xAI",
"tinfoil_voxtral": "Nachweisbar private Transkription"
@@ -3096,8 +3097,8 @@
"paidPlanRequired": "Cloud-Transkription für Dateien über 25 MB erfordert einen kostenpflichtigen Plan. Große Dateien werden mit OpenWhispr Cloud für zuverlässige, hochwertige Ergebnisse verarbeitet.",
"largeFileNote": "Große Dateien werden aufgeteilt und parallel transkribiert.",
"chunkProgress": "Transkribiere Teil {{completed}} von {{total}}...",
- "byokTooLarge": "API-Schlüssel von Drittanbietern unterstützen nur Dateien bis 25 MB.",
- "byokTooLargeDetail": "Für Dateien über 25 MB übernimmt OpenWhispr Cloud das Aufteilen, die parallele Verarbeitung und das Zusammenführen — so erhältst du zuverlässige Ergebnisse ohne die Einschränkungen von Drittanbietern.",
+ "byokTooLarge": "API-Schlüssel von Drittanbietern unterstützen nur Dateien bis {{size}} MB.",
+ "byokTooLargeDetail": "Für Dateien über {{size}} MB übernimmt OpenWhispr Cloud das Aufteilen, die parallele Verarbeitung und das Zusammenführen — so erhältst du zuverlässige Ergebnisse ohne die Einschränkungen von Drittanbietern.",
"byokTooLargeNeedsAccount": "Erstelle ein Konto und wechsle zu einem kostenpflichtigen Plan, um größere Dateien mit OpenWhispr Cloud zu transkribieren.",
"byokTooLargeNeedsUpgrade": "Wechsle zu einem kostenpflichtigen Plan, um größere Dateien mit OpenWhispr Cloud zu transkribieren.",
"switchToCloudForLargeFiles": "Wechsle zu OpenWhispr Cloud, um diese Datei zu transkribieren.",
diff --git a/src/locales/en/translation.json b/src/locales/en/translation.json
index 4dfbdef91..d8dc3c302 100644
--- a/src/locales/en/translation.json
+++ b/src/locales/en/translation.json
@@ -2575,6 +2575,7 @@
"groq_whisper_large_v3": "High accuracy speech recognition",
"groq_whisper_large_v3_turbo": "216x real-time speed",
"mistral_voxtral_mini_latest": "Fast multilingual transcription",
+ "gemini_3_5_transcribe": "Google's Gemini speech-to-text",
"corti_transcribe": "Clinical-grade medical transcription",
"xai_grok_stt": "High-accuracy speech recognition by xAI",
"tinfoil_voxtral": "Verifiably private transcription"
@@ -3096,8 +3097,8 @@
"paidPlanRequired": "Cloud transcription for files over 25 MB requires a paid plan. Large files are processed with OpenWhispr Cloud for reliable, high-quality results.",
"largeFileNote": "Large files are split and transcribed in parallel.",
"chunkProgress": "Transcribing part {{completed}} of {{total}}...",
- "byokTooLarge": "Third-party API keys only support files up to 25 MB.",
- "byokTooLargeDetail": "For files over 25 MB, OpenWhispr Cloud handles splitting, parallel processing, and reassembly — so you get reliable results without depending on third-party provider limits.",
+ "byokTooLarge": "Third-party API keys only support files up to {{size}} MB.",
+ "byokTooLargeDetail": "For files over {{size}} MB, OpenWhispr Cloud handles splitting, parallel processing, and reassembly — so you get reliable results without depending on third-party provider limits.",
"byokTooLargeNeedsAccount": "Create an account and upgrade to a paid plan to transcribe larger files with OpenWhispr Cloud.",
"byokTooLargeNeedsUpgrade": "Upgrade to a paid plan to transcribe larger files with OpenWhispr Cloud.",
"switchToCloudForLargeFiles": "Switch to OpenWhispr Cloud to transcribe this file.",
diff --git a/src/locales/es/translation.json b/src/locales/es/translation.json
index 9a2dc0118..d1b4f81b4 100644
--- a/src/locales/es/translation.json
+++ b/src/locales/es/translation.json
@@ -2530,6 +2530,7 @@
"groq_whisper_large_v3": "Reconocimiento de voz de alta precisión",
"groq_whisper_large_v3_turbo": "Velocidad 216x en tiempo real",
"mistral_voxtral_mini_latest": "Transcripción multilingüe rápida",
+ "gemini_3_5_transcribe": "Voz a texto con Gemini de Google",
"corti_transcribe": "Transcripción médica de nivel clínico",
"xai_grok_stt": "Reconocimiento de voz de alta precisión por xAI",
"tinfoil_voxtral": "Transcripción verificablemente privada"
@@ -2933,8 +2934,8 @@
"largeFileNote": "Los archivos grandes se dividen y transcriben en paralelo.",
"chunkProgress": "Transcribiendo parte {{completed}} de {{total}}...",
"paidPlanRequired": "La transcripción en la nube para archivos de más de 25 MB requiere un plan de pago. Los archivos grandes se procesan con OpenWhispr Cloud para resultados fiables y de alta calidad.",
- "byokTooLarge": "Las claves API de terceros solo admiten archivos de hasta 25 MB.",
- "byokTooLargeDetail": "Para archivos de más de 25 MB, OpenWhispr Cloud se encarga de dividir, procesar en paralelo y reensamblar, obteniendo resultados fiables sin depender de los límites de proveedores de terceros.",
+ "byokTooLarge": "Las claves API de terceros solo admiten archivos de hasta {{size}} MB.",
+ "byokTooLargeDetail": "Para archivos de más de {{size}} MB, OpenWhispr Cloud se encarga de dividir, procesar en paralelo y reensamblar, obteniendo resultados fiables sin depender de los límites de proveedores de terceros.",
"byokTooLargeNeedsAccount": "Crea una cuenta y cambia a un plan de pago para transcribir archivos más grandes con OpenWhispr Cloud.",
"byokTooLargeNeedsUpgrade": "Cambia a un plan de pago para transcribir archivos más grandes con OpenWhispr Cloud.",
"switchToCloudForLargeFiles": "Cambia a OpenWhispr Cloud para transcribir este archivo.",
diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.json
index 445259bf9..2146dcc40 100644
--- a/src/locales/fr/translation.json
+++ b/src/locales/fr/translation.json
@@ -2575,6 +2575,7 @@
"groq_whisper_large_v3": "Reconnaissance vocale haute précision",
"groq_whisper_large_v3_turbo": "Vitesse 216x en temps réel",
"mistral_voxtral_mini_latest": "Transcription multilingue rapide",
+ "gemini_3_5_transcribe": "Reconnaissance vocale Gemini de Google",
"corti_transcribe": "Transcription médicale de qualité clinique",
"xai_grok_stt": "Reconnaissance vocale haute précision par xAI",
"tinfoil_voxtral": "Transcription vérifiablement privée"
@@ -3096,8 +3097,8 @@
"paidPlanRequired": "La transcription cloud pour les fichiers de plus de 25 Mo nécessite un abonnement payant. Les fichiers volumineux sont traités avec OpenWhispr Cloud pour des résultats fiables et de haute qualité.",
"largeFileNote": "Les fichiers volumineux sont découpés et transcrits en parallèle.",
"chunkProgress": "Transcription de la partie {{completed}} sur {{total}}...",
- "byokTooLarge": "Les clés API tierces ne prennent en charge que les fichiers jusqu'à 25 Mo.",
- "byokTooLargeDetail": "Pour les fichiers de plus de 25 Mo, OpenWhispr Cloud gère le découpage, le traitement parallèle et le réassemblage — pour des résultats fiables sans dépendre des limites des fournisseurs tiers.",
+ "byokTooLarge": "Les clés API tierces ne prennent en charge que les fichiers jusqu'à {{size}} Mo.",
+ "byokTooLargeDetail": "Pour les fichiers de plus de {{size}} Mo, OpenWhispr Cloud gère le découpage, le traitement parallèle et le réassemblage — pour des résultats fiables sans dépendre des limites des fournisseurs tiers.",
"byokTooLargeNeedsAccount": "Créez un compte et passez à un abonnement payant pour transcrire des fichiers plus volumineux avec OpenWhispr Cloud.",
"byokTooLargeNeedsUpgrade": "Passez à un abonnement payant pour transcrire des fichiers plus volumineux avec OpenWhispr Cloud.",
"switchToCloudForLargeFiles": "Passez à OpenWhispr Cloud pour transcrire ce fichier.",
diff --git a/src/locales/it/translation.json b/src/locales/it/translation.json
index 7cc638969..87f85cff4 100644
--- a/src/locales/it/translation.json
+++ b/src/locales/it/translation.json
@@ -2482,6 +2482,7 @@
"groq_whisper_large_v3_turbo": "Velocità 216x in tempo reale",
"groq_whisper_large_v3": "Modello Large v3, veloce e preciso",
"mistral_voxtral_mini_latest": "Trascrizione multilingue veloce",
+ "gemini_3_5_transcribe": "Riconoscimento vocale Gemini di Google",
"corti_transcribe": "Trascrizione medica di livello clinico",
"xai_grok_stt": "Riconoscimento vocale ad alta precisione di xAI",
"tinfoil_voxtral": "Trascrizione verificabilmente privata"
@@ -2885,8 +2886,8 @@
"largeFileNote": "I file di grandi dimensioni vengono suddivisi e trascritti in parallelo.",
"chunkProgress": "Trascrizione parte {{completed}} di {{total}}...",
"paidPlanRequired": "La trascrizione cloud per file superiori a 25 MB richiede un piano a pagamento. I file di grandi dimensioni vengono elaborati con OpenWhispr Cloud per risultati affidabili e di alta qualità.",
- "byokTooLarge": "Le chiavi API di terze parti supportano solo file fino a 25 MB.",
- "byokTooLargeDetail": "Per file superiori a 25 MB, OpenWhispr Cloud gestisce la suddivisione, l'elaborazione parallela e il riassemblaggio — per risultati affidabili senza dipendere dai limiti dei provider di terze parti.",
+ "byokTooLarge": "Le chiavi API di terze parti supportano solo file fino a {{size}} MB.",
+ "byokTooLargeDetail": "Per file superiori a {{size}} MB, OpenWhispr Cloud gestisce la suddivisione, l'elaborazione parallela e il riassemblaggio — per risultati affidabili senza dipendere dai limiti dei provider di terze parti.",
"byokTooLargeNeedsAccount": "Crea un account e passa a un piano a pagamento per trascrivere file più grandi con OpenWhispr Cloud.",
"byokTooLargeNeedsUpgrade": "Passa a un piano a pagamento per trascrivere file più grandi con OpenWhispr Cloud.",
"switchToCloudForLargeFiles": "Passa a OpenWhispr Cloud per trascrivere questo file.",
diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.json
index 7de4cb9af..4531c3e0d 100644
--- a/src/locales/ja/translation.json
+++ b/src/locales/ja/translation.json
@@ -2482,6 +2482,7 @@
"groq_whisper_large_v3_turbo": "リアルタイムの 216 倍速",
"groq_whisper_large_v3": "Large v3 モデル、高速かつ高精度",
"mistral_voxtral_mini_latest": "高速多言語文字起こし",
+ "gemini_3_5_transcribe": "GoogleのGemini音声文字起こし",
"corti_transcribe": "臨床グレードの医療文字起こし",
"xai_grok_stt": "xAIによる高精度音声認識",
"tinfoil_voxtral": "検証可能なプライベート文字起こし"
@@ -2869,8 +2870,8 @@
"largeFileNote": "大容量ファイルは分割され、並列で文字起こしされます。",
"chunkProgress": "パート {{completed}} / {{total}} を文字起こし中...",
"paidPlanRequired": "25 MB を超えるファイルのクラウド文字起こしには有料プランが必要です。大容量ファイルは OpenWhispr Cloud で信頼性の高い高品質な結果を得られます。",
- "byokTooLarge": "サードパーティの API キーは 25 MB までのファイルのみ対応しています。",
- "byokTooLargeDetail": "25 MB を超えるファイルは、OpenWhispr Cloud が分割・並列処理・結合を行います。サードパーティの制限に依存せず、信頼性の高い結果が得られます。",
+ "byokTooLarge": "サードパーティの API キーは {{size}} MB までのファイルのみ対応しています。",
+ "byokTooLargeDetail": "{{size}} MB を超えるファイルは、OpenWhispr Cloud が分割・並列処理・結合を行います。サードパーティの制限に依存せず、信頼性の高い結果が得られます。",
"byokTooLargeNeedsAccount": "アカウントを作成し、有料プランにアップグレードすると、OpenWhispr Cloud でより大きなファイルを文字起こしできます。",
"byokTooLargeNeedsUpgrade": "有料プランにアップグレードすると、OpenWhispr Cloud でより大きなファイルを文字起こしできます。",
"switchToCloudForLargeFiles": "このファイルを文字起こしするには OpenWhispr Cloud に切り替えてください。",
diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.json
index d79503277..2bc2457ae 100644
--- a/src/locales/pt/translation.json
+++ b/src/locales/pt/translation.json
@@ -2461,6 +2461,7 @@
"groq_whisper_large_v3_turbo": "Velocidade 216x em tempo real",
"groq_whisper_large_v3": "Modelo Large v3, rápido e preciso",
"mistral_voxtral_mini_latest": "Transcrição multilíngue rápida",
+ "gemini_3_5_transcribe": "Fala para texto Gemini do Google",
"corti_transcribe": "Transcrição médica de nível clínico",
"xai_grok_stt": "Reconhecimento de voz de alta precisão pela xAI",
"tinfoil_voxtral": "Transcrição verificavelmente privada"
@@ -2885,8 +2886,8 @@
"largeFileNote": "Arquivos grandes são divididos e transcritos em paralelo.",
"chunkProgress": "Transcrevendo parte {{completed}} de {{total}}...",
"paidPlanRequired": "A transcrição na nuvem para arquivos acima de 25 MB requer um plano pago. Arquivos grandes são processados com o OpenWhispr Cloud para resultados confiáveis e de alta qualidade.",
- "byokTooLarge": "Chaves API de terceiros suportam apenas arquivos de até 25 MB.",
- "byokTooLargeDetail": "Para arquivos acima de 25 MB, o OpenWhispr Cloud cuida da divisão, processamento paralelo e remontagem — para resultados confiáveis sem depender dos limites de provedores de terceiros.",
+ "byokTooLarge": "Chaves API de terceiros suportam apenas arquivos de até {{size}} MB.",
+ "byokTooLargeDetail": "Para arquivos acima de {{size}} MB, o OpenWhispr Cloud cuida da divisão, processamento paralelo e remontagem — para resultados confiáveis sem depender dos limites de provedores de terceiros.",
"byokTooLargeNeedsAccount": "Crie uma conta e mude para um plano pago para transcrever arquivos maiores com o OpenWhispr Cloud.",
"byokTooLargeNeedsUpgrade": "Mude para um plano pago para transcrever arquivos maiores com o OpenWhispr Cloud.",
"switchToCloudForLargeFiles": "Mude para o OpenWhispr Cloud para transcrever este arquivo.",
diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.json
index 448652a2f..ee40d31b8 100644
--- a/src/locales/ru/translation.json
+++ b/src/locales/ru/translation.json
@@ -2498,6 +2498,7 @@
"groq_whisper_large_v3_turbo": "Скорость в 216 раз быстрее реального времени",
"groq_whisper_large_v3": "Модель Large v3, быстрая и точная",
"mistral_voxtral_mini_latest": "Быстрая многоязычная транскрипция",
+ "gemini_3_5_transcribe": "Распознавание речи Gemini от Google",
"corti_transcribe": "Медицинская транскрипция клинического уровня",
"xai_grok_stt": "Высокоточное распознавание речи от xAI",
"tinfoil_voxtral": "Проверяемо приватная транскрипция"
@@ -2907,8 +2908,8 @@
"largeFileNote": "Большие файлы разделяются и транскрибируются параллельно.",
"chunkProgress": "Транскрибирование части {{completed}} из {{total}}...",
"paidPlanRequired": "Облачная транскрипция файлов более 25 МБ требует платного плана. Большие файлы обрабатываются через OpenWhispr Cloud для надёжных и качественных результатов.",
- "byokTooLarge": "Сторонние API-ключи поддерживают файлы только до 25 МБ.",
- "byokTooLargeDetail": "Для файлов более 25 МБ OpenWhispr Cloud выполняет разделение, параллельную обработку и сборку — для надёжных результатов без ограничений сторонних провайдеров.",
+ "byokTooLarge": "Сторонние API-ключи поддерживают файлы только до {{size}} МБ.",
+ "byokTooLargeDetail": "Для файлов более {{size}} МБ OpenWhispr Cloud выполняет разделение, параллельную обработку и сборку — для надёжных результатов без ограничений сторонних провайдеров.",
"byokTooLargeNeedsAccount": "Создайте аккаунт и перейдите на платный план, чтобы транскрибировать большие файлы через OpenWhispr Cloud.",
"byokTooLargeNeedsUpgrade": "Перейдите на платный план, чтобы транскрибировать большие файлы через OpenWhispr Cloud.",
"switchToCloudForLargeFiles": "Переключитесь на OpenWhispr Cloud, чтобы транскрибировать этот файл.",
diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.json
index 0da490f2b..0c6f49930 100644
--- a/src/locales/zh-CN/translation.json
+++ b/src/locales/zh-CN/translation.json
@@ -2482,6 +2482,7 @@
"groq_whisper_large_v3_turbo": "216 倍实时速度",
"groq_whisper_large_v3": "Large v3 模型,快速且精准",
"mistral_voxtral_mini_latest": "快速多语言转录",
+ "gemini_3_5_transcribe": "Google Gemini 语音转文字",
"corti_transcribe": "临床级医疗转录",
"xai_grok_stt": "xAI 高精度语音识别",
"tinfoil_voxtral": "可验证的私密转录"
@@ -2885,8 +2886,8 @@
"largeFileNote": "大文件将被分割并平行转录。",
"chunkProgress": "正在转录第 {{completed}} 部分,共 {{total}} 部分...",
"paidPlanRequired": "超过 25 MB 的文件需要付费计划才能使用云端转录。大文件通过 OpenWhispr Cloud 处理,确保可靠、高质量的结果。",
- "byokTooLarge": "第三方 API 密钥仅支持 25 MB 以内的文件。",
- "byokTooLargeDetail": "超过 25 MB 的文件由 OpenWhispr Cloud 负责分割、并行处理和重组——无需依赖第三方服务限制,即可获得可靠的结果。",
+ "byokTooLarge": "第三方 API 密钥仅支持 {{size}} MB 以内的文件。",
+ "byokTooLargeDetail": "超过 {{size}} MB 的文件由 OpenWhispr Cloud 负责分割、并行处理和重组——无需依赖第三方服务限制,即可获得可靠的结果。",
"byokTooLargeNeedsAccount": "创建账户并升级到付费计划,即可使用 OpenWhispr Cloud 转录更大的文件。",
"byokTooLargeNeedsUpgrade": "升级到付费计划,即可使用 OpenWhispr Cloud 转录更大的文件。",
"switchToCloudForLargeFiles": "切换到 OpenWhispr Cloud 来转录此文件。",
diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.json
index 8823089b2..9fd63510a 100644
--- a/src/locales/zh-TW/translation.json
+++ b/src/locales/zh-TW/translation.json
@@ -2482,6 +2482,7 @@
"groq_whisper_large_v3_turbo": "216 倍即時速度",
"groq_whisper_large_v3": "Large v3 模型,快速且精準",
"mistral_voxtral_mini_latest": "快速多語言轉錄",
+ "gemini_3_5_transcribe": "Google Gemini 語音轉文字",
"corti_transcribe": "臨床級醫療轉錄",
"xai_grok_stt": "xAI 高精度語音辨識",
"tinfoil_voxtral": "可驗證的私密轉錄"
@@ -2885,8 +2886,8 @@
"largeFileNote": "大型檔案將被分割並平行轉錄。",
"chunkProgress": "正在轉錄第 {{completed}} 部分,共 {{total}} 部分...",
"paidPlanRequired": "超過 25 MB 的檔案需要付費方案才能使用雲端轉錄。大型檔案透過 OpenWhispr Cloud 處理,確保可靠、高品質的結果。",
- "byokTooLarge": "第三方 API 金鑰僅支援 25 MB 以內的檔案。",
- "byokTooLargeDetail": "超過 25 MB 的檔案由 OpenWhispr Cloud 負責分割、平行處理和重組——無需依賴第三方服務限制,即可獲得可靠的結果。",
+ "byokTooLarge": "第三方 API 金鑰僅支援 {{size}} MB 以內的檔案。",
+ "byokTooLargeDetail": "超過 {{size}} MB 的檔案由 OpenWhispr Cloud 負責分割、平行處理和重組——無需依賴第三方服務限制,即可獲得可靠的結果。",
"byokTooLargeNeedsAccount": "建立帳戶並升級至付費方案,即可使用 OpenWhispr Cloud 轉錄更大的檔案。",
"byokTooLargeNeedsUpgrade": "升級至付費方案,即可使用 OpenWhispr Cloud 轉錄更大的檔案。",
"switchToCloudForLargeFiles": "切換到 OpenWhispr Cloud 來轉錄此檔案。",
diff --git a/src/models/modelRegistryData.json b/src/models/modelRegistryData.json
index ade854344..0c3a08b52 100644
--- a/src/models/modelRegistryData.json
+++ b/src/models/modelRegistryData.json
@@ -245,6 +245,19 @@
}
]
},
+ {
+ "id": "gemini",
+ "name": "Gemini",
+ "baseUrl": "https://generativelanguage.googleapis.com/v1beta",
+ "models": [
+ {
+ "id": "gemini-3.5-transcribe",
+ "name": "Gemini 3.5 Transcribe",
+ "description": "Google's Gemini speech-to-text",
+ "descriptionKey": "models.descriptions.transcription.gemini_3_5_transcribe"
+ }
+ ]
+ },
{
"id": "corti",
"name": "Corti",
diff --git a/src/services/fileTranscription.ts b/src/services/fileTranscription.ts
index 7b4e087e2..8bbe94143 100644
--- a/src/services/fileTranscription.ts
+++ b/src/services/fileTranscription.ts
@@ -50,6 +50,7 @@ export interface TranscriptionApiKeys {
groqApiKey: string;
xaiApiKey: string;
mistralApiKey: string;
+ geminiApiKey: string;
tinfoilApiKey: string;
customTranscriptionApiKey?: string;
}
@@ -64,6 +65,8 @@ export function getTranscriptionApiKey(provider: string, keys: TranscriptionApiK
return keys.xaiApiKey;
case "mistral":
return keys.mistralApiKey;
+ case "gemini":
+ return keys.geminiApiKey;
case "tinfoil":
return keys.tinfoilApiKey;
case "custom":
diff --git a/src/types/electron.ts b/src/types/electron.ts
index 7b2f7875b..8ffbfd71f 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -1785,6 +1785,12 @@ declare global {
// Gemini API key management
getGeminiKey: () => Promise<string | null>;
saveGeminiKey: (key: string) => Promise<void>;
+ proxyGeminiTranscription?: (data: {
+ audioBuffer: ArrayBuffer;
+ model?: string;
+ language?: string;
+ keyterms?: string[];
+ }) => Promise<ProxyTranscriptionResult>;
// Groq API key management
getGroqKey: () => Promise<string | null>;
diff --git a/test/helpers/geminiTranscription.test.js b/test/helpers/geminiTranscription.test.js
new file mode 100644
index 000000000..fe492db80
--- /dev/null
+++ b/test/helpers/geminiTranscription.test.js
@@ -0,0 +1,163 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { transcribeWithGemini } = require("../../src/helpers/geminiTranscription");
+
+const AUDIO = Buffer.from("fake-audio-bytes");
+
+function makeFetch(response) {
+ const calls = [];
+ const fetchImpl = async (url, init) => {
+ calls.push({ url, init });
+ return {
+ ok: true,
+ status: 200,
+ json: async () => response,
+ text: async () => JSON.stringify(response),
+ };
+ };
+ return { fetchImpl, calls };
+}
+
+function requestBody(calls) {
+ return JSON.parse(calls[0].init.body);
+}
+
+test("posts JSON with the key header to the Interactions endpoint", async () => {
+ const { fetchImpl, calls } = makeFetch({ status: "completed", output_text: "hello" });
+
+ const result = await transcribeWithGemini(
+ { audioBuffer: AUDIO, model: "gemini-3.5-transcribe", contentType: "audio/mp3", apiKey: "k1" },
+ fetchImpl
+ );
+
+ assert.equal(calls[0].url, "https://generativelanguage.googleapis.com/v1beta/interactions");
+ assert.equal(calls[0].init.method, "POST");
+ assert.deepEqual(calls[0].init.headers, {
+ "Content-Type": "application/json",
+ "x-goog-api-key": "k1",
+ });
+ const body = requestBody(calls);
+ assert.equal(body.model, "gemini-3.5-transcribe");
+ assert.deepEqual(body.input, [
+ { type: "audio", data: AUDIO.toString("base64"), mime_type: "audio/mp3" },
+ ]);
+ assert.equal(result.text, "hello");
+ assert.equal(result.model, "gemini-3.5-transcribe");
+});
+
+test("defaults the model and omits generation_config when empty", async () => {
+ const { fetchImpl, calls } = makeFetch({ status: "completed", output_text: "ok" });
+
+ await transcribeWithGemini(
+ { audioBuffer: AUDIO, contentType: "audio/webm", language: "auto", apiKey: "k" },
+ fetchImpl
+ );
+
+ const body = requestBody(calls);
+ assert.equal(body.model, "gemini-3.5-transcribe");
+ assert.equal(body.input[0].mime_type, "audio/webm");
+ assert.equal("generation_config" in body, false, "auto language must not send a config");
+});
+
+test("language and keyterms land in transcription_config", async () => {
+ const { fetchImpl, calls } = makeFetch({ status: "completed", output_text: "ok" });
+
+ await transcribeWithGemini(
+ { audioBuffer: AUDIO, language: "de", keyterms: ["OpenWhispr", "Gizmo"], apiKey: "k" },
+ fetchImpl
+ );
+
+ assert.deepEqual(requestBody(calls).generation_config, {
+ transcription_config: {
+ language_codes: ["de"],
+ custom_vocabulary: ["OpenWhispr", "Gizmo"],
+ },
+ });
+});
+
+test("falls back to joining step text when output_text is absent", async () => {
+ const { fetchImpl } = makeFetch({
+ status: "completed",
+ steps: [
+ {
+ type: "model_output",
+ content: [
+ { type: "text", text: "Hello" },
+ { type: "word_info", text: "ignored" },
+ { type: "text", text: "world" },
+ ],
+ },
+ ],
+ });
+
+ const { text } = await transcribeWithGemini({ audioBuffer: AUDIO, apiKey: "k" }, fetchImpl);
+ assert.equal(text, "Hello world");
+});
+
+test("a missing key fails before any request is made", async () => {
+ const { fetchImpl, calls } = makeFetch({});
+ await assert.rejects(
+ transcribeWithGemini({ audioBuffer: AUDIO, apiKey: " " }, fetchImpl),
+ (error) => error.code === "API_KEY_MISSING"
+ );
+ assert.equal(calls.length, 0);
+});
+
+test("HTTP statuses map to coded errors without leaking the key", async () => {
+ const statusError = async (status, body = "denied") => {
+ const fetchImpl = async () => ({
+ ok: false,
+ status,
+ text: async () => body,
+ });
+ return transcribeWithGemini({ audioBuffer: AUDIO, apiKey: "sk-secret" }, fetchImpl).then(
+ () => assert.fail(`status ${status} must reject`),
+ (error) => error
+ );
+ };
+
+ for (const status of [401, 403]) {
+ const error = await statusError(status);
+ assert.equal(error.code, "INVALID_KEY");
+ assert.equal(error.message.includes("sk-secret"), false);
+ }
+ assert.equal((await statusError(429)).code, "PROVIDER_RATE_LIMITED");
+ assert.equal((await statusError(500)).code, "SERVER_ERROR");
+
+ // Google's real answer to a bad key, captured from the live API.
+ const badKey = await statusError(400, '{"error":{"reason":"API_KEY_INVALID"}}');
+ assert.equal(badKey.code, "INVALID_KEY");
+ assert.equal(badKey.message.includes("sk-secret"), false);
+
+ assert.equal((await statusError(400, "unsupported mime")).code, undefined);
+});
+
+test("a failed interaction status rejects even on HTTP 200", async () => {
+ const { fetchImpl } = makeFetch({
+ status: "failed",
+ error: { code: "api_error", message: "decode error" },
+ });
+ await assert.rejects(
+ transcribeWithGemini({ audioBuffer: AUDIO, apiKey: "k" }, fetchImpl),
+ /Gemini transcription failed: decode error/
+ );
+});
+
+test("any non-completed status rejects instead of returning empty text", async () => {
+ const { fetchImpl } = makeFetch({ status: "budget_exceeded", steps: [] });
+ await assert.rejects(
+ transcribeWithGemini({ audioBuffer: AUDIO, apiKey: "k" }, fetchImpl),
+ /did not complete \(status: budget_exceeded\)/
+ );
+});
+
+test("canonical mime types map onto Gemini's documented ones", async () => {
+ const { fetchImpl, calls } = makeFetch({ status: "completed", output_text: "ok" });
+
+ await transcribeWithGemini(
+ { audioBuffer: AUDIO, contentType: "audio/mpeg", apiKey: "k" },
+ fetchImpl
+ );
+
+ assert.equal(requestBody(calls).input[0].mime_type, "audio/mp3");
+});
diff --git a/test/helpers/transcriptionRoute.test.js b/test/helpers/transcriptionRoute.test.js
index 233a40320..7e780ecbf 100644
--- a/test/helpers/transcriptionRoute.test.js
+++ b/test/helpers/transcriptionRoute.test.js
@@ -42,7 +42,7 @@ test("self-hosted routes to the configured server and wins over stale flags", as
});
test("self-hosted mode without a URL fails closed unless the provider is custom", async () => {
- for (const provider of ["openai", "groq", "mistral", "xai", "corti", "tinfoil"]) {
+ for (const provider of ["openai", "groq", "mistral", "xai", "corti", "gemini", "tinfoil"]) {
const route = await resolve({
transcriptionMode: "self-hosted",
remoteTranscriptionUrl: "",
@@ -115,6 +115,21 @@ test("proxied providers carry their quirks as route data", async () => {
assert.equal(corti.language, "en", "Corti needs a concrete language even on auto");
assert.equal(corti.cortiEnvironment, "us");
assert.equal(corti.cortiTenant, "acme");
+
+ const gemini = await resolve({ cloudTranscriptionProvider: "gemini" });
+ assert.equal(gemini.transport, "proxied");
+ assert.equal(gemini.model, "gemini-3.5-transcribe");
+ assert.equal(
+ gemini.sizeCapBytes,
+ 14 * 1024 * 1024,
+ "inline base64 audio must fit Gemini's 20 MB request cap"
+ );
+});
+
+test("byokFileSizeLimit matches the per-provider route caps", async () => {
+ const { byokFileSizeLimit } = await load();
+ assert.equal(byokFileSizeLimit("gemini"), 14 * 1024 * 1024);
+ assert.equal(byokFileSizeLimit("openai"), 25 * 1024 * 1024);
});
test("custom requires a configured secure endpoint (empty, sentinel, garbage all fail)", async () => {
diff --git a/test/helpers/transcriptionRouteModelAgreement.test.js b/test/helpers/transcriptionRouteModelAgreement.test.js
index 69aa37132..96225b2be 100644
--- a/test/helpers/transcriptionRouteModelAgreement.test.js
+++ b/test/helpers/transcriptionRouteModelAgreement.test.js
@@ -11,7 +11,7 @@ const load = () => import("../../src/helpers/transcriptionRoute.ts");
//
// Tinfoil is excluded: its batch model comes from the registry directly
// (getBatchTranscriptionModel) and the route sends none.
-const PREFIX_VALIDATED_PROVIDERS = ["openai", "groq", "xai", "mistral", "corti"];
+const PREFIX_VALIDATED_PROVIDERS = ["openai", "groq", "xai", "mistral", "corti", "gemini"];
test("every shipping registry model survives resolveByokModel", async () => {
const { resolveByokModel } = await load();
@@ -72,4 +72,5 @@ test("a model belonging to another provider degrades to the provider default", a
assert.equal(resolveByokModel("openai", "voxtral-mini-latest"), "gpt-4o-mini-transcribe");
assert.equal(resolveByokModel("mistral", "whisper-1"), "voxtral-mini-latest");
assert.equal(resolveByokModel("corti", "whisper-1"), "corti-transcribe");
+ assert.equal(resolveByokModel("gemini", "whisper-1"), "gemini-3.5-transcribe");
});