Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
feat: migrate streaming transcription from AssemblyAI to Deepgram (#249)
Gabriel Stein232d210ddac8
diff --git a/electron-builder.json b/electron-builder.jsonindex 054270c26..1afa33302 100644--- a/electron-builder.json+++ b/electron-builder.json@@ -87,7 +87,6 @@],"extraResources": ["src/assets/**/*",- "resources/macos-globe-listener.swift","resources/bin/macos-globe-listener","resources/bin/macos-fast-paste",{@@ -105,6 +104,7 @@"entitlements": "resources/mac/entitlements.mac.plist","entitlementsInherit": "resources/mac/entitlements.mac.plist","identity": "HeroTools Inc. (9R85XFMH59)",+ "notarize": true,"extendInfo": {"LSUIElement": false,"NSMicrophoneUsageDescription": "OpenWhispr needs microphone access for speech-to-text transcription.",diff --git a/preload.js b/preload.jsindex aa6bf8d2c..87f557917 100644--- a/preload.js+++ b/preload.js@@ -268,6 +268,18 @@ contextBridge.exposeInMainWorld("electronAPI", {onAssemblyAiError: registerListener("assemblyai-error", (callback) => (_event, error) => callback(error)),onAssemblyAiSessionEnd: registerListener("assemblyai-session-end", (callback) => (_event, data) => callback(data)),+ // Deepgram Streaming+ deepgramStreamingWarmup: (options) => ipcRenderer.invoke("deepgram-streaming-warmup", options),+ deepgramStreamingStart: (options) => ipcRenderer.invoke("deepgram-streaming-start", options),+ deepgramStreamingSend: (audioBuffer) => ipcRenderer.send("deepgram-streaming-send", audioBuffer),+ deepgramStreamingFinalize: () => ipcRenderer.send("deepgram-streaming-finalize"),+ deepgramStreamingStop: () => ipcRenderer.invoke("deepgram-streaming-stop"),+ deepgramStreamingStatus: () => ipcRenderer.invoke("deepgram-streaming-status"),+ onDeepgramPartialTranscript: registerListener("deepgram-partial-transcript", (callback) => (_event, text) => callback(text)),+ onDeepgramFinalTranscript: registerListener("deepgram-final-transcript", (callback) => (_event, text) => callback(text)),+ onDeepgramError: registerListener("deepgram-error", (callback) => (_event, error) => callback(error)),+ onDeepgramSessionEnd: registerListener("deepgram-session-end", (callback) => (_event, data) => callback(data)),+// Usage limit events (for showing UpgradePrompt in ControlPanel)notifyLimitReached: (data) => ipcRenderer.send("limit-reached", data),onLimitReached: registerListener("limit-reached", (callback) => (_event, data) => callback(data)),diff --git a/scripts/download-sherpa-onnx.js b/scripts/download-sherpa-onnx.jsindex 063d448f9..a87155e32 100644--- a/scripts/download-sherpa-onnx.js+++ b/scripts/download-sherpa-onnx.js@@ -126,16 +126,40 @@ async function downloadBinary(platformArch, config) {if (config.libPattern) {const libraries = findLibrariesInDir(extractDir, config.libPattern);+ // Separate versioned and unversioned libraries to create symlinks where possible+ // e.g. libonnxruntime.dylib -> libonnxruntime.1.23.2.dylib (saves ~71MB)+ const versionedLibs = new Map(); // base name -> versioned file name+for (const libPath of libraries) {const libName = path.basename(libPath);const destPath = path.join(BIN_DIR, libName);+ // Detect versioned dylib pattern: libFoo.X.Y.Z.dylib+ const versionMatch = libName.match(/^(lib.+?)\.(\d+\.\d+\.\d+)\.(dylib|so|dll)$/);+ if (versionMatch) {+ const baseName = `${versionMatch[1]}.${versionMatch[3]}`; // e.g. libonnxruntime.dylib+ versionedLibs.set(baseName, libName);+ }+if (!fs.existsSync(destPath)) {fs.copyFileSync(libPath, destPath);setExecutable(destPath);console.log(` ${platformArch}: Copied library ${libName}`);}}++ // Replace unversioned copies with symlinks to versioned ones (macOS/Linux only)+ if (process.platform !== "win32") {+ for (const [baseName, versionedName] of versionedLibs) {+ const basePath = path.join(BIN_DIR, baseName);+ const versionedPath = path.join(BIN_DIR, versionedName);+ if (fs.existsSync(basePath) && fs.existsSync(versionedPath) && !fs.lstatSync(basePath).isSymbolicLink()) {+ fs.unlinkSync(basePath);+ fs.symlinkSync(versionedName, basePath);+ console.log(` ${platformArch}: Symlinked ${baseName} -> ${versionedName}`);+ }+ }+ }}} else {console.error(` ${platformArch}: Binary '${binaryName}' not found in archive`);diff --git a/src/.prettierignore b/src/.prettierignorenew file mode 100644index 000000000..849ddff3b--- /dev/null+++ b/src/.prettierignore@@ -0,0 +1 @@+dist/diff --git a/src/App.jsx b/src/App.jsxindex 05e1ea76c..0cdf92b5b 100644--- a/src/App.jsx+++ b/src/App.jsx@@ -158,12 +158,14 @@ export default function App() {onToggle: handleDictationToggle,});- // Trigger streaming warmup when user signs in (covers first-time account creation)+ // Trigger streaming warmup when user signs in (covers first-time account creation).+ // Pass isSignedIn directly to bypass the localStorage race condition where+ // useAuth's useEffect may not have written localStorage yet.useEffect(() => {if (isSignedIn) {- warmupStreaming();+ warmupStreaming({ isSignedIn: true });}- }, [isSignedIn]);+ }, [isSignedIn, warmupStreaming]);// Listen for auto-hide setting changes relayed from the main processuseEffect(() => {diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex acb60dc9c..d06ec05cd 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -59,6 +59,8 @@ class AudioManager {this.persistentAudioContext = null;this.workletModuleLoaded = false;this.workletBlobUrl = null;+ this.streamingStartInProgress = false;+ this.stopRequestedDuringStreamingStart = false;}getWorkletBlobUrl() {@@ -1713,14 +1715,15 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);isRecording: this.isRecording,isProcessing: this.isProcessing,isStreaming: this.isStreaming,+ isStreamingStartInProgress: this.streamingStartInProgress,};}- shouldUseStreaming() {+ shouldUseStreaming(isSignedInOverride) {const cloudTranscriptionMode = localStorage.getItem("cloudTranscriptionMode") || "openwhispr";- const isSignedIn = localStorage.getItem("isSignedIn") === "true";+ const isSignedIn = isSignedInOverride ?? localStorage.getItem("isSignedIn") === "true";const useLocalWhisper = localStorage.getItem("useLocalWhisper") === "true";- const streamingDisabled = localStorage.getItem("assemblyAiStreaming") === "false";+ const streamingDisabled = localStorage.getItem("deepgramStreaming") === "false";return (!useLocalWhisper &&@@ -1730,8 +1733,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor););}- async warmupStreamingConnection() {- if (!this.shouldUseStreaming()) {+ async warmupStreamingConnection({ isSignedIn: isSignedInOverride } = {}) {+ if (!this.shouldUseStreaming(isSignedInOverride)) {logger.debug("Streaming warmup skipped - not in streaming mode", {}, "streaming");return false;}@@ -1740,7 +1743,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);const [, wsResult] = await Promise.all([this.cacheMicrophoneDeviceId(),withSessionRefresh(async () => {- const res = await window.electronAPI.assemblyAiStreamingWarmup({+ const res = await window.electronAPI.deepgramStreamingWarmup({sampleRate: 16000,language: getBaseLanguageCode(localStorage.getItem("preferredLanguage")),keyterms: this.getKeyterms(),@@ -1792,7 +1795,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);}logger.info(- "AssemblyAI streaming connection warmed up",+ "Deepgram streaming connection warmed up",{ alreadyWarm: wsResult.alreadyWarm, micCached: !!this.cachedMicDeviceId },"streaming");@@ -1801,11 +1804,11 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);logger.debug("Streaming warmup skipped - API not configured", {}, "streaming");return false;} else {- logger.warn("AssemblyAI warmup failed", { error: wsResult.error }, "streaming");+ logger.warn("Deepgram warmup failed", { error: wsResult.error }, "streaming");return false;}} catch (error) {- logger.error("AssemblyAI warmup error", { error: error.message }, "streaming");+ logger.error("Deepgram warmup error", { error: error.message }, "streaming");return false;}}@@ -1824,37 +1827,25 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);async startStreamingRecording() {try {- if (this.isRecording || this.isStreaming || this.isProcessing) {+ if (+ this.isRecording ||+ this.isStreaming ||+ this.isProcessing ||+ this.streamingStartInProgress+ ) {return false;}+ this.streamingStartInProgress = true;+ this.stopRequestedDuringStreamingStart = false;+const t0 = performance.now();const constraints = await this.getAudioConstraints();const tConstraints = performance.now();- // Run getUserMedia and WebSocket connect in parallel.- // With warmup, WS resolves in ~5ms; getUserMedia (~500ms) dominates.- const [stream, result] = await Promise.all([- navigator.mediaDevices.getUserMedia(constraints),- withSessionRefresh(async () => {- const res = await window.electronAPI.assemblyAiStreamingStart({- sampleRate: 16000,- language: getBaseLanguageCode(localStorage.getItem("preferredLanguage")),- keyterms: this.getKeyterms(),- });-- if (!res.success) {- if (res.code === "NO_API") {- return { needsFallback: true };- }- const err = new Error(res.error || "Failed to start streaming session");- err.code = res.code;- throw err;- }- return res;- }),- ]);- const tParallel = performance.now();+ // 1. Get mic stream (can take 10-15s on cold macOS mic driver)+ const stream = await navigator.mediaDevices.getUserMedia(constraints);+ const tMedia = performance.now();const audioTrack = stream.getAudioTracks()[0];if (audioTrack) {@@ -1871,16 +1862,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor););}- if (result.needsFallback) {- stream.getTracks().forEach((track) => track.stop());- logger.debug(- "Streaming API not configured, falling back to regular recording",- {},- "streaming"- );- return this.startRecording();- }-+ // 2. Set up audio pipeline so frames flow the instant WebSocket is ready.+ // Frames sent before WebSocket connects are silently dropped by sendAudio().const audioContext = await this.getOrCreateAudioContext();this.streamingAudioContext = audioContext;this.streamingSource = audioContext.createMediaStreamSource(stream);@@ -1895,50 +1878,33 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.streamingProcessor.port.onmessage = (event) => {if (!this.isStreaming) return;- window.electronAPI.assemblyAiStreamingSend(event.data);+ window.electronAPI.deepgramStreamingSend(event.data);};- this.streamingSource.connect(this.streamingProcessor);-- const tReady = performance.now();- logger.info(- "Streaming start timing",- {- constraintsMs: Math.round(tConstraints - t0),- getUserMediaAndWsMs: Math.round(tParallel - tConstraints),- pipelineMs: Math.round(tReady - tParallel),- totalMs: Math.round(tReady - t0),- usedWarmConnection: result.usedWarmConnection,- micDriverWarmedUp: !!this.micDriverWarmedUp,- },- "streaming"- );-- // Show recording indicator only AFTER mic is live and audio pipeline is connected.- // This ensures no words are lost — the user sees "recording" exactly when audio flows.this.isStreaming = true;- this.isRecording = true;- this.recordingStartTime = Date.now();- this.onStateChange?.({ isRecording: true, isProcessing: false, isStreaming: true });+ this.streamingSource.connect(this.streamingProcessor);+ const tPipeline = performance.now();+ // 3. Register IPC event listeners BEFORE connecting, so no transcript+ // events are lost during the connect handshake.this.streamingFinalText = "";this.streamingPartialText = "";this.streamingTextResolve = null;this.streamingTextDebounce = null;- const partialCleanup = window.electronAPI.onAssemblyAiPartialTranscript((text) => {+ const partialCleanup = window.electronAPI.onDeepgramPartialTranscript((text) => {this.streamingPartialText = text;this.onPartialTranscript?.(text);});- const finalCleanup = window.electronAPI.onAssemblyAiFinalTranscript((text) => {+ const finalCleanup = window.electronAPI.onDeepgramFinalTranscript((text) => {this.streamingFinalText = text;this.streamingPartialText = "";this.onPartialTranscript?.(text);});- const errorCleanup = window.electronAPI.onAssemblyAiError((error) => {- logger.error("AssemblyAI streaming error", { error }, "streaming");+ const errorCleanup = window.electronAPI.onDeepgramError((error) => {+ logger.error("Deepgram streaming error", { error }, "streaming");this.onError?.({title: "Streaming Error",description: error,@@ -1955,17 +1921,77 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);}});- const sessionEndCleanup = window.electronAPI.onAssemblyAiSessionEnd((data) => {- logger.debug("AssemblyAI session ended", data, "streaming");+ const sessionEndCleanup = window.electronAPI.onDeepgramSessionEnd((data) => {+ logger.debug("Deepgram session ended", data, "streaming");if (data.text) {this.streamingFinalText = data.text;}});this.streamingCleanupFns = [partialCleanup, finalCleanup, errorCleanup, sessionEndCleanup];+ this.isRecording = true;+ this.recordingStartTime = Date.now();+ this.onStateChange?.({ isRecording: true, isProcessing: false, isStreaming: true });++ // 4. Connect WebSocket — audio is already flowing from the pipeline above,+ // so Deepgram receives data immediately (no idle timeout).+ const result = await withSessionRefresh(async () => {+ const res = await window.electronAPI.deepgramStreamingStart({+ sampleRate: 16000,+ language: getBaseLanguageCode(localStorage.getItem("preferredLanguage")),+ });++ if (!res.success) {+ if (res.code === "NO_API") {+ return { needsFallback: true };+ }+ const err = new Error(res.error || "Failed to start streaming session");+ err.code = res.code;+ throw err;+ }+ return res;+ });+ const tWs = performance.now();++ if (result.needsFallback) {+ this.isRecording = false;+ this.recordingStartTime = null;+ this.stopRequestedDuringStreamingStart = false;+ await this.cleanupStreaming();+ this.onStateChange?.({ isRecording: false, isProcessing: false, isStreaming: false });+ this.streamingStartInProgress = false;+ logger.debug(+ "Streaming API not configured, falling back to regular recording",+ {},+ "streaming"+ );+ return this.startRecording();+ }+ logger.info(+ "Streaming start timing",+ {+ constraintsMs: Math.round(tConstraints - t0),+ getUserMediaMs: Math.round(tMedia - tConstraints),+ pipelineMs: Math.round(tPipeline - tMedia),+ wsConnectMs: Math.round(tWs - tPipeline),+ totalMs: Math.round(tWs - t0),+ usedWarmConnection: result.usedWarmConnection,+ micDriverWarmedUp: !!this.micDriverWarmedUp,+ },+ "streaming"+ );++ this.streamingStartInProgress = false;+ if (this.stopRequestedDuringStreamingStart) {+ this.stopRequestedDuringStreamingStart = false;+ logger.debug("Applying deferred streaming stop requested during startup", {}, "streaming");+ return this.stopStreamingRecording();+ }return true;} catch (error) {+ this.streamingStartInProgress = false;+ this.stopRequestedDuringStreamingStart = false;logger.error("Failed to start streaming recording", { error: error.message }, "streaming");let errorTitle = "Streaming Error";@@ -1987,11 +2013,20 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);});await this.cleanupStreaming();+ this.isRecording = false;+ this.recordingStartTime = null;+ this.onStateChange?.({ isRecording: false, isProcessing: false, isStreaming: false });return false;}}async stopStreamingRecording() {+ if (this.streamingStartInProgress) {+ this.stopRequestedDuringStreamingStart = true;+ logger.debug("Streaming stop requested while start is in progress", {}, "streaming");+ return true;+ }+if (!this.isStreaming) return false;const durationSeconds = this.recordingStartTime@@ -2037,13 +2072,14 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);await new Promise((resolve) => setTimeout(resolve, 120));this.isStreaming = false;- // 4. ForceEndpoint finalizes any in-progress turn, then Terminate closes the session.- // The server MUST process ALL remaining audio and send ALL Turn messages before- // responding with Termination — so awaiting this guarantees we get every word.- window.electronAPI.assemblyAiStreamingForceEndpoint?.();+ // 4. Finalize tells Deepgram to process any buffered audio and send final Results.+ // Wait briefly so the server sends back the from_finalize transcript before+ // CloseStream triggers connection close.+ window.electronAPI.deepgramStreamingFinalize?.();+ await new Promise((resolve) => setTimeout(resolve, 300));const tForceEndpoint = performance.now();- const stopResult = await window.electronAPI.assemblyAiStreamingStop().catch((e) => {+ const stopResult = await window.electronAPI.deepgramStreamingStop().catch((e) => {logger.debug("Streaming disconnect error", { error: e.message }, "streaming");return { success: false };});@@ -2147,7 +2183,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.onTranscriptionComplete?.({success: true,text: finalText,- source: "assemblyai-streaming",+ source: "deepgram-streaming",});logger.info(@@ -2240,7 +2276,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.workletBlobUrl = null;}try {- window.electronAPI?.assemblyAiStreamingStop?.();+ window.electronAPI?.deepgramStreamingStop?.();} catch (e) {// Ignore errors during cleanup (page may be unloading)}diff --git a/src/helpers/deepgramStreaming.js b/src/helpers/deepgramStreaming.jsnew file mode 100644index 000000000..c6494acd8--- /dev/null+++ b/src/helpers/deepgramStreaming.js@@ -0,0 +1,618 @@+const WebSocket = require("ws");+const debugLogger = require("./debugLogger");++const SAMPLE_RATE = 16000;+const WEBSOCKET_TIMEOUT_MS = 30000;+const TERMINATION_TIMEOUT_MS = 5000;+const TOKEN_REFRESH_BUFFER_MS = 30000;+const TOKEN_EXPIRY_MS = 300000;+const REWARM_DELAY_MS = 2000;+const MAX_REWARM_ATTEMPTS = 10;+const KEEPALIVE_INTERVAL_MS = 3000;+const COLD_START_BUFFER_MAX = 3 * SAMPLE_RATE * 2; // 3 seconds of 16-bit PCM++// 100ms of silence at 16kHz 16-bit mono — sent to warm connections to prevent+// Deepgram's net0001 idle timeout (which only resets on audio data, not KeepAlive).+const SILENCE_FRAME = Buffer.alloc((SAMPLE_RATE / 10) * 2);++class DeepgramStreaming {+ constructor() {+ this.ws = null;+ this.sessionId = null;+ this.isConnected = false;+ this.onPartialTranscript = null;+ this.onFinalTranscript = null;+ this.onError = null;+ this.onSessionEnd = null;+ this.pendingResolve = null;+ this.pendingReject = null;+ this.connectionTimeout = null;+ this.accumulatedText = "";+ this.finalSegments = [];+ this.closeResolve = null;+ this.cachedToken = null;+ this.tokenFetchedAt = null;+ this.warmConnection = null;+ this.warmConnectionReady = false;+ this.warmConnectionOptions = null;+ this.warmSessionId = null;+ this.rewarmAttempts = 0;+ this.rewarmTimer = null;+ this.keepAliveInterval = null;+ this.isDisconnecting = false;+ this.coldStartBuffer = [];+ this.coldStartBufferSize = 0;+ }++ buildWebSocketUrl(options) {+ const sampleRate = options.sampleRate || SAMPLE_RATE;+ const params = new URLSearchParams({+ encoding: "linear16",+ sample_rate: String(sampleRate),+ channels: "1",+ model: "nova-3",+ smart_format: "true",+ punctuate: "true",+ interim_results: "true",+ utterance_end_ms: "1000",+ vad_events: "true",+ });+ if (options.language && options.language !== "auto") {+ params.set("language", options.language);+ }+ return `wss://api.deepgram.com/v1/listen?${params.toString()}`;+ }++ cacheToken(token) {+ this.cachedToken = token;+ this.tokenFetchedAt = Date.now();+ debugLogger.debug("Deepgram token cached", { expiresIn: TOKEN_EXPIRY_MS });+ }++ isTokenValid() {+ if (!this.cachedToken || !this.tokenFetchedAt) return false;+ const age = Date.now() - this.tokenFetchedAt;+ return age < TOKEN_EXPIRY_MS - TOKEN_REFRESH_BUFFER_MS;+ }++ getCachedToken() {+ return this.isTokenValid() ? this.cachedToken : null;+ }++ startKeepAlive(socket) {+ this.stopKeepAlive();+ const target = socket || this.warmConnection;+ const isWarm = !socket;+ debugLogger.debug("Deepgram keep-alive started", { isWarm });+ this.keepAliveInterval = setInterval(() => {+ if (target && target.readyState === WebSocket.OPEN) {+ try {+ // Warm connections send silence audio — Deepgram's net0001 idle timeout+ // only resets on audio data, not KeepAlive messages, for pre-audio sessions.+ // Active connections use KeepAlive JSON (audio flow already resets the timer).+ target.send(isWarm ? SILENCE_FRAME : JSON.stringify({ type: "KeepAlive" }));+ } catch (err) {+ debugLogger.debug("Deepgram keep-alive failed", { error: err.message });+ if (target === this.warmConnection) {+ this.cleanupWarmConnection();+ }+ }+ } else {+ debugLogger.debug("Deepgram keep-alive target gone", {+ isWarm,+ readyState: target?.readyState,+ });+ this.stopKeepAlive();+ }+ }, KEEPALIVE_INTERVAL_MS);+ }++ stopKeepAlive() {+ if (this.keepAliveInterval) {+ debugLogger.debug("Deepgram keep-alive stopped");+ clearInterval(this.keepAliveInterval);+ this.keepAliveInterval = null;+ }+ }++ async warmup(options = {}) {+ const { token } = options;+ if (!token) {+ throw new Error("Streaming token is required for warmup");+ }++ if (this.warmConnection) {+ debugLogger.debug(+ this.warmConnectionReady+ ? "Deepgram connection already warm"+ : "Deepgram warmup already in progress, skipping"+ );+ return;+ }++ this.warmConnectionReady = false;+ this.warmSessionId = null;+ this.cachedToken = token;+ this.tokenFetchedAt = Date.now();+ this.warmConnectionOptions = options;+ this.rewarmAttempts = 0;++ const url = this.buildWebSocketUrl(options);+ debugLogger.debug("Deepgram warming up connection");++ return new Promise((resolve, reject) => {+ let settled = false;+ const resolveWarmup = (meta = {}) => {+ if (settled) return;+ settled = true;+ clearTimeout(warmupTimeout);+ this.warmConnectionReady = true;+ if (meta.requestId) {+ this.warmSessionId = meta.requestId;+ }+ // Send immediate silence frame so the connection doesn't timeout+ // before the first keep-alive interval fires.+ try {+ this.warmConnection?.send(SILENCE_FRAME);+ } catch (err) {+ // Ignore+ }+ this.startKeepAlive();+ debugLogger.debug("Deepgram connection warmed up", {+ sessionId: this.warmSessionId,+ via: meta.via || "unknown",+ });+ resolve();+ };++ const warmupTimeout = setTimeout(() => {+ this.cleanupWarmConnection();+ reject(new Error("Deepgram warmup connection timeout"));+ }, WEBSOCKET_TIMEOUT_MS);++ this.warmConnection = new WebSocket(url, {+ headers: { Authorization: `Bearer ${token}` },+ });++ this.warmConnection.on("open", () => {+ debugLogger.debug("Deepgram warm connection socket opened");+ // Consider the socket warm as soon as it's open. Deepgram may not emit+ // Metadata until audio starts, which would make startup warmup appear broken.+ resolveWarmup({ via: "open" });+ });++ this.warmConnection.on("message", (data) => {+ try {+ const message = JSON.parse(data.toString());+ if (message.type === "Metadata") {+ this.warmSessionId = message.request_id || null;+ resolveWarmup({ requestId: message.request_id, via: "metadata" });+ }+ } catch (err) {+ debugLogger.error("Deepgram warmup message parse error", { error: err.message });+ }+ });++ this.warmConnection.on("error", (error) => {+ clearTimeout(warmupTimeout);+ debugLogger.error("Deepgram warmup connection error", { error: error.message });+ this.cleanupWarmConnection();+ if (!settled) {+ settled = true;+ reject(error);+ }+ });++ this.warmConnection.on("close", (code, reason) => {+ clearTimeout(warmupTimeout);+ this.stopKeepAlive();+ const wasReady = this.warmConnectionReady;+ const savedOptions = this.warmConnectionOptions ? { ...this.warmConnectionOptions } : null;+ debugLogger.debug("Deepgram warm connection closed", {+ wasReady,+ code,+ reason: reason?.toString(),+ });+ this.cleanupWarmConnection();+ if (!settled) {+ settled = true;+ reject(new Error(`Deepgram warmup connection closed (code: ${code})`));+ return;+ }+ if (wasReady && savedOptions) {+ this.warmConnectionOptions = savedOptions;+ this.scheduleRewarm();+ }+ });+ });+ }++ scheduleRewarm() {+ if (this.rewarmAttempts >= MAX_REWARM_ATTEMPTS) {+ debugLogger.debug("Deepgram max re-warm attempts reached, will cold-start next recording");+ return;+ }+ if (this.isConnected) {+ return;+ }+ const token = this.getCachedToken();+ if (!token || !this.warmConnectionOptions) {+ debugLogger.debug("Deepgram cannot re-warm: no valid token or options");+ return;+ }++ this.rewarmAttempts++;+ const delay = Math.min(REWARM_DELAY_MS * Math.pow(2, this.rewarmAttempts - 1), 60000);+ debugLogger.debug("Deepgram scheduling re-warm", {+ attempt: this.rewarmAttempts,+ delayMs: delay,+ });+ clearTimeout(this.rewarmTimer);+ this.rewarmTimer = setTimeout(() => {+ this.rewarmTimer = null;+ if (this.hasWarmConnection() || this.isConnected) return;+ this.warmup({ ...this.warmConnectionOptions, token }).catch((err) => {+ debugLogger.debug("Deepgram auto re-warm failed", { error: err.message });+ });+ }, delay);+ }++ useWarmConnection() {+ if (!this.warmConnection || !this.warmConnectionReady) {+ return false;+ }++ if (this.warmConnection.readyState !== WebSocket.OPEN) {+ debugLogger.debug("Deepgram warm connection readyState not OPEN, discarding", {+ readyState: this.warmConnection.readyState,+ });+ this.cleanupWarmConnection();+ return false;+ }++ this.stopKeepAlive();++ this.ws = this.warmConnection;+ this.isConnected = true;+ this.sessionId = this.warmSessionId || null;+ this.warmConnection = null;+ this.warmConnectionReady = false;+ this.warmSessionId = null;++ this.ws.removeAllListeners("message");+ this.ws.on("message", (data) => {+ this.handleMessage(data);+ });++ this.ws.removeAllListeners("error");+ this.ws.on("error", (error) => {+ debugLogger.error("Deepgram WebSocket error", { error: error.message });+ this.cleanup();+ this.onError?.(error);+ });++ this.ws.removeAllListeners("close");+ this.ws.on("close", (code, reason) => {+ const wasActive = this.isConnected;+ debugLogger.debug("Deepgram WebSocket closed", {+ code,+ reason: reason?.toString(),+ wasActive,+ });+ if (this.closeResolve) {+ this.closeResolve({ text: this.accumulatedText });+ }+ this.cleanup();+ if (wasActive && !this.isDisconnecting) {+ this.onError?.(new Error(`Connection lost (code: ${code})`));+ }+ });++ this.startKeepAlive(this.ws);+ debugLogger.debug("Deepgram using pre-warmed connection");+ return true;+ }++ cleanupWarmConnection() {+ this.stopKeepAlive();+ if (this.warmConnection) {+ try {+ this.warmConnection.close();+ } catch (err) {+ // Ignore+ }+ this.warmConnection = null;+ }+ this.warmConnectionReady = false;+ this.warmConnectionOptions = null;+ this.warmSessionId = null;+ }++ hasWarmConnection() {+ const result =+ this.warmConnection !== null &&+ this.warmConnectionReady &&+ this.warmConnection.readyState === WebSocket.OPEN;+ if (!result && (this.warmConnection || this.warmConnectionReady)) {+ debugLogger.debug("Deepgram hasWarmConnection=false", {+ hasSocket: this.warmConnection !== null,+ ready: this.warmConnectionReady,+ readyState: this.warmConnection?.readyState,+ hasKeepAlive: this.keepAliveInterval !== null,+ });+ }+ return result;+ }++ async connect(options = {}) {+ const { token } = options;+ if (!token) {+ throw new Error("Streaming token is required");+ }++ if (this.isConnected) {+ debugLogger.debug("Deepgram streaming already connected");+ return;+ }++ this.accumulatedText = "";+ this.finalSegments = [];+ this.coldStartBuffer = [];+ this.coldStartBufferSize = 0;++ if (this.hasWarmConnection()) {+ if (this.useWarmConnection()) {+ debugLogger.debug("Deepgram using warm connection - instant start");+ return;+ }+ }++ const url = this.buildWebSocketUrl(options);+ debugLogger.debug("Deepgram streaming connecting (cold start)");++ return new Promise((resolve, reject) => {+ this.pendingResolve = resolve;+ this.pendingReject = reject;++ this.connectionTimeout = setTimeout(() => {+ this.cleanup();+ reject(new Error("Deepgram WebSocket connection timeout"));+ }, WEBSOCKET_TIMEOUT_MS);++ this.ws = new WebSocket(url, {+ headers: { Authorization: `Bearer ${token}` },+ });++ this.ws.on("open", () => {+ debugLogger.debug("Deepgram WebSocket connected");+ });++ this.ws.on("message", (data) => {+ this.handleMessage(data);+ });++ this.ws.on("error", (error) => {+ debugLogger.error("Deepgram WebSocket error", { error: error.message });+ this.cleanup();+ if (this.pendingReject) {+ this.pendingReject(error);+ this.pendingReject = null;+ this.pendingResolve = null;+ }+ this.onError?.(error);+ });++ this.ws.on("close", (code, reason) => {+ const wasActive = this.isConnected;+ debugLogger.debug("Deepgram WebSocket closed", {+ code,+ reason: reason?.toString(),+ wasActive,+ });+ if (this.closeResolve) {+ this.closeResolve({ text: this.accumulatedText });+ }+ this.cleanup();+ if (wasActive && !this.isDisconnecting) {+ this.onError?.(new Error(`Connection lost (code: ${code})`));+ }+ });+ });+ }++ handleMessage(data) {+ try {+ const message = JSON.parse(data.toString());++ // Resolve pending connect() promise on first valid message.+ // With nova-3, when audio is already flowing, Deepgram may skip+ // the Metadata message and jump straight to SpeechStarted/Results.+ if (this.pendingResolve) {+ this.isConnected = true;+ clearTimeout(this.connectionTimeout);+ this.startKeepAlive(this.ws);+ if (message.type === "Metadata") {+ this.sessionId = message.request_id;+ }+ debugLogger.debug("Deepgram session started", {+ sessionId: this.sessionId,+ firstMessageType: message.type,+ });+ this.pendingResolve();+ this.pendingResolve = null;+ this.pendingReject = null;+ }++ switch (message.type) {+ case "Metadata":+ this.sessionId = message.request_id;+ break;++ case "Results": {+ const transcript = message.channel?.alternatives?.[0]?.transcript;+ if (!transcript) break;++ if (message.is_final || message.from_finalize) {+ const trimmed = transcript.trim();+ if (trimmed) {+ this.finalSegments.push(trimmed);+ this.accumulatedText = this.finalSegments.join(" ");+ this.onFinalTranscript?.(this.accumulatedText);+ debugLogger.debug("Deepgram final transcript", {+ text: trimmed.slice(0, 100),+ totalAccumulated: this.accumulatedText.length,+ });+ }+ } else {+ this.onPartialTranscript?.(transcript);+ }+ break;+ }++ case "UtteranceEnd":+ debugLogger.debug("Deepgram utterance end");+ break;++ case "SpeechStarted":+ debugLogger.debug("Deepgram speech started");+ break;++ case "Error":+ debugLogger.error("Deepgram streaming error", { error: message.description });+ this.onError?.(new Error(message.description || "Deepgram error"));+ break;++ default:+ debugLogger.debug("Deepgram unknown message type", { type: message.type });+ }+ } catch (err) {+ debugLogger.error("Deepgram message parse error", { error: err.message });+ }+ }++ sendAudio(pcmBuffer) {+ if (!this.ws) return false;++ if (this.ws.readyState !== WebSocket.OPEN) {+ // Buffer audio during cold start so no frames are lost while WebSocket connects+ if (+ this.ws.readyState === WebSocket.CONNECTING &&+ this.coldStartBufferSize < COLD_START_BUFFER_MAX+ ) {+ const copy = Buffer.from(pcmBuffer);+ this.coldStartBuffer.push(copy);+ this.coldStartBufferSize += copy.length;+ }+ return false;+ }++ if (this.keepAliveInterval) {+ this.stopKeepAlive();+ }++ // Flush any audio buffered during cold start+ if (this.coldStartBuffer.length > 0) {+ debugLogger.debug("Deepgram flushing cold-start buffer", {+ chunks: this.coldStartBuffer.length,+ bytes: this.coldStartBufferSize,+ });+ for (const buf of this.coldStartBuffer) {+ this.ws.send(buf);+ }+ this.coldStartBuffer = [];+ this.coldStartBufferSize = 0;+ }++ this.ws.send(pcmBuffer);+ return true;+ }++ finalize() {+ if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {+ return false;+ }++ this.ws.send(JSON.stringify({ type: "Finalize" }));+ debugLogger.debug("Deepgram Finalize sent");+ return true;+ }++ async disconnect(closeStream = true) {+ if (!this.ws) return { text: this.accumulatedText };++ this.isDisconnecting = true;++ if (closeStream && this.ws.readyState === WebSocket.OPEN) {+ this.ws.send(JSON.stringify({ type: "CloseStream" }));++ let timeoutId;+ const result = await Promise.race([+ new Promise((resolve) => {+ this.closeResolve = resolve;+ }),+ new Promise((resolve) => {+ timeoutId = setTimeout(() => {+ debugLogger.debug("Deepgram close timeout, using accumulated text");+ resolve({ text: this.accumulatedText });+ }, TERMINATION_TIMEOUT_MS);+ }),+ ]);+ clearTimeout(timeoutId);++ this.closeResolve = null;+ this.cleanup();+ this.isDisconnecting = false;+ this.accumulatedText = "";+ this.finalSegments = [];+ return result;+ }++ const result = { text: this.accumulatedText };+ this.cleanup();+ this.isDisconnecting = false;+ this.accumulatedText = "";+ this.finalSegments = [];+ return result;+ }++ cleanup() {+ this.stopKeepAlive();+ clearTimeout(this.connectionTimeout);+ this.connectionTimeout = null;++ if (this.ws) {+ try {+ this.ws.close();+ } catch (err) {+ // Ignore close errors+ }+ this.ws = null;+ }++ this.isConnected = false;+ this.sessionId = null;+ this.closeResolve = null;+ }++ cleanupAll() {+ this.cleanup();+ this.cleanupWarmConnection();+ clearTimeout(this.rewarmTimer);+ this.rewarmTimer = null;+ this.cachedToken = null;+ this.tokenFetchedAt = null;+ this.warmConnectionOptions = null;+ this.finalSegments = [];+ }++ getStatus() {+ return {+ isConnected: this.isConnected,+ sessionId: this.sessionId,+ hasWarmConnection: this.hasWarmConnection(),+ hasValidToken: this.isTokenValid(),+ };+ }+}++module.exports = DeepgramStreaming;diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 61c44db86..18ad6f2b2 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -7,6 +7,7 @@ const AppUtils = require("../utils");const debugLogger = require("./debugLogger");const GnomeShortcutManager = require("./gnomeShortcut");const AssemblyAiStreaming = require("./assemblyAiStreaming");+const DeepgramStreaming = require("./deepgramStreaming");const MISTRAL_TRANSCRIPTION_URL = "https://api.mistral.ai/v1/audio/transcriptions";@@ -22,6 +23,7 @@ class IPCHandlers {this.windowsKeyManager = managers.windowsKeyManager;this.sessionId = crypto.randomUUID();this.assemblyAiStreaming = null;+ this.deepgramStreaming = null;this.setupHandlers();}@@ -175,6 +177,13 @@ class IPCHandlers {// Clipboard handlersipcMain.handle("paste-text", async (event, text, options) => {+ // If the floating dictation panel currently has focus, blur it first so the+ // paste keystroke lands in the user's target app instead of the overlay.+ const mainWindow = this.windowManager?.mainWindow;+ if (mainWindow && !mainWindow.isDestroyed() && mainWindow.isFocused()) {+ mainWindow.blur();+ await new Promise((resolve) => setTimeout(resolve, 80));+ }return this.clipboardManager.pasteText(text, { ...options, webContents: event.sender });});@@ -1764,6 +1773,200 @@ class IPCHandlers {}return this.assemblyAiStreaming.getStatus();});++ // --- Deepgram Streaming handlers ---++ const fetchDeepgramStreamingToken = async (event) => {+ const apiUrl = getApiUrl();+ if (!apiUrl) {+ throw new Error("OpenWhispr API URL not configured");+ }++ const cookieHeader = await getSessionCookies(event);+ if (!cookieHeader) {+ throw new Error("No session cookies available");+ }++ const tokenResponse = await fetch(`${apiUrl}/api/deepgram-streaming-token`, {+ method: "POST",+ headers: {+ Cookie: cookieHeader,+ },+ });++ if (!tokenResponse.ok) {+ if (tokenResponse.status === 401) {+ const err = new Error("Session expired");+ err.code = "AUTH_EXPIRED";+ throw err;+ }+ const errorData = await tokenResponse.json().catch(() => ({}));+ throw new Error(+ errorData.error || `Failed to get Deepgram streaming token: ${tokenResponse.status}`+ );+ }++ const { token } = await tokenResponse.json();+ if (!token) {+ throw new Error("No token received from API");+ }++ return token;+ };++ ipcMain.handle("deepgram-streaming-warmup", async (event, options = {}) => {+ try {+ const apiUrl = getApiUrl();+ if (!apiUrl) {+ return { success: false, error: "API not configured", code: "NO_API" };+ }++ if (!this.deepgramStreaming) {+ this.deepgramStreaming = new DeepgramStreaming();+ }++ if (this.deepgramStreaming.hasWarmConnection()) {+ debugLogger.debug("Deepgram connection already warm", {}, "streaming");+ return { success: true, alreadyWarm: true };+ }++ let token = this.deepgramStreaming.getCachedToken();+ if (!token) {+ debugLogger.debug("Fetching new Deepgram streaming token for warmup", {}, "streaming");+ token = await fetchDeepgramStreamingToken(event);+ }++ await this.deepgramStreaming.warmup({ ...options, token });+ debugLogger.debug("Deepgram connection warmed up", {}, "streaming");++ return { success: true };+ } catch (error) {+ debugLogger.error("Deepgram warmup error", { error: error.message });+ if (error.code === "AUTH_EXPIRED") {+ return { success: false, error: "Session expired", code: "AUTH_EXPIRED" };+ }+ return { success: false, error: error.message };+ }+ });++ let deepgramStreamingStartInProgress = false;++ ipcMain.handle("deepgram-streaming-start", async (event, options = {}) => {+ if (deepgramStreamingStartInProgress) {+ debugLogger.debug(+ "Deepgram streaming start already in progress, ignoring",+ {},+ "streaming"+ );+ return { success: false, error: "Operation in progress" };+ }++ deepgramStreamingStartInProgress = true;+ try {+ const apiUrl = getApiUrl();+ if (!apiUrl) {+ return { success: false, error: "API not configured", code: "NO_API" };+ }++ const win = BrowserWindow.fromWebContents(event.sender);++ if (!this.deepgramStreaming) {+ this.deepgramStreaming = new DeepgramStreaming();+ }++ if (this.deepgramStreaming.isConnected) {+ debugLogger.debug("Deepgram cleaning up stale connection before start", {}, "streaming");+ await this.deepgramStreaming.disconnect(false);+ }++ const hasWarm = this.deepgramStreaming.hasWarmConnection();+ debugLogger.debug("Deepgram streaming start", { hasWarmConnection: hasWarm }, "streaming");++ let token = this.deepgramStreaming.getCachedToken();+ if (!token) {+ debugLogger.debug("Fetching Deepgram streaming token from API", {}, "streaming");+ token = await fetchDeepgramStreamingToken(event);+ this.deepgramStreaming.cacheToken(token);+ } else {+ debugLogger.debug("Using cached Deepgram streaming token", {}, "streaming");+ }++ this.deepgramStreaming.onPartialTranscript = (text) => {+ if (win && !win.isDestroyed()) {+ win.webContents.send("deepgram-partial-transcript", text);+ }+ };++ this.deepgramStreaming.onFinalTranscript = (text) => {+ if (win && !win.isDestroyed()) {+ win.webContents.send("deepgram-final-transcript", text);+ }+ };++ this.deepgramStreaming.onError = (error) => {+ if (win && !win.isDestroyed()) {+ win.webContents.send("deepgram-error", error.message);+ }+ };++ this.deepgramStreaming.onSessionEnd = (data) => {+ if (win && !win.isDestroyed()) {+ win.webContents.send("deepgram-session-end", data);+ }+ };++ await this.deepgramStreaming.connect({ ...options, token });+ debugLogger.debug("Deepgram streaming started", {}, "streaming");++ return {+ success: true,+ usedWarmConnection: hasWarm,+ };+ } catch (error) {+ debugLogger.error("Deepgram streaming start error", { error: error.message });+ if (error.code === "AUTH_EXPIRED") {+ return { success: false, error: "Session expired", code: "AUTH_EXPIRED" };+ }+ return { success: false, error: error.message };+ } finally {+ deepgramStreamingStartInProgress = false;+ }+ });++ ipcMain.on("deepgram-streaming-send", (event, audioBuffer) => {+ try {+ if (!this.deepgramStreaming) return;+ const buffer = Buffer.from(audioBuffer);+ this.deepgramStreaming.sendAudio(buffer);+ } catch (error) {+ debugLogger.error("Deepgram streaming send error", { error: error.message });+ }+ });++ ipcMain.on("deepgram-streaming-finalize", () => {+ this.deepgramStreaming?.finalize();+ });++ ipcMain.handle("deepgram-streaming-stop", async () => {+ try {+ let result = { text: "" };+ if (this.deepgramStreaming) {+ result = await this.deepgramStreaming.disconnect(true);+ }++ return { success: true, text: result?.text || "" };+ } catch (error) {+ debugLogger.error("Deepgram streaming stop error", { error: error.message });+ return { success: false, error: error.message };+ }+ });++ ipcMain.handle("deepgram-streaming-status", async () => {+ if (!this.deepgramStreaming) {+ return { isConnected: false, sessionId: null };+ }+ return this.deepgramStreaming.getStatus();+ });}broadcastToWindows(channel, payload) {diff --git a/src/hooks/useAudioRecording.js b/src/hooks/useAudioRecording.jsindex 0541a501d..7dddce4f6 100644--- a/src/hooks/useAudioRecording.js+++ b/src/hooks/useAudioRecording.js@@ -39,11 +39,11 @@ export const useAudioRecording = (toast, options = {}) => {}const currentState = audioManagerRef.current.getState();- if (!currentState.isRecording) {+ if (!currentState.isRecording && !currentState.isStreamingStartInProgress) {return false;}- if (currentState.isStreaming) {+ if (currentState.isStreaming || currentState.isStreamingStartInProgress) {void playStopCue(); // streaming stop finalization is async, play cue immediately on stop actionreturn await audioManagerRef.current.stopStreamingRecording();}@@ -229,9 +229,9 @@ export const useAudioRecording = (toast, options = {}) => {}};- const warmupStreaming = () => {- audioManagerRef.current?.warmupStreamingConnection();- };+ const warmupStreaming = useCallback((opts) => {+ audioManagerRef.current?.warmupStreamingConnection(opts);+ }, []);return {isRecording,diff --git a/src/types/electron.ts b/src/types/electron.tsindex 448773ef3..2801bea44 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -526,6 +526,37 @@ declare global {onAssemblyAiSessionEnd?: (callback: (data: { audioDuration?: number; text?: string }) => void) => () => void;++ // Deepgram Streaming+ deepgramStreamingWarmup?: (options?: { sampleRate?: number; language?: string }) => Promise<{+ success: boolean;+ alreadyWarm?: boolean;+ error?: string;+ code?: string;+ }>;+ deepgramStreamingStart?: (options?: { sampleRate?: number; language?: string }) => Promise<{+ success: boolean;+ usedWarmConnection?: boolean;+ error?: string;+ code?: string;+ }>;+ deepgramStreamingSend?: (audioBuffer: ArrayBuffer) => void;+ deepgramStreamingFinalize?: () => void;+ deepgramStreamingStop?: () => Promise<{+ success: boolean;+ text?: string;+ error?: string;+ }>;+ deepgramStreamingStatus?: () => Promise<{+ isConnected: boolean;+ sessionId: string | null;+ }>;+ onDeepgramPartialTranscript?: (callback: (text: string) => void) => () => void;+ onDeepgramFinalTranscript?: (callback: (text: string) => void) => () => void;+ onDeepgramError?: (callback: (error: string) => void) => () => void;+ onDeepgramSessionEnd?: (+ callback: (data: { audioDuration?: number; text?: string }) => void+ ) => () => void;};api?: {