Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(dictation): Finalize the streaming provider the recording started on (#2434)
Chad Pihace2252b66dda
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex 13396e31c..9991b2e92 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -673,6 +673,7 @@ class AudioManager {this._streamingCancellationGeneration = 0;this._activeTranscriptionAbortController = null;this._activeStreamingSessionId = null;+ this._activeStreamingProviderName = null;this.streamingFallbackRecorder = null;this.streamingFallbackChunks = [];this._recordingSpool = null;@@ -1075,8 +1076,23 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.sttConfigFetchedAt = null;}- getStreamingProvider() {- return STREAMING_PROVIDERS[this.getStreamingProviderName()];+ getStreamingProvider(name) {+ return STREAMING_PROVIDERS[name];+ }++ // The provider the active streaming session started on. The sttConfig can be+ // refreshed mid-recording and resolve a different provider, but the open+ // socket, its final and its disconnect belong to the one that started.+ getActiveStreamingProviderName() {+ return this._activeStreamingProviderName ?? this.getStreamingProviderName();+ }++ // Only the session that set the active fields clears them: a newer start+ // may already own them.+ _endStreamingSession(sessionId) {+ if (this._activeStreamingSessionId !== sessionId) return;+ this._activeStreamingSessionId = null;+ this._activeStreamingProviderName = null;}getStreamingProviderName() {@@ -4767,7 +4783,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);return { needsFallback: true };}const fallbackReason = resolveStreamingStartFallback({- providerName: this.getStreamingProviderName(),+ providerName: this.getActiveStreamingProviderName(),cloudTranscriptionMode: getSettings().cloudTranscriptionMode,result: res,});@@ -4815,6 +4831,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);let acquiredStream = null;let usedPreparedCapture = false;let sessionId = null;+ let providerName = null;let startWasCancelled = () => false;this._startInProgress = true;try {@@ -4839,6 +4856,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this._streamingFailoverReason = null;this._streamingSpeechGateState = createLocalSpeechGateState();this._activeStreamingSessionId = sessionId;+ providerName = this.getStreamingProviderName();+ this._activeStreamingProviderName = providerName;const ownsSession = () => this._activeStreamingSessionId === sessionId;const cancellationGeneration = this._streamingCancellationGeneration;startWasCancelled = () => cancellationGeneration !== this._streamingCancellationGeneration;@@ -4902,7 +4921,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);}this.streamingProcessor = new AudioWorkletNode(audioContext, "pcm-streaming-processor");- const provider = this.getStreamingProvider();+ const provider = this.getStreamingProvider(providerName);// Decided once, with the provider, so a config refresh mid-recording// cannot change how this session's stream errors are handled. Only a// session whose fallback recorder started has a capture to fail over@@ -4910,7 +4929,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);const failsOver =Boolean(this.streamingFallbackRecorder) &&isManagedOrukeetStream({- providerName: this.getStreamingProviderName(),+ providerName,cloudTranscriptionMode: getSettings().cloudTranscriptionMode,});@@ -5001,7 +5020,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);if (startWasCancelled()) {// Cancelled while the mic was opening: never flip to recording.await this.cleanupStreaming();- if (ownsSession()) this._activeStreamingSessionId = null;// cancelStreamingRecording may already be awaiting this start's// settlement before it can disconnect the provider.this._settleStreamingStart();@@ -5020,7 +5038,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);const { useLocalWhisper } = streamingSettings;const res = await provider.start(buildStreamingSessionOptions({- providerName: this.getStreamingProviderName(),+ providerName,settings: streamingSettings,language: this.getEffectiveSttLanguage(streamingSettings),keyterms: this.getKeyterms(),@@ -5057,7 +5075,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.recordingStartTime = null;this.stopRequestedDuringStreamingStart = false;await this.cleanupStreaming();- if (ownsSession()) this._activeStreamingSessionId = null;this.onStateChange?.({ isRecording: false, isProcessing: false, isStreaming: false });logger.debug("Streaming unavailable, falling back to regular recording", {}, "streaming");return this.startRecording();@@ -5112,9 +5129,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor););this.cachedMicDeviceId = null;await this.cleanupStreaming();- if (this._activeStreamingSessionId === sessionId) {- this._activeStreamingSessionId = null;- }this.isRecording = false;this.recordingStartTime = null;this.onStateChange?.({ isRecording: false, isProcessing: false, isStreaming: false });@@ -5123,7 +5137,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);logger.error("Failed to start streaming recording",- { provider: this.getStreamingProviderName(), error: error.message, code: error.code },+ { provider: providerName, error: error.message, code: error.code },"streaming");@@ -5158,9 +5172,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);});await this.cleanupStreaming();- if (this._activeStreamingSessionId === sessionId) {- this._activeStreamingSessionId = null;- }this.isRecording = false;this.recordingStartTime = null;this.onStateChange?.({ isRecording: false, isProcessing: false, isStreaming: false });@@ -5220,9 +5231,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this._streamingStopPromise = null;this._streamingStopMode = null;}- if (this._activeStreamingSessionId === sessionId) {- this._activeStreamingSessionId = null;- }+ this._endStreamingSession(sessionId);// Finalization has several provider/reasoning awaits. A thrown error must// never leave the renderer or main process stuck in a busy lifecycle.@@ -5263,6 +5272,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);const sessionId = this._activeStreamingSessionId;const spool = this._takeRecordingSpool();+ const providerName = this.getActiveStreamingProviderName();const cancelPromise = (async () => {this._requestStreamingCancellation();this.stopRequestedDuringStreamingStart = false;@@ -5284,7 +5294,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.micRecovery.stop();this.cleanupStreamingAudio();this.cleanupStreamingListeners(sessionId);- return this.getStreamingProvider().stop?.();+ return this.getStreamingProvider(providerName).stop?.();}).catch((error) => {logger.debug(@@ -5307,9 +5317,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this._streamingStopPromise = null;this._streamingStopMode = null;}- if (this._activeStreamingSessionId === sessionId) {- this._activeStreamingSessionId = null;- }+ this._endStreamingSession(sessionId);this.isRecording = false;this.isProcessing = false;this.isStreaming = false;@@ -5326,6 +5334,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);return false;}+ const providerName = this.getActiveStreamingProviderName();const cancellationGeneration = this._streamingCancellationGeneration;const wasCancelled = () => cancellationGeneration !== this._streamingCancellationGeneration;const abandonFinalization = async () => {@@ -5333,7 +5342,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.cleanupStreamingListeners(sessionId);this._streamingFallbackSegments = [];try {- await this.getStreamingProvider().stop?.();+ await this.getStreamingProvider(providerName).stop?.();} catch (error) {logger.debug("Streaming disconnect after cancellation failed",@@ -5368,7 +5377,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);const t0 = performance.now();let finalText = this.streamingFinalText || "";- const provider = this.getStreamingProvider();+ const provider = this.getStreamingProvider(providerName);let acknowledgedFinal = null;let finalAcknowledged = false;let orukeetFinal = null;@@ -5536,7 +5545,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);let batchWarning = null;let batchFallbackResult = null;let failureReport = null;- const isOrukeetStream = this.getStreamingProviderName() === "orukeet";+ const isOrukeetStream = providerName === "orukeet";const detectedLanguageFields = isOrukeetStream? orukeetDetectedLanguageFields(orukeetFinal): {};@@ -5652,7 +5661,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);customPrompt,language: this.getCleanupLanguage(stSettings),locale: stSettings.uiLanguage || "en",- sttProvider: this.getStreamingProviderName(),+ sttProvider: providerName,sttModel: streamingSttModel,sttProcessingMs: streamingSttProcessingMs,sttWordCount: streamingSttWordCount,@@ -5711,7 +5720,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);? {mode: "cloudReason",meta: {- sttProvider: this.getStreamingProviderName(),+ sttProvider: providerName,sttModel: streamingSttModel,sttProcessingMs: streamingSttProcessingMs,sttWordCount: streamingSttWordCount,@@ -5855,7 +5864,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);durationMs: durationSeconds? Math.round(durationSeconds * 1000): Math.round(tBeforePaste - t0),- provider: batchFallbackResult?.source || `${this.getStreamingProviderName()}-streaming`,+ provider: batchFallbackResult?.source || `${providerName}-streaming`,model: batchFallbackResult ? null : streamingSttModel || null,};if (wasCancelled()) return true;@@ -5863,7 +5872,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);success: true,text: finalText,rawText: batchFallbackResult?.rawText || rawStreamingText || finalText,- source: batchFallbackResult?.source || `${this.getStreamingProviderName()}-streaming`,+ source: batchFallbackResult?.source || `${providerName}-streaming`,clientTranscriptionId,analyticsOccurredAt: resultAnalyticsOccurredAt,// The upgrade prompt opens on these, as after a batch recording.@@ -5887,7 +5896,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);durationSeconds ?? 0,{sendLogs: !usedCloudReasoning,- sttProvider: this.getStreamingProviderName(),+ sttProvider: providerName,sttModel: streamingSttModel,sttProcessingMs: streamingSttProcessingMs,sttLanguage: streamingSttLanguage,@@ -6085,12 +6094,12 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this._takeRecordingSpool()?.finish();this.cleanupStreamingAudio();this.cleanupStreamingListeners(sessionId);- if (this._activeStreamingSessionId === sessionId) {- this._activeStreamingSessionId = null;- }+ this._endStreamingSession(sessionId);}cleanup() {+ // Read before cleanupStreaming ends the session below.+ const streamingProviderName = this.getActiveStreamingProviderName();this.micRecovery.stop();this._unsubscribeSettings?.();this.preparedMicCapture.cancel();@@ -6114,7 +6123,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.workletBlobUrl = null;}try {- this.getStreamingProvider().stop?.();+ this.getStreamingProvider(streamingProviderName).stop?.();} catch (e) {// Ignore errors during cleanup (page may be unloading)}diff --git a/test/helpers/audioManagerStreamingFinalization.test.js b/test/helpers/audioManagerStreamingFinalization.test.jsindex 961f0c587..428f2f672 100644--- a/test/helpers/audioManagerStreamingFinalization.test.js+++ b/test/helpers/audioManagerStreamingFinalization.test.js@@ -374,6 +374,102 @@ test("cancelling while the streaming microphone opens never enters recording", a);});+// A managed Orukeet recording that resolves its provider through the real+// sttConfig routing, with every provider channel recording what reached it.+async function startManagedOrukeetRecording(t, AudioManager) {+ useManagedOrukeetSettings();+ installCapture(t);+ const calls = [];+ const record = (name, result) => async () => {+ calls.push(name);+ return result;+ };+ const listen = () => () => {};+ let raiseStreamError;+ Object.assign(globalThis.window.electronAPI, {+ dictationRealtimeStart: async (options) => {+ calls.push(`${options.provider}:start:${options.model}`);+ return { success: true };+ },+ dictationRealtimeFinalize: record("realtime:finalize", {+ success: true,+ text: "Bonjour",+ language: "fr",+ }),+ dictationRealtimeStop: record("realtime:stop", { success: true }),+ onDictationRealtimePartial: listen,+ onDictationRealtimeFinal: listen,+ onDictationRealtimeError: (listener) => {+ raiseStreamError = listener;+ return () => {};+ },+ onDictationRealtimeSessionEnd: listen,+ deepgramStreamingFinalize: record("deepgram:finalize", { success: true }),+ deepgramStreamingStop: record("deepgram:stop", { success: true }),+ cloudStreamingUsage: async (_text, _durationSeconds, usage) => {+ calls.push(`usage:${usage.sttProvider}:${usage.sttDetectedLanguage}`);+ return { success: true };+ },+ });+ globalThis.window.dispatchEvent = () => true;++ const { manager, errors, completions } = createStartingManager(AudioManager, {});+ delete manager.getStreamingProvider;+ delete manager.getStreamingProviderName;+ manager.setSttConfig({ dictation: { mode: "streaming" }, streamingProvider: "orukeet" });+ // The hook's background TTL refresh lands while the mic opens and would now+ // route a new recording elsewhere.+ const openMic = manager._acquireCaptureStream;+ manager._acquireCaptureStream = async (constraints) => {+ manager.setSttConfig({ dictation: { mode: "streaming" }, streamingProvider: "deepgram" });+ return openMic(constraints);+ };++ assert.equal(await manager.startStreamingRecording(), true);+ assert.equal(manager.getStreamingProviderName(), "deepgram");++ return { manager, calls, completions, errors, raiseStreamError };+}++test("stop finalizes the Orukeet stream after an STT config refresh reroutes while the mic opens", async (t) => {+ const AudioManager = await loadManagerClass(t);+ const { manager, calls, completions } = await startManagedOrukeetRecording(t, AudioManager);++ assert.equal(await manager.stopStreamingRecording(), true);++ assert.deepEqual(calls, [+ "orukeet:start:orukeet-v0.1.0",+ "realtime:finalize",+ "realtime:stop",+ "usage:orukeet:fr",+ ]);+ assert.equal(completions.length, 1);+ assert.equal(completions[0].text, "Bonjour");+ assert.equal(completions[0].source, "orukeet-streaming");+ assert.equal(manager._activeStreamingProviderName, null);+});++test("cancel and cleanup disconnect the Orukeet stream after an STT config refresh reroutes while the mic opens", async (t) => {+ const AudioManager = await loadManagerClass(t);+ for (const teardown of ["cancelStreamingRecording", "cleanup"]) {+ await t.test(teardown, async (t) => {+ const { manager, calls, completions, errors, raiseStreamError } =+ await startManagedOrukeetRecording(t, AudioManager);+ manager.preparedMicCapture.cancel = () => {};+ manager.micStreamHold = { drop() {} };+ // The managed Orukeet session fails over, so the error never surfaces.+ raiseStreamError("Orukeet connection closed");++ await manager[teardown]();++ assert.deepEqual(calls, ["orukeet:start:orukeet-v0.1.0", "realtime:stop"]);+ assert.deepEqual(completions, []);+ assert.deepEqual(errors, []);+ assert.equal(manager._activeStreamingProviderName, null);+ });+ }+});+test("cancel overrides a normal streaming stop before it can publish text", async (t) => {const AudioManager = await loadManagerClass(t);const { manager } = createFinalizingManager(AudioManager);diff --git a/test/helpers/audioManagerStreamingRouting.test.js b/test/helpers/audioManagerStreamingRouting.test.jsindex fac4eebb1..77f5d10a1 100644--- a/test/helpers/audioManagerStreamingRouting.test.js+++ b/test/helpers/audioManagerStreamingRouting.test.js@@ -45,7 +45,7 @@ test("OpenAI dictation realtime requests identify the token provider", async (t)return { success: true };};- const provider = manager.getStreamingProvider();+ const provider = manager.getStreamingProvider(manager.getStreamingProviderName());const options = {model: "gpt-4o-mini-transcribe",mode: "byok",@@ -77,7 +77,7 @@ test("Gemini's live model routes onto the gemini-streaming-* channels", async (t};assert.equal(manager.getStreamingProviderName(), "gemini");- const provider = manager.getStreamingProvider();+ const provider = manager.getStreamingProvider("gemini");assert.equal(provider.awaitsFinalTranscript, true);// Pinned to geminiLiveStreaming.js's DISCONNECT_TIMEOUT_MS: both sides// measure the same 3s from audioStreamEnd.@@ -210,7 +210,7 @@ test("explicit Orukeet custom endpoint uses PCM streaming without cloud login",});assert.equal(manager.shouldUseStreaming(), true);assert.equal(manager.getStreamingProviderName(), "orukeet");- assert.equal(manager.getStreamingProvider().finalizeAcknowledged, true);+ assert.equal(manager.getStreamingProvider("orukeet").finalizeAcknowledged, true);setSettings({transcriptionMode: "self-hosted",cloudTranscriptionProvider: "custom",