Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(dictation): Finalize the streaming provider the recording started on (#2434)

Chad Pihace2252b66dda
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index 13396e31c..9991b2e92 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -673,6 +673,7 @@ class AudioManager {
this._streamingCancellationGeneration = 0;
this._activeTranscriptionAbortController = null;
this._activeStreamingSessionId = null;
+ this._activeStreamingProviderName = null;
this.streamingFallbackRecorder = null;
this.streamingFallbackChunks = [];
this._recordingSpool = null;
@@ -1075,8 +1076,23 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this.sttConfigFetchedAt = null;
}
- getStreamingProvider() {
- return STREAMING_PROVIDERS[this.getStreamingProviderName()];
+ getStreamingProvider(name) {
+ return STREAMING_PROVIDERS[name];
+ }
+
+ // The provider the active streaming session started on. The sttConfig can be
+ // refreshed mid-recording and resolve a different provider, but the open
+ // socket, its final and its disconnect belong to the one that started.
+ getActiveStreamingProviderName() {
+ return this._activeStreamingProviderName ?? this.getStreamingProviderName();
+ }
+
+ // Only the session that set the active fields clears them: a newer start
+ // may already own them.
+ _endStreamingSession(sessionId) {
+ if (this._activeStreamingSessionId !== sessionId) return;
+ this._activeStreamingSessionId = null;
+ this._activeStreamingProviderName = null;
}
getStreamingProviderName() {
@@ -4767,7 +4783,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
return { needsFallback: true };
}
const fallbackReason = resolveStreamingStartFallback({
- providerName: this.getStreamingProviderName(),
+ providerName: this.getActiveStreamingProviderName(),
cloudTranscriptionMode: getSettings().cloudTranscriptionMode,
result: res,
});
@@ -4815,6 +4831,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
let acquiredStream = null;
let usedPreparedCapture = false;
let sessionId = null;
+ let providerName = null;
let startWasCancelled = () => false;
this._startInProgress = true;
try {
@@ -4839,6 +4856,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this._streamingFailoverReason = null;
this._streamingSpeechGateState = createLocalSpeechGateState();
this._activeStreamingSessionId = sessionId;
+ providerName = this.getStreamingProviderName();
+ this._activeStreamingProviderName = providerName;
const ownsSession = () => this._activeStreamingSessionId === sessionId;
const cancellationGeneration = this._streamingCancellationGeneration;
startWasCancelled = () => cancellationGeneration !== this._streamingCancellationGeneration;
@@ -4902,7 +4921,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
}
this.streamingProcessor = new AudioWorkletNode(audioContext, "pcm-streaming-processor");
- const provider = this.getStreamingProvider();
+ const provider = this.getStreamingProvider(providerName);
// Decided once, with the provider, so a config refresh mid-recording
// cannot change how this session's stream errors are handled. Only a
// session whose fallback recorder started has a capture to fail over
@@ -4910,7 +4929,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
const failsOver =
Boolean(this.streamingFallbackRecorder) &&
isManagedOrukeetStream({
- providerName: this.getStreamingProviderName(),
+ providerName,
cloudTranscriptionMode: getSettings().cloudTranscriptionMode,
});
@@ -5001,7 +5020,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
if (startWasCancelled()) {
// Cancelled while the mic was opening: never flip to recording.
await this.cleanupStreaming();
- if (ownsSession()) this._activeStreamingSessionId = null;
// cancelStreamingRecording may already be awaiting this start's
// settlement before it can disconnect the provider.
this._settleStreamingStart();
@@ -5020,7 +5038,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
const { useLocalWhisper } = streamingSettings;
const res = await provider.start(
buildStreamingSessionOptions({
- providerName: this.getStreamingProviderName(),
+ providerName,
settings: streamingSettings,
language: this.getEffectiveSttLanguage(streamingSettings),
keyterms: this.getKeyterms(),
@@ -5057,7 +5075,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this.recordingStartTime = null;
this.stopRequestedDuringStreamingStart = false;
await this.cleanupStreaming();
- if (ownsSession()) this._activeStreamingSessionId = null;
this.onStateChange?.({ isRecording: false, isProcessing: false, isStreaming: false });
logger.debug("Streaming unavailable, falling back to regular recording", {}, "streaming");
return this.startRecording();
@@ -5112,9 +5129,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
);
this.cachedMicDeviceId = null;
await this.cleanupStreaming();
- if (this._activeStreamingSessionId === sessionId) {
- this._activeStreamingSessionId = null;
- }
this.isRecording = false;
this.recordingStartTime = null;
this.onStateChange?.({ isRecording: false, isProcessing: false, isStreaming: false });
@@ -5123,7 +5137,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
logger.error(
"Failed to start streaming recording",
- { provider: this.getStreamingProviderName(), error: error.message, code: error.code },
+ { provider: providerName, error: error.message, code: error.code },
"streaming"
);
@@ -5158,9 +5172,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
});
await this.cleanupStreaming();
- if (this._activeStreamingSessionId === sessionId) {
- this._activeStreamingSessionId = null;
- }
this.isRecording = false;
this.recordingStartTime = null;
this.onStateChange?.({ isRecording: false, isProcessing: false, isStreaming: false });
@@ -5220,9 +5231,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this._streamingStopPromise = null;
this._streamingStopMode = null;
}
- if (this._activeStreamingSessionId === sessionId) {
- this._activeStreamingSessionId = null;
- }
+ this._endStreamingSession(sessionId);
// Finalization has several provider/reasoning awaits. A thrown error must
// never leave the renderer or main process stuck in a busy lifecycle.
@@ -5263,6 +5272,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
const sessionId = this._activeStreamingSessionId;
const spool = this._takeRecordingSpool();
+ const providerName = this.getActiveStreamingProviderName();
const cancelPromise = (async () => {
this._requestStreamingCancellation();
this.stopRequestedDuringStreamingStart = false;
@@ -5284,7 +5294,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this.micRecovery.stop();
this.cleanupStreamingAudio();
this.cleanupStreamingListeners(sessionId);
- return this.getStreamingProvider().stop?.();
+ return this.getStreamingProvider(providerName).stop?.();
})
.catch((error) => {
logger.debug(
@@ -5307,9 +5317,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this._streamingStopPromise = null;
this._streamingStopMode = null;
}
- if (this._activeStreamingSessionId === sessionId) {
- this._activeStreamingSessionId = null;
- }
+ this._endStreamingSession(sessionId);
this.isRecording = false;
this.isProcessing = false;
this.isStreaming = false;
@@ -5326,6 +5334,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
return false;
}
+ const providerName = this.getActiveStreamingProviderName();
const cancellationGeneration = this._streamingCancellationGeneration;
const wasCancelled = () => cancellationGeneration !== this._streamingCancellationGeneration;
const abandonFinalization = async () => {
@@ -5333,7 +5342,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this.cleanupStreamingListeners(sessionId);
this._streamingFallbackSegments = [];
try {
- await this.getStreamingProvider().stop?.();
+ await this.getStreamingProvider(providerName).stop?.();
} catch (error) {
logger.debug(
"Streaming disconnect after cancellation failed",
@@ -5368,7 +5377,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
const t0 = performance.now();
let finalText = this.streamingFinalText || "";
- const provider = this.getStreamingProvider();
+ const provider = this.getStreamingProvider(providerName);
let acknowledgedFinal = null;
let finalAcknowledged = false;
let orukeetFinal = null;
@@ -5536,7 +5545,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
let batchWarning = null;
let batchFallbackResult = null;
let failureReport = null;
- const isOrukeetStream = this.getStreamingProviderName() === "orukeet";
+ const isOrukeetStream = providerName === "orukeet";
const detectedLanguageFields = isOrukeetStream
? orukeetDetectedLanguageFields(orukeetFinal)
: {};
@@ -5652,7 +5661,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
customPrompt,
language: this.getCleanupLanguage(stSettings),
locale: stSettings.uiLanguage || "en",
- sttProvider: this.getStreamingProviderName(),
+ sttProvider: providerName,
sttModel: streamingSttModel,
sttProcessingMs: streamingSttProcessingMs,
sttWordCount: streamingSttWordCount,
@@ -5711,7 +5720,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
? {
mode: "cloudReason",
meta: {
- sttProvider: this.getStreamingProviderName(),
+ sttProvider: providerName,
sttModel: streamingSttModel,
sttProcessingMs: streamingSttProcessingMs,
sttWordCount: streamingSttWordCount,
@@ -5855,7 +5864,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
durationMs: durationSeconds
? Math.round(durationSeconds * 1000)
: Math.round(tBeforePaste - t0),
- provider: batchFallbackResult?.source || `${this.getStreamingProviderName()}-streaming`,
+ provider: batchFallbackResult?.source || `${providerName}-streaming`,
model: batchFallbackResult ? null : streamingSttModel || null,
};
if (wasCancelled()) return true;
@@ -5863,7 +5872,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
success: true,
text: finalText,
rawText: batchFallbackResult?.rawText || rawStreamingText || finalText,
- source: batchFallbackResult?.source || `${this.getStreamingProviderName()}-streaming`,
+ source: batchFallbackResult?.source || `${providerName}-streaming`,
clientTranscriptionId,
analyticsOccurredAt: resultAnalyticsOccurredAt,
// The upgrade prompt opens on these, as after a batch recording.
@@ -5887,7 +5896,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
durationSeconds ?? 0,
{
sendLogs: !usedCloudReasoning,
- sttProvider: this.getStreamingProviderName(),
+ sttProvider: providerName,
sttModel: streamingSttModel,
sttProcessingMs: streamingSttProcessingMs,
sttLanguage: streamingSttLanguage,
@@ -6085,12 +6094,12 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this._takeRecordingSpool()?.finish();
this.cleanupStreamingAudio();
this.cleanupStreamingListeners(sessionId);
- if (this._activeStreamingSessionId === sessionId) {
- this._activeStreamingSessionId = null;
- }
+ this._endStreamingSession(sessionId);
}
cleanup() {
+ // Read before cleanupStreaming ends the session below.
+ const streamingProviderName = this.getActiveStreamingProviderName();
this.micRecovery.stop();
this._unsubscribeSettings?.();
this.preparedMicCapture.cancel();
@@ -6114,7 +6123,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this.workletBlobUrl = null;
}
try {
- this.getStreamingProvider().stop?.();
+ this.getStreamingProvider(streamingProviderName).stop?.();
} catch (e) {
// Ignore errors during cleanup (page may be unloading)
}
diff --git a/test/helpers/audioManagerStreamingFinalization.test.js b/test/helpers/audioManagerStreamingFinalization.test.js
index 961f0c587..428f2f672 100644
--- a/test/helpers/audioManagerStreamingFinalization.test.js
+++ b/test/helpers/audioManagerStreamingFinalization.test.js
@@ -374,6 +374,102 @@ test("cancelling while the streaming microphone opens never enters recording", a
);
});
+// A managed Orukeet recording that resolves its provider through the real
+// sttConfig routing, with every provider channel recording what reached it.
+async function startManagedOrukeetRecording(t, AudioManager) {
+ useManagedOrukeetSettings();
+ installCapture(t);
+ const calls = [];
+ const record = (name, result) => async () => {
+ calls.push(name);
+ return result;
+ };
+ const listen = () => () => {};
+ let raiseStreamError;
+ Object.assign(globalThis.window.electronAPI, {
+ dictationRealtimeStart: async (options) => {
+ calls.push(`${options.provider}:start:${options.model}`);
+ return { success: true };
+ },
+ dictationRealtimeFinalize: record("realtime:finalize", {
+ success: true,
+ text: "Bonjour",
+ language: "fr",
+ }),
+ dictationRealtimeStop: record("realtime:stop", { success: true }),
+ onDictationRealtimePartial: listen,
+ onDictationRealtimeFinal: listen,
+ onDictationRealtimeError: (listener) => {
+ raiseStreamError = listener;
+ return () => {};
+ },
+ onDictationRealtimeSessionEnd: listen,
+ deepgramStreamingFinalize: record("deepgram:finalize", { success: true }),
+ deepgramStreamingStop: record("deepgram:stop", { success: true }),
+ cloudStreamingUsage: async (_text, _durationSeconds, usage) => {
+ calls.push(`usage:${usage.sttProvider}:${usage.sttDetectedLanguage}`);
+ return { success: true };
+ },
+ });
+ globalThis.window.dispatchEvent = () => true;
+
+ const { manager, errors, completions } = createStartingManager(AudioManager, {});
+ delete manager.getStreamingProvider;
+ delete manager.getStreamingProviderName;
+ manager.setSttConfig({ dictation: { mode: "streaming" }, streamingProvider: "orukeet" });
+ // The hook's background TTL refresh lands while the mic opens and would now
+ // route a new recording elsewhere.
+ const openMic = manager._acquireCaptureStream;
+ manager._acquireCaptureStream = async (constraints) => {
+ manager.setSttConfig({ dictation: { mode: "streaming" }, streamingProvider: "deepgram" });
+ return openMic(constraints);
+ };
+
+ assert.equal(await manager.startStreamingRecording(), true);
+ assert.equal(manager.getStreamingProviderName(), "deepgram");
+
+ return { manager, calls, completions, errors, raiseStreamError };
+}
+
+test("stop finalizes the Orukeet stream after an STT config refresh reroutes while the mic opens", async (t) => {
+ const AudioManager = await loadManagerClass(t);
+ const { manager, calls, completions } = await startManagedOrukeetRecording(t, AudioManager);
+
+ assert.equal(await manager.stopStreamingRecording(), true);
+
+ assert.deepEqual(calls, [
+ "orukeet:start:orukeet-v0.1.0",
+ "realtime:finalize",
+ "realtime:stop",
+ "usage:orukeet:fr",
+ ]);
+ assert.equal(completions.length, 1);
+ assert.equal(completions[0].text, "Bonjour");
+ assert.equal(completions[0].source, "orukeet-streaming");
+ assert.equal(manager._activeStreamingProviderName, null);
+});
+
+test("cancel and cleanup disconnect the Orukeet stream after an STT config refresh reroutes while the mic opens", async (t) => {
+ const AudioManager = await loadManagerClass(t);
+ for (const teardown of ["cancelStreamingRecording", "cleanup"]) {
+ await t.test(teardown, async (t) => {
+ const { manager, calls, completions, errors, raiseStreamError } =
+ await startManagedOrukeetRecording(t, AudioManager);
+ manager.preparedMicCapture.cancel = () => {};
+ manager.micStreamHold = { drop() {} };
+ // The managed Orukeet session fails over, so the error never surfaces.
+ raiseStreamError("Orukeet connection closed");
+
+ await manager[teardown]();
+
+ assert.deepEqual(calls, ["orukeet:start:orukeet-v0.1.0", "realtime:stop"]);
+ assert.deepEqual(completions, []);
+ assert.deepEqual(errors, []);
+ assert.equal(manager._activeStreamingProviderName, null);
+ });
+ }
+});
+
test("cancel overrides a normal streaming stop before it can publish text", async (t) => {
const AudioManager = await loadManagerClass(t);
const { manager } = createFinalizingManager(AudioManager);
diff --git a/test/helpers/audioManagerStreamingRouting.test.js b/test/helpers/audioManagerStreamingRouting.test.js
index fac4eebb1..77f5d10a1 100644
--- a/test/helpers/audioManagerStreamingRouting.test.js
+++ b/test/helpers/audioManagerStreamingRouting.test.js
@@ -45,7 +45,7 @@ test("OpenAI dictation realtime requests identify the token provider", async (t)
return { success: true };
};
- const provider = manager.getStreamingProvider();
+ const provider = manager.getStreamingProvider(manager.getStreamingProviderName());
const options = {
model: "gpt-4o-mini-transcribe",
mode: "byok",
@@ -77,7 +77,7 @@ test("Gemini's live model routes onto the gemini-streaming-* channels", async (t
};
assert.equal(manager.getStreamingProviderName(), "gemini");
- const provider = manager.getStreamingProvider();
+ const provider = manager.getStreamingProvider("gemini");
assert.equal(provider.awaitsFinalTranscript, true);
// Pinned to geminiLiveStreaming.js's DISCONNECT_TIMEOUT_MS: both sides
// measure the same 3s from audioStreamEnd.
@@ -210,7 +210,7 @@ test("explicit Orukeet custom endpoint uses PCM streaming without cloud login",
});
assert.equal(manager.shouldUseStreaming(), true);
assert.equal(manager.getStreamingProviderName(), "orukeet");
- assert.equal(manager.getStreamingProvider().finalizeAcknowledged, true);
+ assert.equal(manager.getStreamingProvider("orukeet").finalizeAcknowledged, true);
setSettings({
transcriptionMode: "self-hosted",
cloudTranscriptionProvider: "custom",