Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(gemini): keep the trailing Live partial on stop (#2263)
Henry Su8e9ecd973efc
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex fbbffe533..bbcae6eee 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -430,6 +430,9 @@ const STREAMING_PROVIDERS = {// audioStreamEnd before its own disconnect gives up.awaitsFinalTranscript: true,finalCeilingMs: 3000,+ // Its stop result supersedes the streamed finals: it also carries a last+ // turn the server never finalized.+ preferStopTranscript: true,warmup: (opts) => window.electronAPI.geminiStreamingWarmup(opts),start: (opts) => window.electronAPI.geminiStreamingStart(opts),send: (buf) => window.electronAPI.geminiStreamingSend(buf),@@ -5260,7 +5263,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);});const tTerminate = performance.now();- finalText = this.streamingFinalText || "";+ finalText =+ (provider.preferStopTranscript && stopResult?.text) || this.streamingFinalText || "";if (!finalText && this.streamingPartialText) {finalText = this.streamingPartialText;diff --git a/src/helpers/geminiLiveStreaming.js b/src/helpers/geminiLiveStreaming.jsindex f659bf4b5..aa550fdbc 100644--- a/src/helpers/geminiLiveStreaming.js+++ b/src/helpers/geminiLiveStreaming.js@@ -67,6 +67,7 @@ class GeminiLiveStreaming {this.isConnecting = false;this.isDisconnecting = false;this.completedSegments = [];+ this.openTurnPartial = "";this.onPartialTranscript = null;this.onFinalTranscript = null;this.onError = null;@@ -95,8 +96,23 @@ class GeminiLiveStreaming {this.coldStartBufferSize = 0;}+ // A turn the server never finalized (it can go silent after audioStreamEnd)+ // exists only as its partial, so the transcript keeps it rather than losing+ // everything since the last pause.getFullTranscript() {- return this.completedSegments.join(" ");+ return [...this.completedSegments, this._openTurnText()].filter(Boolean).join(" ");+ }++ // The partial sometimes restates the finalized turns first, run together+ // without spaces ("...October.Before then"); only what follows is new.+ _openTurnText() {+ let text = this.openTurnPartial;+ for (const segment of this.completedSegments) {+ const rest = text.trimStart();+ if (!rest.startsWith(segment)) break;+ text = rest.slice(segment.length);+ }+ return text.trim();}// Test seam: overridden to point at a loopback server.@@ -140,6 +156,7 @@ class GeminiLiveStreaming {if (!this.bufferingAudio) this.beginConnecting();this.completedSegments = [];+ this.openTurnPartial = "";this.audioBytesSent = 0;this.currentModel = resolveLiveModel(options.model);this._connectionLossNotified = false;@@ -238,12 +255,14 @@ class GeminiLiveStreaming {const partial = serverContent.interimInputTranscription?.text;if (partial) {this._turnEnded = false;+ this.openTurnPartial = partial;this.onPartialTranscript?.(partial);}const final = serverContent.inputTranscription?.text?.trim();if (final) {this.completedSegments.push(final);+ this.openTurnPartial = "";const fullText = this.getFullTranscript();this.onFinalTranscript?.(fullText, Date.now());debugLogger.debug("Gemini Live turn completed", {@@ -448,8 +467,13 @@ class GeminiLiveStreaming {}_takeTranscript() {+ const openTurnLength = this._openTurnText().length;+ if (openTurnLength) {+ debugLogger.debug("Gemini Live last turn was never finalized", { openTurnLength });+ }const result = { text: this.getFullTranscript() };this.completedSegments = [];+ this.openTurnPartial = "";return result;}diff --git a/test/helpers/audioManagerStreamingFinalization.test.js b/test/helpers/audioManagerStreamingFinalization.test.jsindex dc7954025..1e66a0e21 100644--- a/test/helpers/audioManagerStreamingFinalization.test.js+++ b/test/helpers/audioManagerStreamingFinalization.test.js@@ -139,6 +139,35 @@ test("streaming completion keeps the recording occurrence time", async (t) => {assert.equal(completion.analyticsOccurredAt, new Date(recordingStartedAt).toISOString());});+test("only a provider that prefers its stop transcript pastes it over the streamed finals", async (t) => {+ const AudioManager = await loadManagerClass(t);+ globalThis.window.dispatchEvent = () => true;++ for (const [preferStopTranscript, expected] of [+ [true, "Turn one. Turn two is still open"],+ [false, "Turn one."],+ ]) {+ const { manager } = createFinalizingManager(AudioManager);+ let completion;+ manager.streamingFinalText = "Turn one.";+ manager.streamingPartialText = "Turn two is still open";+ manager.getStreamingProvider = () => ({+ awaitsFinalTranscript: true,+ preferStopTranscript,+ finalize() {},+ stop: async () => ({ success: true, text: "Turn one. Turn two is still open" }),+ });+ manager.finalizeChineseScript = async (text) => text;+ manager.onTranscriptionComplete = (result) => {+ completion = result;+ };++ await manager.stopStreamingRecording();++ assert.equal(completion.text, expected);+ }+});+test("cancelling an active streaming recording discards it without publishing text", async (t) => {const AudioManager = await loadManagerClass(t);const { manager, states, getProviderStopCalls } = createFinalizingManager(AudioManager);diff --git a/test/helpers/audioManagerStreamingRouting.test.js b/test/helpers/audioManagerStreamingRouting.test.jsindex fb5dc59f1..87a7c4ee5 100644--- a/test/helpers/audioManagerStreamingRouting.test.js+++ b/test/helpers/audioManagerStreamingRouting.test.js@@ -82,6 +82,7 @@ test("Gemini's live model routes onto the gemini-streaming-* channels", async (t// Pinned to geminiLiveStreaming.js's DISCONNECT_TIMEOUT_MS: both sides// measure the same 3s from audioStreamEnd.assert.equal(provider.finalCeilingMs, 3000);+ assert.equal(provider.preferStopTranscript, true);const options = { provider: "gemini", model: "gemini-3.5-transcribe-live", mode: "byok" };await provider.warmup(options);diff --git a/test/helpers/geminiLiveStreaming.test.js b/test/helpers/geminiLiveStreaming.test.jsindex a790ec5f8..119330f5f 100644--- a/test/helpers/geminiLiveStreaming.test.js+++ b/test/helpers/geminiLiveStreaming.test.js@@ -460,6 +460,66 @@ for (const [code, reason] of [});}+test("disconnect keeps a last turn the server never finalized", async () => {+ await withServer(+ async ({ streaming }) => {+ await streaming.connect({ token: "t", mode: "byok" });+ streaming.sendAudio(FRAME);+ await new Promise((resolve) => setTimeout(resolve, 30));++ assert.equal(streaming.finalize(), true);+ streaming._audioStreamEndSentAt -= 10000;++ // An earlier turn was finalized; the last one only ever had a partial.+ assert.deepEqual(await streaming.disconnect(true), {+ text: "Turn one. Turn two is still open",+ });+ },+ (socket, message) => {+ if (message.setup) {+ socket.send(JSON.stringify({ setupComplete: {} }));+ return;+ }+ if (!message.realtimeInput?.audio) return;+ socket.send(JSON.stringify({ serverContent: { inputTranscription: { text: "Turn one." } } }));+ socket.send(JSON.stringify({ serverContent: { generationComplete: true } }));+ socket.send(+ JSON.stringify({+ serverContent: { interimInputTranscription: { text: "Turn two is still open" } },+ })+ );+ }+ );+});++test("a partial that restates the finalized turns adds only the open turn", async () => {+ await withServer(+ async ({ streaming }) => {+ await streaming.connect({ token: "t", mode: "byok" });+ streaming.sendAudio(FRAME);+ await new Promise((resolve) => setTimeout(resolve, 30));++ assert.equal(streaming.getFullTranscript(), "Turn one. Turn two. Turn three");+ },+ (socket, message) => {+ if (message.setup) {+ socket.send(JSON.stringify({ setupComplete: {} }));+ return;+ }+ if (!message.realtimeInput?.audio) return;+ for (const text of ["Turn one.", "Turn two."]) {+ socket.send(JSON.stringify({ serverContent: { inputTranscription: { text } } }));+ }+ // Real Live sessions sometimes repeat every finalized turn, unspaced.+ socket.send(+ JSON.stringify({+ serverContent: { interimInputTranscription: { text: "Turn one.Turn two.Turn three" } },+ })+ );+ }+ );+});+test("errors: a spent managed token is re-minted once and the session survives", async () => {await withServer(async ({ streaming, urls }) => {@@ -510,7 +570,8 @@ test("an unexpected close hands the transcript over and reports the loss once",streaming.sendAudio(FRAME);await new Promise((resolve) => setTimeout(resolve, 60));- assert.deepEqual(sessionEnds, [{ text: "half a sentence" }]);+ // The open turn's partial is handed over with the finalized text.+ assert.deepEqual(sessionEnds, [{ text: "half a sentence and the rest" }]);// Distinct from the pre-ready failure, and without the raw server reason.assert.deepEqual(errors, ["Connection lost (code: 1011)"]);assert.equal(streaming.isConnected, false);@@ -524,6 +585,9 @@ test("an unexpected close hands the transcript over and reports the loss once",socket.send(JSON.stringify({ serverContent: { inputTranscription: { text: "half a sentence" } } }));+ socket.send(+ JSON.stringify({ serverContent: { interimInputTranscription: { text: "and the rest" } } })+ );setTimeout(() => socket.close(1011, "Token has expired"), 10);});