Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(gemini): keep the trailing Live partial on stop (#2263)

Henry Su8e9ecd973efc
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index fbbffe533..bbcae6eee 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -430,6 +430,9 @@ const STREAMING_PROVIDERS = {
// audioStreamEnd before its own disconnect gives up.
awaitsFinalTranscript: true,
finalCeilingMs: 3000,
+ // Its stop result supersedes the streamed finals: it also carries a last
+ // turn the server never finalized.
+ preferStopTranscript: true,
warmup: (opts) => window.electronAPI.geminiStreamingWarmup(opts),
start: (opts) => window.electronAPI.geminiStreamingStart(opts),
send: (buf) => window.electronAPI.geminiStreamingSend(buf),
@@ -5260,7 +5263,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
});
const tTerminate = performance.now();
- finalText = this.streamingFinalText || "";
+ finalText =
+ (provider.preferStopTranscript && stopResult?.text) || this.streamingFinalText || "";
if (!finalText && this.streamingPartialText) {
finalText = this.streamingPartialText;
diff --git a/src/helpers/geminiLiveStreaming.js b/src/helpers/geminiLiveStreaming.js
index f659bf4b5..aa550fdbc 100644
--- a/src/helpers/geminiLiveStreaming.js
+++ b/src/helpers/geminiLiveStreaming.js
@@ -67,6 +67,7 @@ class GeminiLiveStreaming {
this.isConnecting = false;
this.isDisconnecting = false;
this.completedSegments = [];
+ this.openTurnPartial = "";
this.onPartialTranscript = null;
this.onFinalTranscript = null;
this.onError = null;
@@ -95,8 +96,23 @@ class GeminiLiveStreaming {
this.coldStartBufferSize = 0;
}
+ // A turn the server never finalized (it can go silent after audioStreamEnd)
+ // exists only as its partial, so the transcript keeps it rather than losing
+ // everything since the last pause.
getFullTranscript() {
- return this.completedSegments.join(" ");
+ return [...this.completedSegments, this._openTurnText()].filter(Boolean).join(" ");
+ }
+
+ // The partial sometimes restates the finalized turns first, run together
+ // without spaces ("...October.Before then"); only what follows is new.
+ _openTurnText() {
+ let text = this.openTurnPartial;
+ for (const segment of this.completedSegments) {
+ const rest = text.trimStart();
+ if (!rest.startsWith(segment)) break;
+ text = rest.slice(segment.length);
+ }
+ return text.trim();
}
// Test seam: overridden to point at a loopback server.
@@ -140,6 +156,7 @@ class GeminiLiveStreaming {
if (!this.bufferingAudio) this.beginConnecting();
this.completedSegments = [];
+ this.openTurnPartial = "";
this.audioBytesSent = 0;
this.currentModel = resolveLiveModel(options.model);
this._connectionLossNotified = false;
@@ -238,12 +255,14 @@ class GeminiLiveStreaming {
const partial = serverContent.interimInputTranscription?.text;
if (partial) {
this._turnEnded = false;
+ this.openTurnPartial = partial;
this.onPartialTranscript?.(partial);
}
const final = serverContent.inputTranscription?.text?.trim();
if (final) {
this.completedSegments.push(final);
+ this.openTurnPartial = "";
const fullText = this.getFullTranscript();
this.onFinalTranscript?.(fullText, Date.now());
debugLogger.debug("Gemini Live turn completed", {
@@ -448,8 +467,13 @@ class GeminiLiveStreaming {
}
_takeTranscript() {
+ const openTurnLength = this._openTurnText().length;
+ if (openTurnLength) {
+ debugLogger.debug("Gemini Live last turn was never finalized", { openTurnLength });
+ }
const result = { text: this.getFullTranscript() };
this.completedSegments = [];
+ this.openTurnPartial = "";
return result;
}
diff --git a/test/helpers/audioManagerStreamingFinalization.test.js b/test/helpers/audioManagerStreamingFinalization.test.js
index dc7954025..1e66a0e21 100644
--- a/test/helpers/audioManagerStreamingFinalization.test.js
+++ b/test/helpers/audioManagerStreamingFinalization.test.js
@@ -139,6 +139,35 @@ test("streaming completion keeps the recording occurrence time", async (t) => {
assert.equal(completion.analyticsOccurredAt, new Date(recordingStartedAt).toISOString());
});
+test("only a provider that prefers its stop transcript pastes it over the streamed finals", async (t) => {
+ const AudioManager = await loadManagerClass(t);
+ globalThis.window.dispatchEvent = () => true;
+
+ for (const [preferStopTranscript, expected] of [
+ [true, "Turn one. Turn two is still open"],
+ [false, "Turn one."],
+ ]) {
+ const { manager } = createFinalizingManager(AudioManager);
+ let completion;
+ manager.streamingFinalText = "Turn one.";
+ manager.streamingPartialText = "Turn two is still open";
+ manager.getStreamingProvider = () => ({
+ awaitsFinalTranscript: true,
+ preferStopTranscript,
+ finalize() {},
+ stop: async () => ({ success: true, text: "Turn one. Turn two is still open" }),
+ });
+ manager.finalizeChineseScript = async (text) => text;
+ manager.onTranscriptionComplete = (result) => {
+ completion = result;
+ };
+
+ await manager.stopStreamingRecording();
+
+ assert.equal(completion.text, expected);
+ }
+});
+
test("cancelling an active streaming recording discards it without publishing text", async (t) => {
const AudioManager = await loadManagerClass(t);
const { manager, states, getProviderStopCalls } = createFinalizingManager(AudioManager);
diff --git a/test/helpers/audioManagerStreamingRouting.test.js b/test/helpers/audioManagerStreamingRouting.test.js
index fb5dc59f1..87a7c4ee5 100644
--- a/test/helpers/audioManagerStreamingRouting.test.js
+++ b/test/helpers/audioManagerStreamingRouting.test.js
@@ -82,6 +82,7 @@ test("Gemini's live model routes onto the gemini-streaming-* channels", async (t
// Pinned to geminiLiveStreaming.js's DISCONNECT_TIMEOUT_MS: both sides
// measure the same 3s from audioStreamEnd.
assert.equal(provider.finalCeilingMs, 3000);
+ assert.equal(provider.preferStopTranscript, true);
const options = { provider: "gemini", model: "gemini-3.5-transcribe-live", mode: "byok" };
await provider.warmup(options);
diff --git a/test/helpers/geminiLiveStreaming.test.js b/test/helpers/geminiLiveStreaming.test.js
index a790ec5f8..119330f5f 100644
--- a/test/helpers/geminiLiveStreaming.test.js
+++ b/test/helpers/geminiLiveStreaming.test.js
@@ -460,6 +460,66 @@ for (const [code, reason] of [
});
}
+test("disconnect keeps a last turn the server never finalized", async () => {
+ await withServer(
+ async ({ streaming }) => {
+ await streaming.connect({ token: "t", mode: "byok" });
+ streaming.sendAudio(FRAME);
+ await new Promise((resolve) => setTimeout(resolve, 30));
+
+ assert.equal(streaming.finalize(), true);
+ streaming._audioStreamEndSentAt -= 10000;
+
+ // An earlier turn was finalized; the last one only ever had a partial.
+ assert.deepEqual(await streaming.disconnect(true), {
+ text: "Turn one. Turn two is still open",
+ });
+ },
+ (socket, message) => {
+ if (message.setup) {
+ socket.send(JSON.stringify({ setupComplete: {} }));
+ return;
+ }
+ if (!message.realtimeInput?.audio) return;
+ socket.send(JSON.stringify({ serverContent: { inputTranscription: { text: "Turn one." } } }));
+ socket.send(JSON.stringify({ serverContent: { generationComplete: true } }));
+ socket.send(
+ JSON.stringify({
+ serverContent: { interimInputTranscription: { text: "Turn two is still open" } },
+ })
+ );
+ }
+ );
+});
+
+test("a partial that restates the finalized turns adds only the open turn", async () => {
+ await withServer(
+ async ({ streaming }) => {
+ await streaming.connect({ token: "t", mode: "byok" });
+ streaming.sendAudio(FRAME);
+ await new Promise((resolve) => setTimeout(resolve, 30));
+
+ assert.equal(streaming.getFullTranscript(), "Turn one. Turn two. Turn three");
+ },
+ (socket, message) => {
+ if (message.setup) {
+ socket.send(JSON.stringify({ setupComplete: {} }));
+ return;
+ }
+ if (!message.realtimeInput?.audio) return;
+ for (const text of ["Turn one.", "Turn two."]) {
+ socket.send(JSON.stringify({ serverContent: { inputTranscription: { text } } }));
+ }
+ // Real Live sessions sometimes repeat every finalized turn, unspaced.
+ socket.send(
+ JSON.stringify({
+ serverContent: { interimInputTranscription: { text: "Turn one.Turn two.Turn three" } },
+ })
+ );
+ }
+ );
+});
+
test("errors: a spent managed token is re-minted once and the session survives", async () => {
await withServer(
async ({ streaming, urls }) => {
@@ -510,7 +570,8 @@ test("an unexpected close hands the transcript over and reports the loss once",
streaming.sendAudio(FRAME);
await new Promise((resolve) => setTimeout(resolve, 60));
- assert.deepEqual(sessionEnds, [{ text: "half a sentence" }]);
+ // The open turn's partial is handed over with the finalized text.
+ assert.deepEqual(sessionEnds, [{ text: "half a sentence and the rest" }]);
// Distinct from the pre-ready failure, and without the raw server reason.
assert.deepEqual(errors, ["Connection lost (code: 1011)"]);
assert.equal(streaming.isConnected, false);
@@ -524,6 +585,9 @@ test("an unexpected close hands the transcript over and reports the loss once",
socket.send(
JSON.stringify({ serverContent: { inputTranscription: { text: "half a sentence" } } })
);
+ socket.send(
+ JSON.stringify({ serverContent: { interimInputTranscription: { text: "and the rest" } } })
+ );
setTimeout(() => socket.close(1011, "Token has expired"), 10);
}
);