Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(transcription): drain committed Realtime turns before closing (#2767)
Henry Su6fec595ab558
diff --git a/src/helpers/openaiRealtimeStreaming.js b/src/helpers/openaiRealtimeStreaming.jsindex 442408a88..79ceb03ce 100644--- a/src/helpers/openaiRealtimeStreaming.js+++ b/src/helpers/openaiRealtimeStreaming.js@@ -67,6 +67,8 @@ class OpenAIRealtimeStreaming {this.isConnected = false;this.isConnecting = false;this.completedSegments = [];+ this.pendingTranscriptions = new Set();+ this.serverAcksCommits = false;this.currentPartial = "";this.onPartialTranscript = null;this.onFinalTranscript = null;@@ -148,6 +150,8 @@ class OpenAIRealtimeStreaming {this.streamLabel = streamLabel || null;this.vadThreshold = vadThreshold ?? DEFAULT_VAD_THRESHOLD;this.completedSegments = [];+ this.pendingTranscriptions = new Set();+ this.serverAcksCommits = false;this.currentPartial = "";this.audioBytesSent = 0;this.speechStartedAt = null;@@ -342,6 +346,7 @@ class OpenAIRealtimeStreaming {}));}+ this.pendingTranscriptions.delete(event.item_id);this._onTurnSettled?.();break;}@@ -357,6 +362,7 @@ class OpenAIRealtimeStreaming {error: event.error?.message || event.error?.code || null,}));+ this.pendingTranscriptions.delete(event.item_id);this._onTurnSettled?.();break;}@@ -366,8 +372,13 @@ class OpenAIRealtimeStreaming {this.speechStartedCount++;this._logVadEvent(event.type);break;- case "input_audio_buffer.speech_stopped":case "input_audio_buffer.committed":+ this.serverAcksCommits = true;+ if (event.item_id) this.pendingTranscriptions.add(event.item_id);+ this._onTurnSettled?.("committed");+ this._logVadEvent(event.type);+ break;+ case "input_audio_buffer.speech_stopped":this._logVadEvent(event.type);break;@@ -626,16 +637,26 @@ class OpenAIRealtimeStreaming {done();}, timeoutMs);- // Any terminal event for the committed turn ends the wait; an empty- // or failed turn must not stall for the whole budget.- this._onTurnSettled = done;+ // VAD can have several committed turns still transcribing. A terminal+ // event ends the wait only after all known items have settled and,+ // since an earlier turn can finish before our commit lands, after+ // the server acks that commit. Servers that never send `committed`+ // (Tinfoil) settle on the terminal event alone.+ let commitAcknowledged = false;+ this._onTurnSettled = (reason) => {+ if (reason === "committed") commitAcknowledged = true;+ const acknowledged = commitAcknowledged || !this.serverAcksCommits;+ if (reason === "closed" || (acknowledged && !this.pendingTranscriptions.size)) {+ done();+ }+ };this.onError = (err) => {if (err?.message?.includes("buffer too small") ||err?.message?.includes("commit_empty")) {- done();+ this._onTurnSettled?.("committed");} else {prevOnError?.(err);}@@ -665,7 +686,7 @@ class OpenAIRealtimeStreaming {this._sessionTimer = null;this.stopKeepAlive();// A socket that dies mid-commit can never deliver the turn.- this._onTurnSettled?.();+ this._onTurnSettled?.("closed");if (this.ws) {try {diff --git a/test/helpers/openaiRealtimeStreaming.test.js b/test/helpers/openaiRealtimeStreaming.test.jsindex c7dbd2255..6444f3f0a 100644--- a/test/helpers/openaiRealtimeStreaming.test.js+++ b/test/helpers/openaiRealtimeStreaming.test.js@@ -1187,3 +1187,125 @@ test("preconfigured sessions adopt the model the server minted (session.createdassert.equal(bare.model, "gpt-4o-mini-transcribe");bare.cleanup();});++test("stop keeps the socket open until every committed VAD turn settles", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.CONNECTING);+ await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });+ const emit = (event) => socket.emit("message", JSON.stringify(event));+ streaming.sendAudio(Buffer.alloc(4800, 1));+ emit({ type: "input_audio_buffer.committed", item_id: "first" });+ const stopping = streaming.disconnect();+ emit({ type: "input_audio_buffer.committed", item_id: "tail", previous_item_id: "first" });+ emit({+ type: "conversation.item.input_audio_transcription.completed",+ item_id: "first",+ transcript: "First sentence.",+ });+ await new Promise((resolve) => setImmediate(resolve));+ try {+ assert.equal(socket.readyState, WS.OPEN, "the earlier turn cannot close out the tail");+ emit({+ type: "conversation.item.input_audio_transcription.completed",+ item_id: "tail",+ transcript: "Last sentence.",+ });+ assert.equal((await stopping).text, "First sentence. Last sentence.");+ } finally {+ streaming.cleanup();+ await stopping;+ }+});++test("an empty stop commit still waits for an already committed VAD transcript", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.CONNECTING);+ await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });+ const emit = (event) => socket.emit("message", JSON.stringify(event));+ streaming.sendAudio(Buffer.alloc(4800, 1));+ emit({ type: "input_audio_buffer.committed", item_id: "vad-turn" });+ const stopping = streaming.disconnect();+ emit({+ type: "error",+ error: { code: "input_audio_buffer_commit_empty", message: "buffer too small" },+ });+ await new Promise((resolve) => setImmediate(resolve));+ try {+ assert.equal(socket.readyState, WS.OPEN, "an empty buffer says nothing about pending ASR");+ emit({+ type: "conversation.item.input_audio_transcription.completed",+ item_id: "vad-turn",+ transcript: "Already committed speech.",+ });+ assert.equal((await stopping).text, "Already committed speech.");+ } finally {+ streaming.cleanup();+ await stopping;+ }+});++test("a failed VAD turn settles only itself while another transcript is pending", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.CONNECTING);+ await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });+ const emit = (event) => socket.emit("message", JSON.stringify(event));+ streaming.sendAudio(Buffer.alloc(4800, 1));+ emit({ type: "input_audio_buffer.committed", item_id: "noisy" });+ emit({ type: "input_audio_buffer.committed", item_id: "speech" });+ const stopping = streaming.disconnect();+ emit({+ type: "error",+ error: { code: "input_audio_buffer_commit_empty", message: "commit_empty" },+ });+ emit({+ type: "conversation.item.input_audio_transcription.failed",+ item_id: "noisy",+ error: { code: "audio_unintelligible", message: "audio too noisy" },+ });+ await new Promise((resolve) => setImmediate(resolve));+ try {+ assert.equal(socket.readyState, WS.OPEN, "a failed turn cannot discard another turn");+ emit({+ type: "conversation.item.input_audio_transcription.completed",+ item_id: "speech",+ transcript: "Keep this speech.",+ });+ assert.equal((await stopping).text, "Keep this speech.");+ } finally {+ streaming.cleanup();+ await stopping;+ }+});++test("an earlier final cannot finish stop before the trailing commit is acknowledged", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.CONNECTING);+ await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });+ const emit = (event) => socket.emit("message", JSON.stringify(event));+ streaming.sendAudio(Buffer.alloc(4800, 1));+ emit({ type: "input_audio_buffer.committed", item_id: "earlier" });+ const stopping = streaming.disconnect();+ emit({+ type: "conversation.item.input_audio_transcription.completed",+ item_id: "earlier",+ transcript: "Earlier speech.",+ });+ await new Promise((resolve) => setImmediate(resolve));+ try {+ assert.equal(socket.readyState, WS.OPEN, "stop must still await its own commit response");+ emit({ type: "input_audio_buffer.committed", item_id: "tail", previous_item_id: "earlier" });+ emit({+ type: "conversation.item.input_audio_transcription.completed",+ item_id: "tail",+ transcript: "Trailing speech.",+ });+ assert.equal((await stopping).text, "Earlier speech. Trailing speech.");+ } finally {+ streaming.cleanup();+ await stopping;+ }+});