Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(transcription): drain committed Realtime turns before closing (#2767)

Henry Su6fec595ab558
diff --git a/src/helpers/openaiRealtimeStreaming.js b/src/helpers/openaiRealtimeStreaming.js
index 442408a88..79ceb03ce 100644
--- a/src/helpers/openaiRealtimeStreaming.js
+++ b/src/helpers/openaiRealtimeStreaming.js
@@ -67,6 +67,8 @@ class OpenAIRealtimeStreaming {
this.isConnected = false;
this.isConnecting = false;
this.completedSegments = [];
+ this.pendingTranscriptions = new Set();
+ this.serverAcksCommits = false;
this.currentPartial = "";
this.onPartialTranscript = null;
this.onFinalTranscript = null;
@@ -148,6 +150,8 @@ class OpenAIRealtimeStreaming {
this.streamLabel = streamLabel || null;
this.vadThreshold = vadThreshold ?? DEFAULT_VAD_THRESHOLD;
this.completedSegments = [];
+ this.pendingTranscriptions = new Set();
+ this.serverAcksCommits = false;
this.currentPartial = "";
this.audioBytesSent = 0;
this.speechStartedAt = null;
@@ -342,6 +346,7 @@ class OpenAIRealtimeStreaming {
})
);
}
+ this.pendingTranscriptions.delete(event.item_id);
this._onTurnSettled?.();
break;
}
@@ -357,6 +362,7 @@ class OpenAIRealtimeStreaming {
error: event.error?.message || event.error?.code || null,
})
);
+ this.pendingTranscriptions.delete(event.item_id);
this._onTurnSettled?.();
break;
}
@@ -366,8 +372,13 @@ class OpenAIRealtimeStreaming {
this.speechStartedCount++;
this._logVadEvent(event.type);
break;
- case "input_audio_buffer.speech_stopped":
case "input_audio_buffer.committed":
+ this.serverAcksCommits = true;
+ if (event.item_id) this.pendingTranscriptions.add(event.item_id);
+ this._onTurnSettled?.("committed");
+ this._logVadEvent(event.type);
+ break;
+ case "input_audio_buffer.speech_stopped":
this._logVadEvent(event.type);
break;
@@ -626,16 +637,26 @@ class OpenAIRealtimeStreaming {
done();
}, timeoutMs);
- // Any terminal event for the committed turn ends the wait; an empty
- // or failed turn must not stall for the whole budget.
- this._onTurnSettled = done;
+ // VAD can have several committed turns still transcribing. A terminal
+ // event ends the wait only after all known items have settled and,
+ // since an earlier turn can finish before our commit lands, after
+ // the server acks that commit. Servers that never send `committed`
+ // (Tinfoil) settle on the terminal event alone.
+ let commitAcknowledged = false;
+ this._onTurnSettled = (reason) => {
+ if (reason === "committed") commitAcknowledged = true;
+ const acknowledged = commitAcknowledged || !this.serverAcksCommits;
+ if (reason === "closed" || (acknowledged && !this.pendingTranscriptions.size)) {
+ done();
+ }
+ };
this.onError = (err) => {
if (
err?.message?.includes("buffer too small") ||
err?.message?.includes("commit_empty")
) {
- done();
+ this._onTurnSettled?.("committed");
} else {
prevOnError?.(err);
}
@@ -665,7 +686,7 @@ class OpenAIRealtimeStreaming {
this._sessionTimer = null;
this.stopKeepAlive();
// A socket that dies mid-commit can never deliver the turn.
- this._onTurnSettled?.();
+ this._onTurnSettled?.("closed");
if (this.ws) {
try {
diff --git a/test/helpers/openaiRealtimeStreaming.test.js b/test/helpers/openaiRealtimeStreaming.test.js
index c7dbd2255..6444f3f0a 100644
--- a/test/helpers/openaiRealtimeStreaming.test.js
+++ b/test/helpers/openaiRealtimeStreaming.test.js
@@ -1187,3 +1187,125 @@ test("preconfigured sessions adopt the model the server minted (session.created
assert.equal(bare.model, "gpt-4o-mini-transcribe");
bare.cleanup();
});
+
+test("stop keeps the socket open until every committed VAD turn settles", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });
+ const emit = (event) => socket.emit("message", JSON.stringify(event));
+ streaming.sendAudio(Buffer.alloc(4800, 1));
+ emit({ type: "input_audio_buffer.committed", item_id: "first" });
+ const stopping = streaming.disconnect();
+ emit({ type: "input_audio_buffer.committed", item_id: "tail", previous_item_id: "first" });
+ emit({
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "first",
+ transcript: "First sentence.",
+ });
+ await new Promise((resolve) => setImmediate(resolve));
+ try {
+ assert.equal(socket.readyState, WS.OPEN, "the earlier turn cannot close out the tail");
+ emit({
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "tail",
+ transcript: "Last sentence.",
+ });
+ assert.equal((await stopping).text, "First sentence. Last sentence.");
+ } finally {
+ streaming.cleanup();
+ await stopping;
+ }
+});
+
+test("an empty stop commit still waits for an already committed VAD transcript", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });
+ const emit = (event) => socket.emit("message", JSON.stringify(event));
+ streaming.sendAudio(Buffer.alloc(4800, 1));
+ emit({ type: "input_audio_buffer.committed", item_id: "vad-turn" });
+ const stopping = streaming.disconnect();
+ emit({
+ type: "error",
+ error: { code: "input_audio_buffer_commit_empty", message: "buffer too small" },
+ });
+ await new Promise((resolve) => setImmediate(resolve));
+ try {
+ assert.equal(socket.readyState, WS.OPEN, "an empty buffer says nothing about pending ASR");
+ emit({
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "vad-turn",
+ transcript: "Already committed speech.",
+ });
+ assert.equal((await stopping).text, "Already committed speech.");
+ } finally {
+ streaming.cleanup();
+ await stopping;
+ }
+});
+
+test("a failed VAD turn settles only itself while another transcript is pending", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });
+ const emit = (event) => socket.emit("message", JSON.stringify(event));
+ streaming.sendAudio(Buffer.alloc(4800, 1));
+ emit({ type: "input_audio_buffer.committed", item_id: "noisy" });
+ emit({ type: "input_audio_buffer.committed", item_id: "speech" });
+ const stopping = streaming.disconnect();
+ emit({
+ type: "error",
+ error: { code: "input_audio_buffer_commit_empty", message: "commit_empty" },
+ });
+ emit({
+ type: "conversation.item.input_audio_transcription.failed",
+ item_id: "noisy",
+ error: { code: "audio_unintelligible", message: "audio too noisy" },
+ });
+ await new Promise((resolve) => setImmediate(resolve));
+ try {
+ assert.equal(socket.readyState, WS.OPEN, "a failed turn cannot discard another turn");
+ emit({
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "speech",
+ transcript: "Keep this speech.",
+ });
+ assert.equal((await stopping).text, "Keep this speech.");
+ } finally {
+ streaming.cleanup();
+ await stopping;
+ }
+});
+
+test("an earlier final cannot finish stop before the trailing commit is acknowledged", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe" });
+ const emit = (event) => socket.emit("message", JSON.stringify(event));
+ streaming.sendAudio(Buffer.alloc(4800, 1));
+ emit({ type: "input_audio_buffer.committed", item_id: "earlier" });
+ const stopping = streaming.disconnect();
+ emit({
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "earlier",
+ transcript: "Earlier speech.",
+ });
+ await new Promise((resolve) => setImmediate(resolve));
+ try {
+ assert.equal(socket.readyState, WS.OPEN, "stop must still await its own commit response");
+ emit({ type: "input_audio_buffer.committed", item_id: "tail", previous_item_id: "earlier" });
+ emit({
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "tail",
+ transcript: "Trailing speech.",
+ });
+ assert.equal((await stopping).text, "Earlier speech. Trailing speech.");
+ } finally {
+ streaming.cleanup();
+ await stopping;
+ }
+});