Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix: OpenAI Realtime streaming reliability — cold-start buffer, lifecycle guards, quiet empty-buffer errors

Gabriel Steinee4baed26bf9
diff --git a/src/helpers/openaiRealtimeStreaming.js b/src/helpers/openaiRealtimeStreaming.js
index 1a0330824..8ec1aac47 100644
--- a/src/helpers/openaiRealtimeStreaming.js
+++ b/src/helpers/openaiRealtimeStreaming.js
@@ -3,11 +3,14 @@ const debugLogger = require("./debugLogger");
const WEBSOCKET_TIMEOUT_MS = 15000;
const DISCONNECT_TIMEOUT_MS = 3000;
+const SAMPLE_RATE = 24000;
+const COLD_START_BUFFER_MAX = 3 * SAMPLE_RATE * 2; // 3 seconds of 16-bit PCM
class OpenAIRealtimeStreaming {
constructor() {
this.ws = null;
this.isConnected = false;
+ this.isConnecting = false;
this.completedSegments = [];
this.currentPartial = "";
this.onPartialTranscript = null;
@@ -21,6 +24,8 @@ class OpenAIRealtimeStreaming {
this.isDisconnecting = false;
this.audioBytesSent = 0;
this.model = "gpt-4o-mini-transcribe";
+ this.coldStartBuffer = [];
+ this.coldStartBufferSize = 0;
}
getFullTranscript() {
@@ -31,16 +36,19 @@ class OpenAIRealtimeStreaming {
const { apiKey, model, preconfigured } = options;
if (!apiKey) throw new Error("OpenAI API key is required");
- if (this.isConnected) {
- debugLogger.debug("OpenAI Realtime already connected");
+ if (this.isConnected || this.isConnecting) {
+ debugLogger.debug("OpenAI Realtime already connected/connecting");
return;
}
+ this.isConnecting = true;
this.model = model || "gpt-4o-mini-transcribe";
this.preconfigured = !!preconfigured;
this.completedSegments = [];
this.currentPartial = "";
this.audioBytesSent = 0;
+ this.coldStartBuffer = [];
+ this.coldStartBufferSize = 0;
const url = "wss://api.openai.com/v1/realtime?intent=transcription";
debugLogger.debug("OpenAI Realtime connecting", { model: this.model });
@@ -50,6 +58,7 @@ class OpenAIRealtimeStreaming {
this.pendingReject = reject;
this.connectionTimeout = setTimeout(() => {
+ this.isConnecting = false;
this.cleanup();
reject(new Error("OpenAI Realtime connection timeout"));
}, WEBSOCKET_TIMEOUT_MS);
@@ -71,6 +80,7 @@ class OpenAIRealtimeStreaming {
this.ws.on("error", (error) => {
debugLogger.error("OpenAI Realtime WebSocket error", { error: error.message });
+ this.isConnecting = false;
this.cleanup();
if (this.pendingReject) {
this.pendingReject(error);
@@ -82,6 +92,7 @@ class OpenAIRealtimeStreaming {
this.ws.on("close", (code, reason) => {
const wasActive = this.isConnected;
+ this.isConnecting = false;
debugLogger.debug("OpenAI Realtime WebSocket closed", {
code,
reason: reason?.toString(),
@@ -116,6 +127,7 @@ class OpenAIRealtimeStreaming {
model: this.model,
});
this.isConnected = true;
+ this.isConnecting = false;
clearTimeout(this.connectionTimeout);
if (this.pendingResolve) {
this.pendingResolve();
@@ -126,6 +138,7 @@ class OpenAIRealtimeStreaming {
debugLogger.debug("OpenAI Realtime session created, sending configuration", {
model: this.model,
});
+ if (!this.ws || this.ws.readyState !== WebSocket.OPEN) break;
this.ws.send(
JSON.stringify({
type: "transcription_session.update",
@@ -150,6 +163,7 @@ class OpenAIRealtimeStreaming {
case "transcription_session.updated": {
if (this.pendingResolve) {
this.isConnected = true;
+ this.isConnecting = false;
clearTimeout(this.connectionTimeout);
debugLogger.debug("OpenAI Realtime session configured", {
model: this.model,
@@ -188,13 +202,26 @@ class OpenAIRealtimeStreaming {
case "input_audio_buffer.committed":
break;
- case "error":
- debugLogger.error("OpenAI Realtime error event", {
- code: event.error?.code,
- message: event.error?.message,
- });
- this.onError?.(new Error(event.error?.message || "OpenAI Realtime error"));
+ case "error": {
+ const errCode = event.error?.code;
+ const errMsg = event.error?.message || "OpenAI Realtime error";
+ const isEmptyBuffer =
+ errCode === "input_audio_buffer_commit_empty" ||
+ errMsg.includes("buffer too small") ||
+ errMsg.includes("commit_empty");
+ if (isEmptyBuffer) {
+ debugLogger.debug("OpenAI Realtime empty buffer (server VAD already committed)", {
+ code: errCode,
+ });
+ } else {
+ debugLogger.error("OpenAI Realtime error event", {
+ code: errCode,
+ message: errMsg,
+ });
+ }
+ this.onError?.(new Error(errMsg));
break;
+ }
default:
break;
@@ -205,10 +232,34 @@ class OpenAIRealtimeStreaming {
}
sendAudio(pcmBuffer) {
- if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+ if (!this.ws) return false;
+
+ if (this.ws.readyState !== WebSocket.OPEN) {
+ if (
+ this.ws.readyState === WebSocket.CONNECTING &&
+ this.coldStartBufferSize < COLD_START_BUFFER_MAX
+ ) {
+ const copy = Buffer.from(pcmBuffer);
+ this.coldStartBuffer.push(copy);
+ this.coldStartBufferSize += copy.length;
+ }
return false;
}
+ if (this.coldStartBuffer.length > 0) {
+ debugLogger.debug("OpenAI Realtime flushing cold-start buffer", {
+ chunks: this.coldStartBuffer.length,
+ bytes: this.coldStartBufferSize,
+ });
+ for (const buf of this.coldStartBuffer) {
+ const b64 = buf.toString("base64");
+ this.ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: b64 }));
+ this.audioBytesSent += buf.length;
+ }
+ this.coldStartBuffer = [];
+ this.coldStartBufferSize = 0;
+ }
+
const base64Audio = Buffer.from(pcmBuffer).toString("base64");
this.ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: base64Audio }));
this.audioBytesSent += pcmBuffer.length;
@@ -220,12 +271,20 @@ class OpenAIRealtimeStreaming {
audioBytesSent: this.audioBytesSent,
segments: this.completedSegments.length,
textLength: this.getFullTranscript().length,
+ readyState: this.ws?.readyState,
});
if (!this.ws) return { text: this.getFullTranscript() };
this.isDisconnecting = true;
+ if (this.ws.readyState === WebSocket.CONNECTING) {
+ this.ws.once("open", () => this.ws?.close());
+ const result = { text: this.getFullTranscript() };
+ this.isDisconnecting = false;
+ return result;
+ }
+
if (this.ws.readyState === WebSocket.OPEN) {
if (this.audioBytesSent > 0) {
const prevOnFinal = this.onFinalTranscript;
@@ -249,8 +308,6 @@ class OpenAIRealtimeStreaming {
done();
};
- // Server VAD may have already committed all audio, causing an
- // empty-buffer error. Resolve immediately instead of waiting for timeout.
this.onError = (err) => {
if (
err?.message?.includes("buffer too small") ||
@@ -286,13 +343,12 @@ class OpenAIRealtimeStreaming {
if (this.ws) {
try {
this.ws.close();
- } catch (err) {
- // ignore
- }
+ } catch {}
this.ws = null;
}
this.isConnected = false;
+ this.isConnecting = false;
this.closeResolve = null;
}
}