Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix: OpenAI Realtime streaming reliability — cold-start buffer, lifecycle guards, quiet empty-buffer errors
Gabriel Steinee4baed26bf9
diff --git a/src/helpers/openaiRealtimeStreaming.js b/src/helpers/openaiRealtimeStreaming.jsindex 1a0330824..8ec1aac47 100644--- a/src/helpers/openaiRealtimeStreaming.js+++ b/src/helpers/openaiRealtimeStreaming.js@@ -3,11 +3,14 @@ const debugLogger = require("./debugLogger");const WEBSOCKET_TIMEOUT_MS = 15000;const DISCONNECT_TIMEOUT_MS = 3000;+const SAMPLE_RATE = 24000;+const COLD_START_BUFFER_MAX = 3 * SAMPLE_RATE * 2; // 3 seconds of 16-bit PCMclass OpenAIRealtimeStreaming {constructor() {this.ws = null;this.isConnected = false;+ this.isConnecting = false;this.completedSegments = [];this.currentPartial = "";this.onPartialTranscript = null;@@ -21,6 +24,8 @@ class OpenAIRealtimeStreaming {this.isDisconnecting = false;this.audioBytesSent = 0;this.model = "gpt-4o-mini-transcribe";+ this.coldStartBuffer = [];+ this.coldStartBufferSize = 0;}getFullTranscript() {@@ -31,16 +36,19 @@ class OpenAIRealtimeStreaming {const { apiKey, model, preconfigured } = options;if (!apiKey) throw new Error("OpenAI API key is required");- if (this.isConnected) {- debugLogger.debug("OpenAI Realtime already connected");+ if (this.isConnected || this.isConnecting) {+ debugLogger.debug("OpenAI Realtime already connected/connecting");return;}+ this.isConnecting = true;this.model = model || "gpt-4o-mini-transcribe";this.preconfigured = !!preconfigured;this.completedSegments = [];this.currentPartial = "";this.audioBytesSent = 0;+ this.coldStartBuffer = [];+ this.coldStartBufferSize = 0;const url = "wss://api.openai.com/v1/realtime?intent=transcription";debugLogger.debug("OpenAI Realtime connecting", { model: this.model });@@ -50,6 +58,7 @@ class OpenAIRealtimeStreaming {this.pendingReject = reject;this.connectionTimeout = setTimeout(() => {+ this.isConnecting = false;this.cleanup();reject(new Error("OpenAI Realtime connection timeout"));}, WEBSOCKET_TIMEOUT_MS);@@ -71,6 +80,7 @@ class OpenAIRealtimeStreaming {this.ws.on("error", (error) => {debugLogger.error("OpenAI Realtime WebSocket error", { error: error.message });+ this.isConnecting = false;this.cleanup();if (this.pendingReject) {this.pendingReject(error);@@ -82,6 +92,7 @@ class OpenAIRealtimeStreaming {this.ws.on("close", (code, reason) => {const wasActive = this.isConnected;+ this.isConnecting = false;debugLogger.debug("OpenAI Realtime WebSocket closed", {code,reason: reason?.toString(),@@ -116,6 +127,7 @@ class OpenAIRealtimeStreaming {model: this.model,});this.isConnected = true;+ this.isConnecting = false;clearTimeout(this.connectionTimeout);if (this.pendingResolve) {this.pendingResolve();@@ -126,6 +138,7 @@ class OpenAIRealtimeStreaming {debugLogger.debug("OpenAI Realtime session created, sending configuration", {model: this.model,});+ if (!this.ws || this.ws.readyState !== WebSocket.OPEN) break;this.ws.send(JSON.stringify({type: "transcription_session.update",@@ -150,6 +163,7 @@ class OpenAIRealtimeStreaming {case "transcription_session.updated": {if (this.pendingResolve) {this.isConnected = true;+ this.isConnecting = false;clearTimeout(this.connectionTimeout);debugLogger.debug("OpenAI Realtime session configured", {model: this.model,@@ -188,13 +202,26 @@ class OpenAIRealtimeStreaming {case "input_audio_buffer.committed":break;- case "error":- debugLogger.error("OpenAI Realtime error event", {- code: event.error?.code,- message: event.error?.message,- });- this.onError?.(new Error(event.error?.message || "OpenAI Realtime error"));+ case "error": {+ const errCode = event.error?.code;+ const errMsg = event.error?.message || "OpenAI Realtime error";+ const isEmptyBuffer =+ errCode === "input_audio_buffer_commit_empty" ||+ errMsg.includes("buffer too small") ||+ errMsg.includes("commit_empty");+ if (isEmptyBuffer) {+ debugLogger.debug("OpenAI Realtime empty buffer (server VAD already committed)", {+ code: errCode,+ });+ } else {+ debugLogger.error("OpenAI Realtime error event", {+ code: errCode,+ message: errMsg,+ });+ }+ this.onError?.(new Error(errMsg));break;+ }default:break;@@ -205,10 +232,34 @@ class OpenAIRealtimeStreaming {}sendAudio(pcmBuffer) {- if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {+ if (!this.ws) return false;++ if (this.ws.readyState !== WebSocket.OPEN) {+ if (+ this.ws.readyState === WebSocket.CONNECTING &&+ this.coldStartBufferSize < COLD_START_BUFFER_MAX+ ) {+ const copy = Buffer.from(pcmBuffer);+ this.coldStartBuffer.push(copy);+ this.coldStartBufferSize += copy.length;+ }return false;}+ if (this.coldStartBuffer.length > 0) {+ debugLogger.debug("OpenAI Realtime flushing cold-start buffer", {+ chunks: this.coldStartBuffer.length,+ bytes: this.coldStartBufferSize,+ });+ for (const buf of this.coldStartBuffer) {+ const b64 = buf.toString("base64");+ this.ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: b64 }));+ this.audioBytesSent += buf.length;+ }+ this.coldStartBuffer = [];+ this.coldStartBufferSize = 0;+ }+const base64Audio = Buffer.from(pcmBuffer).toString("base64");this.ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: base64Audio }));this.audioBytesSent += pcmBuffer.length;@@ -220,12 +271,20 @@ class OpenAIRealtimeStreaming {audioBytesSent: this.audioBytesSent,segments: this.completedSegments.length,textLength: this.getFullTranscript().length,+ readyState: this.ws?.readyState,});if (!this.ws) return { text: this.getFullTranscript() };this.isDisconnecting = true;+ if (this.ws.readyState === WebSocket.CONNECTING) {+ this.ws.once("open", () => this.ws?.close());+ const result = { text: this.getFullTranscript() };+ this.isDisconnecting = false;+ return result;+ }+if (this.ws.readyState === WebSocket.OPEN) {if (this.audioBytesSent > 0) {const prevOnFinal = this.onFinalTranscript;@@ -249,8 +308,6 @@ class OpenAIRealtimeStreaming {done();};- // Server VAD may have already committed all audio, causing an- // empty-buffer error. Resolve immediately instead of waiting for timeout.this.onError = (err) => {if (err?.message?.includes("buffer too small") ||@@ -286,13 +343,12 @@ class OpenAIRealtimeStreaming {if (this.ws) {try {this.ws.close();- } catch (err) {- // ignore- }+ } catch {}this.ws = null;}this.isConnected = false;+ this.isConnecting = false;this.closeResolve = null;}}