Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(transcription): harden realtime dictation streaming connection (#1044)

Alcahest Malus Dienheim957539a1ca48
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index 169fc8c4a..7b67a7b2c 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -2648,7 +2648,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
}
// 2. Set up audio pipeline so frames flow the instant WebSocket is ready.
- // Frames sent before WebSocket connects are silently dropped by sendAudio().
+ // Frames sent before the connection is open are buffered (bounded) by
+ // sendAudio(), not dropped.
const audioContext = await this.getOrCreateAudioContext();
this.streamingAudioContext = audioContext;
this.streamingSource = audioContext.createMediaStreamSource(stream);
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 1076096fb..fa15bda33 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -5367,29 +5367,40 @@ class IPCHandlers {
const connectInner = async () => {
const isCloud = options.mode !== "byok";
- const apiKey = await fetchRealtimeToken(event, {
- mode: options.mode,
- provider: options.provider,
- });
const streaming = new OpenAIRealtimeStreaming();
setupDictationCallbacks(streaming, event);
- if (options.provider === "tinfoil-realtime") {
- const model = options.model || TINFOIL_REALTIME_MODEL;
- await streaming.connect({
- apiKey,
- model,
- // The capture worklet emits 16kHz PCM; declare the true rate.
- inputRate: 16000,
- createSocket: () => createTinfoilRealtimeSocket({ model, apiKey }),
- });
- } else {
- await streaming.connect({
- apiKey,
- model: options.model || "gpt-4o-mini-transcribe",
- preconfigured: isCloud,
+ // Assign before the token fetch (a real network round trip) so
+ // dictation-realtime-send has a live instance to buffer into instead
+ // of silently dropping the start of the recording.
+ streaming.beginConnecting();
+ this._dictationStreaming = streaming;
+ try {
+ const apiKey = await fetchRealtimeToken(event, {
+ mode: options.mode,
+ provider: options.provider,
});
+ if (options.provider === "tinfoil-realtime") {
+ const model = options.model || TINFOIL_REALTIME_MODEL;
+ await streaming.connect({
+ apiKey,
+ model,
+ // The capture worklet emits 16kHz PCM; declare the true rate.
+ inputRate: 16000,
+ createSocket: () => createTinfoilRealtimeSocket({ model, apiKey }),
+ });
+ } else {
+ await streaming.connect({
+ apiKey,
+ model: options.model || "gpt-4o-mini-transcribe",
+ // OpenAI rejects rates below 24kHz; the 16kHz capture is upsampled instead.
+ captureRate: 16000,
+ preconfigured: isCloud,
+ });
+ }
+ } catch (err) {
+ if (this._dictationStreaming === streaming) this._dictationStreaming = null;
+ throw err;
}
- this._dictationStreaming = streaming;
};
this._dictationConnectPromise = connectInner();
diff --git a/src/helpers/openaiRealtimeStreaming.js b/src/helpers/openaiRealtimeStreaming.js
index 388b036e1..e038e174b 100644
--- a/src/helpers/openaiRealtimeStreaming.js
+++ b/src/helpers/openaiRealtimeStreaming.js
@@ -5,6 +5,7 @@ const WEBSOCKET_TIMEOUT_MS = 15000;
const DISCONNECT_TIMEOUT_MS = 3000;
const SAMPLE_RATE = 24000;
const COLD_START_BUFFER_MAX = 3 * SAMPLE_RATE * 2; // 3 seconds of 16-bit PCM
+const KEEPALIVE_INTERVAL_MS = 15000;
// A socket factory does network work before the socket exists, so the dial
// must be bounded; a socket resolving after the deadline is closed, not leaked.
@@ -41,9 +42,22 @@ class OpenAIRealtimeStreaming {
this.isDisconnecting = false;
this.audioBytesSent = 0;
this.model = "gpt-4o-mini-transcribe";
+ this.inputRate = SAMPLE_RATE;
+ this.captureRate = SAMPLE_RATE;
this.coldStartBuffer = [];
this.coldStartBufferSize = 0;
this.speechStartedAt = null;
+ this.bufferingAudio = false;
+ this.keepAliveInterval = null;
+ }
+
+ // Starts buffering audio immediately, before the WebSocket even exists —
+ // covers the token-fetch + handshake window so sendAudio() doesn't drop
+ // frames while a connection is still being established.
+ beginConnecting() {
+ this.bufferingAudio = true;
+ this.coldStartBuffer = [];
+ this.coldStartBufferSize = 0;
}
getFullTranscript() {
@@ -51,7 +65,7 @@ class OpenAIRealtimeStreaming {
}
async connect(options = {}) {
- const { apiKey, model, preconfigured, inputRate, createSocket } = options;
+ const { apiKey, model, preconfigured, inputRate, captureRate, createSocket } = options;
if (!apiKey) throw new Error("OpenAI API key is required");
if (this.isConnected || this.isConnecting) {
@@ -59,15 +73,18 @@ class OpenAIRealtimeStreaming {
return;
}
+ // Callers may already be buffering (beginConnecting() called before the
+ // apiKey was fetched) — don't wipe audio collected during that window.
+ if (!this.bufferingAudio) this.beginConnecting();
+
this.isConnecting = true;
this.model = model || "gpt-4o-mini-transcribe";
this.preconfigured = !!preconfigured;
this.inputRate = inputRate || SAMPLE_RATE;
+ this.captureRate = captureRate || this.inputRate;
this.completedSegments = [];
this.currentPartial = "";
this.audioBytesSent = 0;
- this.coldStartBuffer = [];
- this.coldStartBufferSize = 0;
this.speechStartedAt = null;
const url = "wss://api.openai.com/v1/realtime?intent=transcription";
@@ -150,14 +167,7 @@ class OpenAIRealtimeStreaming {
debugLogger.debug("OpenAI Realtime session created (preconfigured)", {
model: this.model,
});
- this.isConnected = true;
- this.isConnecting = false;
- clearTimeout(this.connectionTimeout);
- if (this.pendingResolve) {
- this.pendingResolve();
- this.pendingResolve = null;
- this.pendingReject = null;
- }
+ this._markConnected();
} else {
debugLogger.debug("OpenAI Realtime session created, sending configuration", {
model: this.model,
@@ -189,15 +199,10 @@ class OpenAIRealtimeStreaming {
case "session.updated": {
if (this.pendingResolve) {
- this.isConnected = true;
- this.isConnecting = false;
- clearTimeout(this.connectionTimeout);
debugLogger.debug("OpenAI Realtime session configured", {
model: this.model,
});
- this.pendingResolve();
- this.pendingResolve = null;
- this.pendingReject = null;
+ this._markConnected();
}
break;
}
@@ -267,14 +272,80 @@ class OpenAIRealtimeStreaming {
}
}
+ _markConnected() {
+ this.isConnected = true;
+ this.isConnecting = false;
+ clearTimeout(this.connectionTimeout);
+ this.startKeepAlive();
+ if (this.pendingResolve) {
+ this.pendingResolve();
+ this.pendingResolve = null;
+ this.pendingReject = null;
+ }
+ }
+
+ // A warm connection sits idle between dictations for up to 5 minutes and is
+ // reused with no other liveness check; a network path that dies silently
+ // (NAT/firewall drop, sleep/wake, VPN toggle) leaves isConnected stuck true
+ // and the next recording gets sent into a dead socket.
+ startKeepAlive() {
+ this.stopKeepAlive();
+ const socket = this.ws;
+ if (!socket) return;
+
+ socket.isAlive = true;
+ socket.on("pong", () => {
+ socket.isAlive = true;
+ });
+
+ this.keepAliveInterval = setInterval(() => {
+ if (socket !== this.ws || socket.readyState !== WebSocket.OPEN) {
+ this.stopKeepAlive();
+ return;
+ }
+ if (socket.isAlive === false) {
+ debugLogger.debug("OpenAI Realtime keep-alive missed pong, terminating stale connection");
+ socket.terminate();
+ return;
+ }
+ socket.isAlive = false;
+ try {
+ socket.ping();
+ } catch (err) {
+ debugLogger.debug("OpenAI Realtime keep-alive ping failed", { error: err.message });
+ socket.terminate();
+ }
+ }, KEEPALIVE_INTERVAL_MS);
+ }
+
+ stopKeepAlive() {
+ if (this.keepAliveInterval) {
+ clearInterval(this.keepAliveInterval);
+ this.keepAliveInterval = null;
+ }
+ }
+
+ // OpenAI rejects PCM session rates below 24kHz, so 16kHz capture can't be
+ // declared as-is; it is upsampled to the declared rate before sending.
+ _resampleToInputRate(pcmBuffer) {
+ if (this.captureRate === this.inputRate) return pcmBuffer;
+ const src = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);
+ const ratio = this.inputRate / this.captureRate;
+ const out = new Int16Array(Math.floor(src.length * ratio));
+ for (let i = 0; i < out.length; i++) {
+ const pos = i / ratio;
+ const low = Math.floor(pos);
+ const high = Math.min(low + 1, src.length - 1);
+ out[i] = Math.round(src[low] + (src[high] - src[low]) * (pos - low));
+ }
+ return Buffer.from(out.buffer);
+ }
+
sendAudio(pcmBuffer) {
- if (!this.ws) return false;
+ const isOpen = this.ws?.readyState === WebSocket.OPEN;
- if (this.ws.readyState !== WebSocket.OPEN) {
- if (
- this.ws.readyState === WebSocket.CONNECTING &&
- this.coldStartBufferSize < COLD_START_BUFFER_MAX
- ) {
+ if (!isOpen) {
+ if (this.bufferingAudio && this.coldStartBufferSize < COLD_START_BUFFER_MAX) {
const copy = Buffer.from(pcmBuffer);
this.coldStartBuffer.push(copy);
this.coldStartBufferSize += copy.length;
@@ -288,17 +359,21 @@ class OpenAIRealtimeStreaming {
bytes: this.coldStartBufferSize,
});
for (const buf of this.coldStartBuffer) {
- const b64 = buf.toString("base64");
- this.ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: b64 }));
- this.audioBytesSent += buf.length;
+ const audio = this._resampleToInputRate(buf);
+ this.ws.send(
+ JSON.stringify({ type: "input_audio_buffer.append", audio: audio.toString("base64") })
+ );
+ this.audioBytesSent += audio.length;
}
this.coldStartBuffer = [];
this.coldStartBufferSize = 0;
}
- const base64Audio = Buffer.from(pcmBuffer).toString("base64");
- this.ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: base64Audio }));
- this.audioBytesSent += pcmBuffer.length;
+ const audio = this._resampleToInputRate(Buffer.from(pcmBuffer));
+ this.ws.send(
+ JSON.stringify({ type: "input_audio_buffer.append", audio: audio.toString("base64") })
+ );
+ this.audioBytesSent += audio.length;
return true;
}
@@ -375,6 +450,7 @@ class OpenAIRealtimeStreaming {
cleanup() {
clearTimeout(this.connectionTimeout);
this.connectionTimeout = null;
+ this.stopKeepAlive();
if (this.ws) {
try {
@@ -385,6 +461,7 @@ class OpenAIRealtimeStreaming {
this.isConnected = false;
this.isConnecting = false;
+ this.bufferingAudio = false;
}
}
diff --git a/test/helpers/openaiRealtimeStreaming.test.js b/test/helpers/openaiRealtimeStreaming.test.js
new file mode 100644
index 000000000..c003cb44f
--- /dev/null
+++ b/test/helpers/openaiRealtimeStreaming.test.js
@@ -0,0 +1,206 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { EventEmitter } = require("node:events");
+const WS = require("ws");
+
+const load = () => import("../../src/helpers/openaiRealtimeStreaming.js");
+
+function makeFakeSocket(readyState) {
+ const socket = new EventEmitter();
+ socket.readyState = readyState;
+ socket.sent = [];
+ socket.send = (data) => socket.sent.push(data);
+ socket.ping = () => {};
+ socket.terminate = () => {
+ socket.readyState = WS.CLOSED;
+ socket.emit("close", 1006, Buffer.from(""));
+ };
+ socket.close = () => {
+ socket.readyState = WS.CLOSED;
+ };
+ return socket;
+}
+
+test("sendAudio buffers frames arriving before the socket exists (token-fetch window)", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+
+ streaming.beginConnecting();
+ assert.equal(streaming.ws, null, "socket not created yet, mirrors the token-fetch window");
+
+ const sent = streaming.sendAudio(Buffer.from([1, 2, 3, 4]));
+
+ assert.equal(sent, false);
+ assert.equal(streaming.coldStartBuffer.length, 1);
+ assert.equal(streaming.coldStartBufferSize, 4);
+});
+
+test("sendAudio drops frames when no connection attempt is in flight (idle/dead instance)", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+
+ const sent = streaming.sendAudio(Buffer.from([1, 2, 3, 4]));
+
+ assert.equal(sent, false);
+ assert.equal(
+ streaming.coldStartBuffer.length,
+ 0,
+ "must not buffer forever with no connect in flight"
+ );
+});
+
+test("sendAudio stops buffering once COLD_START_BUFFER_MAX is reached", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ streaming.beginConnecting();
+
+ const chunk = Buffer.alloc(50000, 1);
+ streaming.sendAudio(chunk); // size 0 -> 50000
+ streaming.sendAudio(chunk); // size 50000 -> 100000
+ streaming.sendAudio(chunk); // size 100000 -> 150000 (still under cap when checked)
+ streaming.sendAudio(chunk); // size 150000, over the 144000 cap: dropped
+
+ assert.equal(
+ streaming.coldStartBuffer.length,
+ 3,
+ "4th chunk must be dropped once the cap is exceeded"
+ );
+ assert.equal(streaming.coldStartBufferSize, 150000);
+});
+
+test("sendAudio flushes buffered audio in order once the socket opens, then sends the live chunk", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ streaming.beginConnecting();
+
+ streaming.sendAudio(Buffer.from("first"));
+ streaming.sendAudio(Buffer.from("second"));
+
+ streaming.ws = makeFakeSocket(WS.OPEN);
+ const sent = streaming.sendAudio(Buffer.from("third"));
+
+ assert.equal(sent, true);
+ assert.equal(streaming.ws.sent.length, 3);
+ const payloads = streaming.ws.sent.map((raw) => JSON.parse(raw).audio);
+ assert.deepEqual(payloads, [
+ Buffer.from("first").toString("base64"),
+ Buffer.from("second").toString("base64"),
+ Buffer.from("third").toString("base64"),
+ ]);
+ assert.equal(streaming.coldStartBuffer.length, 0, "buffer must be cleared after flush");
+});
+
+test("connect() preserves audio buffered during beginConnecting() instead of wiping it", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+
+ streaming.beginConnecting();
+ streaming.sendAudio(Buffer.from("pre-token-fetch audio"));
+ assert.equal(streaming.coldStartBuffer.length, 1);
+
+ const socket = makeFakeSocket(WS.CONNECTING);
+ const connected = streaming.connect({
+ apiKey: "key",
+ preconfigured: true,
+ createSocket: async () => socket,
+ });
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.equal(streaming.coldStartBuffer.length, 1, "buffer must survive into connect()");
+
+ socket.readyState = WS.OPEN;
+ socket.emit("message", JSON.stringify({ type: "session.created" }));
+ await connected;
+
+ streaming.sendAudio(Buffer.from("live"));
+ const payloads = streaming.ws.sent.map((raw) => JSON.parse(raw).audio);
+ assert.deepEqual(payloads, [
+ Buffer.from("pre-token-fetch audio").toString("base64"),
+ Buffer.from("live").toString("base64"),
+ ]);
+ streaming.cleanup();
+});
+
+test("sendAudio upsamples 16kHz capture to the 24kHz session rate", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ streaming.inputRate = 24000;
+ streaming.captureRate = 16000;
+ streaming.ws = makeFakeSocket(WS.OPEN);
+
+ const pcm = new Int16Array([0, 100, 200, 300]);
+ streaming.sendAudio(Buffer.from(pcm.buffer));
+
+ const raw = Buffer.from(JSON.parse(streaming.ws.sent[0]).audio, "base64");
+ const out = new Int16Array(raw.buffer, raw.byteOffset, raw.length / 2);
+ assert.deepEqual([...out], [0, 67, 133, 200, 267, 300]);
+ assert.equal(streaming.audioBytesSent, out.length * 2);
+});
+
+test("cleanup() resets bufferingAudio so a dead instance stops buffering", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ streaming.beginConnecting();
+
+ streaming.cleanup();
+
+ assert.equal(streaming.bufferingAudio, false);
+ const sent = streaming.sendAudio(Buffer.from([1, 2, 3]));
+ assert.equal(sent, false);
+ assert.equal(streaming.coldStartBuffer.length, 0);
+});
+
+test("cleanup() stops the keep-alive interval", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ streaming.ws = makeFakeSocket(WS.OPEN);
+ streaming.startKeepAlive();
+
+ assert.notEqual(streaming.keepAliveInterval, null);
+ streaming.cleanup();
+ assert.equal(streaming.keepAliveInterval, null);
+});
+
+test("keep-alive terminates a connection that misses a pong", (t) => {
+ t.mock.timers.enable({ apis: ["setInterval"] });
+ return (async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.OPEN);
+ let terminated = false;
+ socket.terminate = () => {
+ terminated = true;
+ socket.readyState = WS.CLOSED;
+ };
+ streaming.ws = socket;
+
+ streaming.startKeepAlive();
+
+ t.mock.timers.tick(15000); // first tick: sends a ping, no pong arrives
+ assert.equal(terminated, false);
+
+ t.mock.timers.tick(15000); // second tick: no pong was received since the first ping
+ assert.equal(terminated, true);
+ })();
+});
+
+test("keep-alive stays alive when a pong is received between pings", (t) => {
+ t.mock.timers.enable({ apis: ["setInterval"] });
+ return (async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.OPEN);
+ let terminated = false;
+ socket.terminate = () => {
+ terminated = true;
+ };
+ streaming.ws = socket;
+
+ streaming.startKeepAlive();
+
+ t.mock.timers.tick(15000);
+ socket.emit("pong");
+ t.mock.timers.tick(15000);
+
+ assert.equal(terminated, false);
+ })();
+});