Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(transcription): harden realtime dictation streaming connection (#1044)
Alcahest Malus Dienheim957539a1ca48
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex 169fc8c4a..7b67a7b2c 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -2648,7 +2648,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);}// 2. Set up audio pipeline so frames flow the instant WebSocket is ready.- // Frames sent before WebSocket connects are silently dropped by sendAudio().+ // Frames sent before the connection is open are buffered (bounded) by+ // sendAudio(), not dropped.const audioContext = await this.getOrCreateAudioContext();this.streamingAudioContext = audioContext;this.streamingSource = audioContext.createMediaStreamSource(stream);diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 1076096fb..fa15bda33 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -5367,29 +5367,40 @@ class IPCHandlers {const connectInner = async () => {const isCloud = options.mode !== "byok";- const apiKey = await fetchRealtimeToken(event, {- mode: options.mode,- provider: options.provider,- });const streaming = new OpenAIRealtimeStreaming();setupDictationCallbacks(streaming, event);- if (options.provider === "tinfoil-realtime") {- const model = options.model || TINFOIL_REALTIME_MODEL;- await streaming.connect({- apiKey,- model,- // The capture worklet emits 16kHz PCM; declare the true rate.- inputRate: 16000,- createSocket: () => createTinfoilRealtimeSocket({ model, apiKey }),- });- } else {- await streaming.connect({- apiKey,- model: options.model || "gpt-4o-mini-transcribe",- preconfigured: isCloud,+ // Assign before the token fetch (a real network round trip) so+ // dictation-realtime-send has a live instance to buffer into instead+ // of silently dropping the start of the recording.+ streaming.beginConnecting();+ this._dictationStreaming = streaming;+ try {+ const apiKey = await fetchRealtimeToken(event, {+ mode: options.mode,+ provider: options.provider,});+ if (options.provider === "tinfoil-realtime") {+ const model = options.model || TINFOIL_REALTIME_MODEL;+ await streaming.connect({+ apiKey,+ model,+ // The capture worklet emits 16kHz PCM; declare the true rate.+ inputRate: 16000,+ createSocket: () => createTinfoilRealtimeSocket({ model, apiKey }),+ });+ } else {+ await streaming.connect({+ apiKey,+ model: options.model || "gpt-4o-mini-transcribe",+ // OpenAI rejects rates below 24kHz; the 16kHz capture is upsampled instead.+ captureRate: 16000,+ preconfigured: isCloud,+ });+ }+ } catch (err) {+ if (this._dictationStreaming === streaming) this._dictationStreaming = null;+ throw err;}- this._dictationStreaming = streaming;};this._dictationConnectPromise = connectInner();diff --git a/src/helpers/openaiRealtimeStreaming.js b/src/helpers/openaiRealtimeStreaming.jsindex 388b036e1..e038e174b 100644--- a/src/helpers/openaiRealtimeStreaming.js+++ b/src/helpers/openaiRealtimeStreaming.js@@ -5,6 +5,7 @@ const WEBSOCKET_TIMEOUT_MS = 15000;const DISCONNECT_TIMEOUT_MS = 3000;const SAMPLE_RATE = 24000;const COLD_START_BUFFER_MAX = 3 * SAMPLE_RATE * 2; // 3 seconds of 16-bit PCM+const KEEPALIVE_INTERVAL_MS = 15000;// A socket factory does network work before the socket exists, so the dial// must be bounded; a socket resolving after the deadline is closed, not leaked.@@ -41,9 +42,22 @@ class OpenAIRealtimeStreaming {this.isDisconnecting = false;this.audioBytesSent = 0;this.model = "gpt-4o-mini-transcribe";+ this.inputRate = SAMPLE_RATE;+ this.captureRate = SAMPLE_RATE;this.coldStartBuffer = [];this.coldStartBufferSize = 0;this.speechStartedAt = null;+ this.bufferingAudio = false;+ this.keepAliveInterval = null;+ }++ // Starts buffering audio immediately, before the WebSocket even exists —+ // covers the token-fetch + handshake window so sendAudio() doesn't drop+ // frames while a connection is still being established.+ beginConnecting() {+ this.bufferingAudio = true;+ this.coldStartBuffer = [];+ this.coldStartBufferSize = 0;}getFullTranscript() {@@ -51,7 +65,7 @@ class OpenAIRealtimeStreaming {}async connect(options = {}) {- const { apiKey, model, preconfigured, inputRate, createSocket } = options;+ const { apiKey, model, preconfigured, inputRate, captureRate, createSocket } = options;if (!apiKey) throw new Error("OpenAI API key is required");if (this.isConnected || this.isConnecting) {@@ -59,15 +73,18 @@ class OpenAIRealtimeStreaming {return;}+ // Callers may already be buffering (beginConnecting() called before the+ // apiKey was fetched) — don't wipe audio collected during that window.+ if (!this.bufferingAudio) this.beginConnecting();+this.isConnecting = true;this.model = model || "gpt-4o-mini-transcribe";this.preconfigured = !!preconfigured;this.inputRate = inputRate || SAMPLE_RATE;+ this.captureRate = captureRate || this.inputRate;this.completedSegments = [];this.currentPartial = "";this.audioBytesSent = 0;- this.coldStartBuffer = [];- this.coldStartBufferSize = 0;this.speechStartedAt = null;const url = "wss://api.openai.com/v1/realtime?intent=transcription";@@ -150,14 +167,7 @@ class OpenAIRealtimeStreaming {debugLogger.debug("OpenAI Realtime session created (preconfigured)", {model: this.model,});- this.isConnected = true;- this.isConnecting = false;- clearTimeout(this.connectionTimeout);- if (this.pendingResolve) {- this.pendingResolve();- this.pendingResolve = null;- this.pendingReject = null;- }+ this._markConnected();} else {debugLogger.debug("OpenAI Realtime session created, sending configuration", {model: this.model,@@ -189,15 +199,10 @@ class OpenAIRealtimeStreaming {case "session.updated": {if (this.pendingResolve) {- this.isConnected = true;- this.isConnecting = false;- clearTimeout(this.connectionTimeout);debugLogger.debug("OpenAI Realtime session configured", {model: this.model,});- this.pendingResolve();- this.pendingResolve = null;- this.pendingReject = null;+ this._markConnected();}break;}@@ -267,14 +272,80 @@ class OpenAIRealtimeStreaming {}}+ _markConnected() {+ this.isConnected = true;+ this.isConnecting = false;+ clearTimeout(this.connectionTimeout);+ this.startKeepAlive();+ if (this.pendingResolve) {+ this.pendingResolve();+ this.pendingResolve = null;+ this.pendingReject = null;+ }+ }++ // A warm connection sits idle between dictations for up to 5 minutes and is+ // reused with no other liveness check; a network path that dies silently+ // (NAT/firewall drop, sleep/wake, VPN toggle) leaves isConnected stuck true+ // and the next recording gets sent into a dead socket.+ startKeepAlive() {+ this.stopKeepAlive();+ const socket = this.ws;+ if (!socket) return;++ socket.isAlive = true;+ socket.on("pong", () => {+ socket.isAlive = true;+ });++ this.keepAliveInterval = setInterval(() => {+ if (socket !== this.ws || socket.readyState !== WebSocket.OPEN) {+ this.stopKeepAlive();+ return;+ }+ if (socket.isAlive === false) {+ debugLogger.debug("OpenAI Realtime keep-alive missed pong, terminating stale connection");+ socket.terminate();+ return;+ }+ socket.isAlive = false;+ try {+ socket.ping();+ } catch (err) {+ debugLogger.debug("OpenAI Realtime keep-alive ping failed", { error: err.message });+ socket.terminate();+ }+ }, KEEPALIVE_INTERVAL_MS);+ }++ stopKeepAlive() {+ if (this.keepAliveInterval) {+ clearInterval(this.keepAliveInterval);+ this.keepAliveInterval = null;+ }+ }++ // OpenAI rejects PCM session rates below 24kHz, so 16kHz capture can't be+ // declared as-is; it is upsampled to the declared rate before sending.+ _resampleToInputRate(pcmBuffer) {+ if (this.captureRate === this.inputRate) return pcmBuffer;+ const src = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);+ const ratio = this.inputRate / this.captureRate;+ const out = new Int16Array(Math.floor(src.length * ratio));+ for (let i = 0; i < out.length; i++) {+ const pos = i / ratio;+ const low = Math.floor(pos);+ const high = Math.min(low + 1, src.length - 1);+ out[i] = Math.round(src[low] + (src[high] - src[low]) * (pos - low));+ }+ return Buffer.from(out.buffer);+ }+sendAudio(pcmBuffer) {- if (!this.ws) return false;+ const isOpen = this.ws?.readyState === WebSocket.OPEN;- if (this.ws.readyState !== WebSocket.OPEN) {- if (- this.ws.readyState === WebSocket.CONNECTING &&- this.coldStartBufferSize < COLD_START_BUFFER_MAX- ) {+ if (!isOpen) {+ if (this.bufferingAudio && this.coldStartBufferSize < COLD_START_BUFFER_MAX) {const copy = Buffer.from(pcmBuffer);this.coldStartBuffer.push(copy);this.coldStartBufferSize += copy.length;@@ -288,17 +359,21 @@ class OpenAIRealtimeStreaming {bytes: this.coldStartBufferSize,});for (const buf of this.coldStartBuffer) {- const b64 = buf.toString("base64");- this.ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: b64 }));- this.audioBytesSent += buf.length;+ const audio = this._resampleToInputRate(buf);+ this.ws.send(+ JSON.stringify({ type: "input_audio_buffer.append", audio: audio.toString("base64") })+ );+ this.audioBytesSent += audio.length;}this.coldStartBuffer = [];this.coldStartBufferSize = 0;}- const base64Audio = Buffer.from(pcmBuffer).toString("base64");- this.ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: base64Audio }));- this.audioBytesSent += pcmBuffer.length;+ const audio = this._resampleToInputRate(Buffer.from(pcmBuffer));+ this.ws.send(+ JSON.stringify({ type: "input_audio_buffer.append", audio: audio.toString("base64") })+ );+ this.audioBytesSent += audio.length;return true;}@@ -375,6 +450,7 @@ class OpenAIRealtimeStreaming {cleanup() {clearTimeout(this.connectionTimeout);this.connectionTimeout = null;+ this.stopKeepAlive();if (this.ws) {try {@@ -385,6 +461,7 @@ class OpenAIRealtimeStreaming {this.isConnected = false;this.isConnecting = false;+ this.bufferingAudio = false;}}diff --git a/test/helpers/openaiRealtimeStreaming.test.js b/test/helpers/openaiRealtimeStreaming.test.jsnew file mode 100644index 000000000..c003cb44f--- /dev/null+++ b/test/helpers/openaiRealtimeStreaming.test.js@@ -0,0 +1,206 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const { EventEmitter } = require("node:events");+const WS = require("ws");++const load = () => import("../../src/helpers/openaiRealtimeStreaming.js");++function makeFakeSocket(readyState) {+ const socket = new EventEmitter();+ socket.readyState = readyState;+ socket.sent = [];+ socket.send = (data) => socket.sent.push(data);+ socket.ping = () => {};+ socket.terminate = () => {+ socket.readyState = WS.CLOSED;+ socket.emit("close", 1006, Buffer.from(""));+ };+ socket.close = () => {+ socket.readyState = WS.CLOSED;+ };+ return socket;+}++test("sendAudio buffers frames arriving before the socket exists (token-fetch window)", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();++ streaming.beginConnecting();+ assert.equal(streaming.ws, null, "socket not created yet, mirrors the token-fetch window");++ const sent = streaming.sendAudio(Buffer.from([1, 2, 3, 4]));++ assert.equal(sent, false);+ assert.equal(streaming.coldStartBuffer.length, 1);+ assert.equal(streaming.coldStartBufferSize, 4);+});++test("sendAudio drops frames when no connection attempt is in flight (idle/dead instance)", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();++ const sent = streaming.sendAudio(Buffer.from([1, 2, 3, 4]));++ assert.equal(sent, false);+ assert.equal(+ streaming.coldStartBuffer.length,+ 0,+ "must not buffer forever with no connect in flight"+ );+});++test("sendAudio stops buffering once COLD_START_BUFFER_MAX is reached", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ streaming.beginConnecting();++ const chunk = Buffer.alloc(50000, 1);+ streaming.sendAudio(chunk); // size 0 -> 50000+ streaming.sendAudio(chunk); // size 50000 -> 100000+ streaming.sendAudio(chunk); // size 100000 -> 150000 (still under cap when checked)+ streaming.sendAudio(chunk); // size 150000, over the 144000 cap: dropped++ assert.equal(+ streaming.coldStartBuffer.length,+ 3,+ "4th chunk must be dropped once the cap is exceeded"+ );+ assert.equal(streaming.coldStartBufferSize, 150000);+});++test("sendAudio flushes buffered audio in order once the socket opens, then sends the live chunk", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ streaming.beginConnecting();++ streaming.sendAudio(Buffer.from("first"));+ streaming.sendAudio(Buffer.from("second"));++ streaming.ws = makeFakeSocket(WS.OPEN);+ const sent = streaming.sendAudio(Buffer.from("third"));++ assert.equal(sent, true);+ assert.equal(streaming.ws.sent.length, 3);+ const payloads = streaming.ws.sent.map((raw) => JSON.parse(raw).audio);+ assert.deepEqual(payloads, [+ Buffer.from("first").toString("base64"),+ Buffer.from("second").toString("base64"),+ Buffer.from("third").toString("base64"),+ ]);+ assert.equal(streaming.coldStartBuffer.length, 0, "buffer must be cleared after flush");+});++test("connect() preserves audio buffered during beginConnecting() instead of wiping it", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();++ streaming.beginConnecting();+ streaming.sendAudio(Buffer.from("pre-token-fetch audio"));+ assert.equal(streaming.coldStartBuffer.length, 1);++ const socket = makeFakeSocket(WS.CONNECTING);+ const connected = streaming.connect({+ apiKey: "key",+ preconfigured: true,+ createSocket: async () => socket,+ });+ await new Promise((resolve) => setImmediate(resolve));+ assert.equal(streaming.coldStartBuffer.length, 1, "buffer must survive into connect()");++ socket.readyState = WS.OPEN;+ socket.emit("message", JSON.stringify({ type: "session.created" }));+ await connected;++ streaming.sendAudio(Buffer.from("live"));+ const payloads = streaming.ws.sent.map((raw) => JSON.parse(raw).audio);+ assert.deepEqual(payloads, [+ Buffer.from("pre-token-fetch audio").toString("base64"),+ Buffer.from("live").toString("base64"),+ ]);+ streaming.cleanup();+});++test("sendAudio upsamples 16kHz capture to the 24kHz session rate", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ streaming.inputRate = 24000;+ streaming.captureRate = 16000;+ streaming.ws = makeFakeSocket(WS.OPEN);++ const pcm = new Int16Array([0, 100, 200, 300]);+ streaming.sendAudio(Buffer.from(pcm.buffer));++ const raw = Buffer.from(JSON.parse(streaming.ws.sent[0]).audio, "base64");+ const out = new Int16Array(raw.buffer, raw.byteOffset, raw.length / 2);+ assert.deepEqual([...out], [0, 67, 133, 200, 267, 300]);+ assert.equal(streaming.audioBytesSent, out.length * 2);+});++test("cleanup() resets bufferingAudio so a dead instance stops buffering", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ streaming.beginConnecting();++ streaming.cleanup();++ assert.equal(streaming.bufferingAudio, false);+ const sent = streaming.sendAudio(Buffer.from([1, 2, 3]));+ assert.equal(sent, false);+ assert.equal(streaming.coldStartBuffer.length, 0);+});++test("cleanup() stops the keep-alive interval", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ streaming.ws = makeFakeSocket(WS.OPEN);+ streaming.startKeepAlive();++ assert.notEqual(streaming.keepAliveInterval, null);+ streaming.cleanup();+ assert.equal(streaming.keepAliveInterval, null);+});++test("keep-alive terminates a connection that misses a pong", (t) => {+ t.mock.timers.enable({ apis: ["setInterval"] });+ return (async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.OPEN);+ let terminated = false;+ socket.terminate = () => {+ terminated = true;+ socket.readyState = WS.CLOSED;+ };+ streaming.ws = socket;++ streaming.startKeepAlive();++ t.mock.timers.tick(15000); // first tick: sends a ping, no pong arrives+ assert.equal(terminated, false);++ t.mock.timers.tick(15000); // second tick: no pong was received since the first ping+ assert.equal(terminated, true);+ })();+});++test("keep-alive stays alive when a pong is received between pings", (t) => {+ t.mock.timers.enable({ apis: ["setInterval"] });+ return (async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.OPEN);+ let terminated = false;+ socket.terminate = () => {+ terminated = true;+ };+ streaming.ws = socket;++ streaming.startKeepAlive();++ t.mock.timers.tick(15000);+ socket.emit("pong");+ t.mock.timers.tick(15000);++ assert.equal(terminated, false);+ })();+});