Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

feat(transcription): support gpt-live-transcribe sessions in the OpenAI realtime client (#2077)

Gabriel Steindff92c04c1de
diff --git a/src/helpers/openaiRealtimeStreaming.js b/src/helpers/openaiRealtimeStreaming.js
index 58920f40e..b4468bd60 100644
--- a/src/helpers/openaiRealtimeStreaming.js
+++ b/src/helpers/openaiRealtimeStreaming.js
@@ -3,6 +3,11 @@ const debugLogger = require("./debugLogger");
const WEBSOCKET_TIMEOUT_MS = 15000;
const DISCONNECT_TIMEOUT_MS = 3000;
+// gpt-live-transcribe acks a commit only once every appended byte is
+// transcribed, and it ran at 0.3–0.8x real time in testing, so its final trails
+// the stop by a large fraction of the dictation length (partials keep streaming
+// meanwhile).
+const LIVE_TRANSCRIBE_COMMIT_TIMEOUT_MS = 30000;
const SAMPLE_RATE = 24000;
const COLD_START_BUFFER_MAX = 3 * SAMPLE_RATE * 2; // 3 seconds of 16-bit PCM
const KEEPALIVE_INTERVAL_MS = 15000;
@@ -12,6 +17,23 @@ const SESSION_PREEMPT_MS = 55 * 60 * 1000;
// on a cleaner channel pass their own vadThreshold.
const DEFAULT_VAD_THRESHOLD = 0.6;
+// Matches the server's check, which may pin a snapshot id.
+const isLiveTranscribe = (model) => model.startsWith("gpt-live-transcribe");
+
+// gpt-live-transcribe rejects every turn_detection config ("Turn detection is
+// not supported for this transcription model"), so it never emits
+// speech_started/speech_stopped and a turn only completes once the client
+// commits — which disconnect() does on stop. Legacy models keep server VAD.
+const turnDetectionFor = (model, vadThreshold) =>
+ isLiveTranscribe(model)
+ ? null
+ : {
+ type: "server_vad",
+ threshold: vadThreshold,
+ silence_duration_ms: 600,
+ prefix_padding_ms: 500,
+ };
+
// A socket factory does network work before the socket exists, so the dial
// must be bounded; a socket resolving after the deadline is closed, not leaked.
async function createSocketWithTimeout(createSocket, timeoutMs) {
@@ -68,6 +90,7 @@ class OpenAIRealtimeStreaming {
this.speechStartedAt = null;
this.bufferingAudio = false;
this.keepAliveInterval = null;
+ this._onTurnSettled = null;
}
// Starts buffering audio immediately, before the WebSocket even exists —
@@ -213,6 +236,9 @@ class OpenAIRealtimeStreaming {
// Echo the server's VAD + format — the only place preconfigured
// (cloud) session settings ever appear in field logs.
const sessionInput = event.session?.audio?.input;
+ // The server picks the managed model when it mints the secret; the
+ // session echo is where the client learns which one it got.
+ this.model = sessionInput?.transcription?.model ?? this.model;
debugLogger.debug(
`${this.providerLabel} session created (preconfigured)`,
this._logContext({
@@ -241,12 +267,7 @@ class OpenAIRealtimeStreaming {
input: {
format: { type: "audio/pcm", rate: this.inputRate },
transcription: { model: this.model },
- turn_detection: {
- type: "server_vad",
- threshold: this.vadThreshold,
- silence_duration_ms: 600,
- prefix_padding_ms: 500,
- },
+ turn_detection: turnDetectionFor(this.model, this.vadThreshold),
},
},
},
@@ -306,6 +327,7 @@ class OpenAIRealtimeStreaming {
})
);
}
+ this._onTurnSettled?.();
break;
}
@@ -320,6 +342,7 @@ class OpenAIRealtimeStreaming {
error: event.error?.message || event.error?.code || null,
})
);
+ this._onTurnSettled?.();
break;
}
@@ -567,29 +590,30 @@ class OpenAIRealtimeStreaming {
if (this.ws.readyState === WebSocket.OPEN) {
if (commit && this.audioBytesSent > 0) {
- const prevOnFinal = this.onFinalTranscript;
const prevOnError = this.onError;
+ const timeoutMs = isLiveTranscribe(this.model)
+ ? LIVE_TRANSCRIBE_COMMIT_TIMEOUT_MS
+ : DISCONNECT_TIMEOUT_MS;
await new Promise((resolve) => {
- const tid = setTimeout(() => {
- debugLogger.debug(
- `${this.providerLabel} commit timeout, using accumulated text`,
- this._logContext()
- );
- resolve();
- }, DISCONNECT_TIMEOUT_MS);
-
const done = () => {
clearTimeout(tid);
- this.onFinalTranscript = prevOnFinal;
+ this._onTurnSettled = null;
this.onError = prevOnError;
resolve();
};
- this.onFinalTranscript = (text) => {
- prevOnFinal?.(text);
+ const tid = setTimeout(() => {
+ debugLogger.debug(
+ `${this.providerLabel} commit timeout, using accumulated text`,
+ this._logContext()
+ );
done();
- };
+ }, timeoutMs);
+
+ // Any terminal event for the committed turn ends the wait; an empty
+ // or failed turn must not stall for the whole budget.
+ this._onTurnSettled = done;
this.onError = (err) => {
if (
@@ -610,7 +634,7 @@ class OpenAIRealtimeStreaming {
});
}
- this.ws.close();
+ this.ws?.close();
}
const result = { text: this.getFullTranscript() };
@@ -625,6 +649,8 @@ class OpenAIRealtimeStreaming {
clearTimeout(this._sessionTimer);
this._sessionTimer = null;
this.stopKeepAlive();
+ // A socket that dies mid-commit can never deliver the turn.
+ this._onTurnSettled?.();
if (this.ws) {
try {
diff --git a/test/helpers/meetingTranscriptionRouting.test.js b/test/helpers/meetingTranscriptionRouting.test.js
index ce9063e83..fb9226bca 100644
--- a/test/helpers/meetingTranscriptionRouting.test.js
+++ b/test/helpers/meetingTranscriptionRouting.test.js
@@ -4,6 +4,7 @@ const fs = require("node:fs");
const path = require("node:path");
const load = () => import("../../src/helpers/meetingTranscriptionRouting.js");
+const modelRegistryData = require("../../src/models/modelRegistryData.json");
// Sentinels this module throws, paired with the MEETING_ERROR_KEYS entry that
// MeetingRecordingMount looks up. A healed legacy profile can reach any of them,
@@ -227,3 +228,45 @@ test("every thrown sentinel is translated and rendered by the mount", () => {
// have somewhere to put it.
assert.match(translation.notes.meeting.unsupportedProvider, /\{\{provider\}\}/);
});
+
+// gpt-live-transcribe has no server VAD and only completes a turn when the client
+// commits, which dictation does on stop and a long-running meeting stream never
+// does. Until that guard exists, the desktop must not be able to route a meeting
+// onto it: no registry entry, and a stale selection falls back to the default.
+test("gpt-live-transcribe is never offered for Note Recording", async () => {
+ const { resolveMeetingTranscriptionOptions } = await load();
+ // Mirrors getStreamingTranscriptionProviders(): the meeting BYOK picker.
+ const registryStreamingProviders = modelRegistryData.transcriptionProviders
+ .map((provider) => ({ ...provider, models: provider.models.filter((m) => m.streaming) }))
+ .filter((provider) => provider.models.length > 0);
+ for (const provider of registryStreamingProviders) {
+ for (const model of provider.models) {
+ assert.equal(model.id.startsWith("gpt-live-transcribe"), false, model.id);
+ }
+ }
+
+ assert.equal(
+ resolveMeetingTranscriptionOptions({
+ ...baseOptions,
+ selectedProvider: "openai",
+ selectedModel: "gpt-live-transcribe",
+ byokProviders: registryStreamingProviders,
+ }).model,
+ "gpt-4o-mini-transcribe"
+ );
+ const managedCatalogs = [
+ null,
+ [{ id: "openai", models: [{ id: "gpt-4o-mini-transcribe", default: true }] }],
+ ];
+ for (const managedProviders of managedCatalogs) {
+ assert.equal(
+ resolveMeetingTranscriptionOptions({
+ ...baseOptions,
+ transcriptionMode: "openwhispr",
+ managedProviders,
+ selectedModel: "gpt-live-transcribe",
+ }).model,
+ "gpt-4o-mini-transcribe"
+ );
+ }
+});
diff --git a/test/helpers/openaiRealtimeStreaming.test.js b/test/helpers/openaiRealtimeStreaming.test.js
index 6852825c6..c97bef9bf 100644
--- a/test/helpers/openaiRealtimeStreaming.test.js
+++ b/test/helpers/openaiRealtimeStreaming.test.js
@@ -33,6 +33,22 @@ async function connectPreconfigured(streaming, socket) {
await connected;
}
+const sentEvents = (socket, type) =>
+ socket.sent.map((raw) => JSON.parse(raw)).filter((e) => e.type === type);
+
+async function connectByok(streaming, socket, options) {
+ const connected = streaming.connect({
+ apiKey: "sk-test",
+ createSocket: async () => socket,
+ ...options,
+ });
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.readyState = WS.OPEN;
+ socket.emit("message", JSON.stringify({ type: "session.created" }));
+ socket.emit("message", JSON.stringify({ type: "session.updated" }));
+ await connected;
+}
+
function captureLogs(t) {
const debugLogger = require("../../src/helpers/debugLogger");
const entries = [];
@@ -661,7 +677,7 @@ test("dictation-style connect (inputRate 16000, custom socket factory) declares
streaming.cleanup();
});
-test("characterization: a final that lands during disconnect()'s commit window reaches onFinalTranscript WITHOUT its timestamp (Phase 1 forwards it)", async () => {
+test("a final that lands during disconnect()'s commit window reaches onFinalTranscript with its timestamp", async () => {
const OpenAIRealtimeStreaming = (await load()).default;
const streaming = new OpenAIRealtimeStreaming();
const socket = makeFakeSocket(WS.CONNECTING);
@@ -693,10 +709,7 @@ test("characterization: a final that lands during disconnect()'s commit window r
assert.equal(result.text, "tail words", "the tail is returned to the caller");
assert.equal(calls.length, 1, "the tail also reaches the live onFinalTranscript handler");
assert.equal(calls[0].text, "tail words");
- // TODAY the temporary onFinalTranscript wrapper that disconnect() installs while
- // awaiting the commit forwards only `text` to the previous handler.
- // Phase 1 changes this line to `typeof calls[0].timestamp === "number"`.
- assert.equal(calls[0].timestamp, undefined);
+ assert.equal(typeof calls[0].timestamp, "number");
});
// -- diagnostic logging: stream labels, VAD events, failed/empty turns --
@@ -867,3 +880,254 @@ test("without a streamLabel (dictation) log metadata carries no stream key", asy
assert.ok(line);
assert.equal("stream" in line.meta, false, "unlabelled sockets keep today's metadata shape");
});
+
+// -- gpt-live-transcribe: no server VAD, the client commits the turn on stop --
+
+const LIVE_UPDATE_BODY = {
+ type: "session.update",
+ session: {
+ type: "transcription",
+ audio: {
+ input: {
+ format: { type: "audio/pcm", rate: 24000 },
+ transcription: { model: "gpt-live-transcribe" },
+ turn_detection: null,
+ },
+ },
+ },
+};
+
+const completedEvent = (transcript) =>
+ JSON.stringify({
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "item_1",
+ transcript,
+ });
+
+test("gpt-live-transcribe session.update sends turn_detection: null (any VAD config is rejected); the rest is the legacy payload", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model: "gpt-live-transcribe", vadThreshold: 0.3 });
+
+ const updates = sentEvents(socket, "session.update");
+ assert.equal(updates.length, 1);
+ assert.deepEqual(updates[0], LIVE_UPDATE_BODY);
+ assert.ok(socket.sent[0].includes('"turn_detection":null'), "null is on the wire, not dropped");
+ streaming.cleanup();
+});
+
+test("legacy models keep the server_vad block byte-for-byte", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ for (const model of ["gpt-4o-mini-transcribe", "gpt-4o-transcribe"]) {
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model });
+
+ const [update] = sentEvents(socket, "session.update");
+ assert.equal(update.session.audio.input.transcription.model, model);
+ assert.deepEqual(
+ update.session.audio.input.turn_detection,
+ { type: "server_vad", threshold: 0.6, silence_duration_ms: 600, prefix_padding_ms: 500 },
+ model
+ );
+ streaming.cleanup();
+ }
+});
+
+test("gpt-live-transcribe stop: disconnect() commits once, then consumes the .completed final it triggered", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model: "gpt-live-transcribe" });
+ const partials = [];
+ const finals = [];
+ streaming.onPartialTranscript = (text) => partials.push(text);
+ streaming.onFinalTranscript = (text, timestamp) => finals.push({ text, timestamp });
+
+ // Partials stream during speech with no speech_started/speech_stopped around them.
+ for (const delta of ["Hello", " world"]) {
+ socket.emit(
+ "message",
+ JSON.stringify({ type: "conversation.item.input_audio_transcription.delta", delta })
+ );
+ }
+ streaming.sendAudio(Buffer.alloc(4800, 1));
+
+ const disconnecting = streaming.disconnect();
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 1);
+ assert.equal(finals.length, 0, "the final only exists once the server answers the commit");
+
+ socket.emit(
+ "message",
+ JSON.stringify({ type: "input_audio_buffer.committed", item_id: "item_1" })
+ );
+ socket.emit("message", completedEvent("Hello world."));
+ const result = await disconnecting;
+
+ assert.deepEqual(partials, ["Hello", "Hello world"]);
+ assert.equal(result.text, "Hello world.");
+ assert.deepEqual(
+ finals.map((f) => f.text),
+ ["Hello world."]
+ );
+ assert.equal(typeof finals[0].timestamp, "number", "timestamp falls back without speech_started");
+ assert.equal(streaming.speechStartedCount, 0);
+ assert.equal(
+ sentEvents(socket, "input_audio_buffer.commit").length,
+ 1,
+ "still exactly one commit"
+ );
+ assert.equal(socket.readyState, WS.CLOSED);
+});
+
+test("gpt-live-transcribe stop with no audio sent never commits (an empty commit is a server error)", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model: "gpt-live-transcribe" });
+
+ const result = await streaming.disconnect();
+
+ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 0);
+ assert.equal(result.text, "");
+ assert.equal(socket.readyState, WS.CLOSED);
+});
+
+test("gpt-live-transcribe stop outlives the legacy 3 s commit budget; legacy keeps it", (t) => {
+ t.mock.timers.enable({ apis: ["setTimeout"] });
+ return (async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const settle = async () => new Promise((resolve) => setImmediate(resolve));
+
+ const live = new OpenAIRealtimeStreaming();
+ const liveSocket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(live, liveSocket, { model: "gpt-live-transcribe" });
+ live.sendAudio(Buffer.alloc(4800, 1));
+ let liveSettled = false;
+ const liveDisconnect = live.disconnect().then((result) => {
+ liveSettled = true;
+ return result;
+ });
+ await settle();
+ t.mock.timers.tick(3000);
+ await settle();
+ assert.equal(liveSettled, false, "a long dictation's final trails stop by more than 3 s");
+ liveSocket.emit("message", completedEvent("late tail"));
+ assert.equal((await liveDisconnect).text, "late tail");
+
+ const legacy = new OpenAIRealtimeStreaming();
+ const legacySocket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(legacy, legacySocket, { model: "gpt-4o-mini-transcribe" });
+ legacy.sendAudio(Buffer.alloc(4800, 1));
+ let legacySettled = false;
+ const legacyDisconnect = legacy.disconnect().then(() => {
+ legacySettled = true;
+ });
+ await settle();
+ t.mock.timers.tick(3000);
+ await legacyDisconnect;
+ assert.equal(legacySettled, true);
+ })();
+});
+
+test("an empty or failed committed turn ends the stop wait at once instead of stalling for the budget", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const terminalEvents = [
+ completedEvent(" "),
+ JSON.stringify({
+ type: "conversation.item.input_audio_transcription.failed",
+ item_id: "item_1",
+ error: { code: "audio_unintelligible", message: "audio too noisy" },
+ }),
+ ];
+ for (const terminal of terminalEvents) {
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model: "gpt-live-transcribe" });
+ let finalCalled = false;
+ streaming.onFinalTranscript = () => {
+ finalCalled = true;
+ };
+ streaming.sendAudio(Buffer.alloc(4800, 1));
+
+ const disconnecting = streaming.disconnect();
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.emit("message", terminal);
+ const result = await Promise.race([
+ disconnecting,
+ new Promise((_, reject) => setTimeout(() => reject(new Error("stop stalled")), 500)),
+ ]);
+
+ assert.equal(result.text, "");
+ assert.equal(finalCalled, false);
+ assert.equal(streaming._onTurnSettled, null, "hook is released once the wait ends");
+ }
+});
+
+test("a socket that closes during the commit wait ends the stop at once with the accumulated text", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model: "gpt-live-transcribe" });
+ streaming.sendAudio(Buffer.alloc(4800, 1));
+
+ const disconnecting = streaming.disconnect();
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.readyState = WS.CLOSED;
+ socket.emit("close", 1006, Buffer.from(""));
+ const result = await Promise.race([
+ disconnecting,
+ new Promise((_, reject) => setTimeout(() => reject(new Error("stop stalled")), 500)),
+ ]);
+
+ assert.equal(result.text, "");
+ assert.equal(streaming.ws, null);
+});
+
+test("preconfigured sessions adopt the model the server minted (session.created echo) and never send session.update", async (t) => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const logs = captureLogs(t);
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+
+ const connected = streaming.connect({
+ apiKey: "cs-secret",
+ preconfigured: true,
+ // The desktop's BYOK default; the server decides the managed model.
+ model: "gpt-4o-mini-transcribe",
+ createSocket: async () => socket,
+ });
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.readyState = WS.OPEN;
+ socket.emit(
+ "message",
+ JSON.stringify({
+ type: "session.created",
+ session: {
+ audio: {
+ input: {
+ format: { type: "audio/pcm", rate: 24000 },
+ transcription: { model: "gpt-live-transcribe", language: null },
+ turn_detection: null,
+ },
+ },
+ },
+ })
+ );
+ await connected;
+
+ assert.equal(streaming.model, "gpt-live-transcribe");
+ assert.equal(sentEvents(socket, "session.update").length, 0);
+ const line = logs.find((entry) => entry.message.includes("session created (preconfigured)"));
+ assert.equal(line.meta.model, "gpt-live-transcribe");
+ assert.equal(line.meta.turnDetection, null);
+ streaming.cleanup();
+
+ // No echo (older servers, bare mocks): the caller's model stands.
+ const bare = new OpenAIRealtimeStreaming();
+ await connectPreconfigured(bare, makeFakeSocket(WS.CONNECTING));
+ assert.equal(bare.model, "gpt-4o-mini-transcribe");
+ bare.cleanup();
+});