Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
feat(transcription): support gpt-live-transcribe sessions in the OpenAI realtime client (#2077)
Gabriel Steindff92c04c1de
diff --git a/src/helpers/openaiRealtimeStreaming.js b/src/helpers/openaiRealtimeStreaming.jsindex 58920f40e..b4468bd60 100644--- a/src/helpers/openaiRealtimeStreaming.js+++ b/src/helpers/openaiRealtimeStreaming.js@@ -3,6 +3,11 @@ const debugLogger = require("./debugLogger");const WEBSOCKET_TIMEOUT_MS = 15000;const DISCONNECT_TIMEOUT_MS = 3000;+// gpt-live-transcribe acks a commit only once every appended byte is+// transcribed, and it ran at 0.3–0.8x real time in testing, so its final trails+// the stop by a large fraction of the dictation length (partials keep streaming+// meanwhile).+const LIVE_TRANSCRIBE_COMMIT_TIMEOUT_MS = 30000;const SAMPLE_RATE = 24000;const COLD_START_BUFFER_MAX = 3 * SAMPLE_RATE * 2; // 3 seconds of 16-bit PCMconst KEEPALIVE_INTERVAL_MS = 15000;@@ -12,6 +17,23 @@ const SESSION_PREEMPT_MS = 55 * 60 * 1000;// on a cleaner channel pass their own vadThreshold.const DEFAULT_VAD_THRESHOLD = 0.6;+// Matches the server's check, which may pin a snapshot id.+const isLiveTranscribe = (model) => model.startsWith("gpt-live-transcribe");++// gpt-live-transcribe rejects every turn_detection config ("Turn detection is+// not supported for this transcription model"), so it never emits+// speech_started/speech_stopped and a turn only completes once the client+// commits — which disconnect() does on stop. Legacy models keep server VAD.+const turnDetectionFor = (model, vadThreshold) =>+ isLiveTranscribe(model)+ ? null+ : {+ type: "server_vad",+ threshold: vadThreshold,+ silence_duration_ms: 600,+ prefix_padding_ms: 500,+ };+// A socket factory does network work before the socket exists, so the dial// must be bounded; a socket resolving after the deadline is closed, not leaked.async function createSocketWithTimeout(createSocket, timeoutMs) {@@ -68,6 +90,7 @@ class OpenAIRealtimeStreaming {this.speechStartedAt = null;this.bufferingAudio = false;this.keepAliveInterval = null;+ this._onTurnSettled = null;}// Starts buffering audio immediately, before the WebSocket even exists —@@ -213,6 +236,9 @@ class OpenAIRealtimeStreaming {// Echo the server's VAD + format — the only place preconfigured// (cloud) session settings ever appear in field logs.const sessionInput = event.session?.audio?.input;+ // The server picks the managed model when it mints the secret; the+ // session echo is where the client learns which one it got.+ this.model = sessionInput?.transcription?.model ?? this.model;debugLogger.debug(`${this.providerLabel} session created (preconfigured)`,this._logContext({@@ -241,12 +267,7 @@ class OpenAIRealtimeStreaming {input: {format: { type: "audio/pcm", rate: this.inputRate },transcription: { model: this.model },- turn_detection: {- type: "server_vad",- threshold: this.vadThreshold,- silence_duration_ms: 600,- prefix_padding_ms: 500,- },+ turn_detection: turnDetectionFor(this.model, this.vadThreshold),},},},@@ -306,6 +327,7 @@ class OpenAIRealtimeStreaming {}));}+ this._onTurnSettled?.();break;}@@ -320,6 +342,7 @@ class OpenAIRealtimeStreaming {error: event.error?.message || event.error?.code || null,}));+ this._onTurnSettled?.();break;}@@ -567,29 +590,30 @@ class OpenAIRealtimeStreaming {if (this.ws.readyState === WebSocket.OPEN) {if (commit && this.audioBytesSent > 0) {- const prevOnFinal = this.onFinalTranscript;const prevOnError = this.onError;+ const timeoutMs = isLiveTranscribe(this.model)+ ? LIVE_TRANSCRIBE_COMMIT_TIMEOUT_MS+ : DISCONNECT_TIMEOUT_MS;await new Promise((resolve) => {- const tid = setTimeout(() => {- debugLogger.debug(- `${this.providerLabel} commit timeout, using accumulated text`,- this._logContext()- );- resolve();- }, DISCONNECT_TIMEOUT_MS);-const done = () => {clearTimeout(tid);- this.onFinalTranscript = prevOnFinal;+ this._onTurnSettled = null;this.onError = prevOnError;resolve();};- this.onFinalTranscript = (text) => {- prevOnFinal?.(text);+ const tid = setTimeout(() => {+ debugLogger.debug(+ `${this.providerLabel} commit timeout, using accumulated text`,+ this._logContext()+ );done();- };+ }, timeoutMs);++ // Any terminal event for the committed turn ends the wait; an empty+ // or failed turn must not stall for the whole budget.+ this._onTurnSettled = done;this.onError = (err) => {if (@@ -610,7 +634,7 @@ class OpenAIRealtimeStreaming {});}- this.ws.close();+ this.ws?.close();}const result = { text: this.getFullTranscript() };@@ -625,6 +649,8 @@ class OpenAIRealtimeStreaming {clearTimeout(this._sessionTimer);this._sessionTimer = null;this.stopKeepAlive();+ // A socket that dies mid-commit can never deliver the turn.+ this._onTurnSettled?.();if (this.ws) {try {diff --git a/test/helpers/meetingTranscriptionRouting.test.js b/test/helpers/meetingTranscriptionRouting.test.jsindex ce9063e83..fb9226bca 100644--- a/test/helpers/meetingTranscriptionRouting.test.js+++ b/test/helpers/meetingTranscriptionRouting.test.js@@ -4,6 +4,7 @@ const fs = require("node:fs");const path = require("node:path");const load = () => import("../../src/helpers/meetingTranscriptionRouting.js");+const modelRegistryData = require("../../src/models/modelRegistryData.json");// Sentinels this module throws, paired with the MEETING_ERROR_KEYS entry that// MeetingRecordingMount looks up. A healed legacy profile can reach any of them,@@ -227,3 +228,45 @@ test("every thrown sentinel is translated and rendered by the mount", () => {// have somewhere to put it.assert.match(translation.notes.meeting.unsupportedProvider, /\{\{provider\}\}/);});++// gpt-live-transcribe has no server VAD and only completes a turn when the client+// commits, which dictation does on stop and a long-running meeting stream never+// does. Until that guard exists, the desktop must not be able to route a meeting+// onto it: no registry entry, and a stale selection falls back to the default.+test("gpt-live-transcribe is never offered for Note Recording", async () => {+ const { resolveMeetingTranscriptionOptions } = await load();+ // Mirrors getStreamingTranscriptionProviders(): the meeting BYOK picker.+ const registryStreamingProviders = modelRegistryData.transcriptionProviders+ .map((provider) => ({ ...provider, models: provider.models.filter((m) => m.streaming) }))+ .filter((provider) => provider.models.length > 0);+ for (const provider of registryStreamingProviders) {+ for (const model of provider.models) {+ assert.equal(model.id.startsWith("gpt-live-transcribe"), false, model.id);+ }+ }++ assert.equal(+ resolveMeetingTranscriptionOptions({+ ...baseOptions,+ selectedProvider: "openai",+ selectedModel: "gpt-live-transcribe",+ byokProviders: registryStreamingProviders,+ }).model,+ "gpt-4o-mini-transcribe"+ );+ const managedCatalogs = [+ null,+ [{ id: "openai", models: [{ id: "gpt-4o-mini-transcribe", default: true }] }],+ ];+ for (const managedProviders of managedCatalogs) {+ assert.equal(+ resolveMeetingTranscriptionOptions({+ ...baseOptions,+ transcriptionMode: "openwhispr",+ managedProviders,+ selectedModel: "gpt-live-transcribe",+ }).model,+ "gpt-4o-mini-transcribe"+ );+ }+});diff --git a/test/helpers/openaiRealtimeStreaming.test.js b/test/helpers/openaiRealtimeStreaming.test.jsindex 6852825c6..c97bef9bf 100644--- a/test/helpers/openaiRealtimeStreaming.test.js+++ b/test/helpers/openaiRealtimeStreaming.test.js@@ -33,6 +33,22 @@ async function connectPreconfigured(streaming, socket) {await connected;}+const sentEvents = (socket, type) =>+ socket.sent.map((raw) => JSON.parse(raw)).filter((e) => e.type === type);++async function connectByok(streaming, socket, options) {+ const connected = streaming.connect({+ apiKey: "sk-test",+ createSocket: async () => socket,+ ...options,+ });+ await new Promise((resolve) => setImmediate(resolve));+ socket.readyState = WS.OPEN;+ socket.emit("message", JSON.stringify({ type: "session.created" }));+ socket.emit("message", JSON.stringify({ type: "session.updated" }));+ await connected;+}+function captureLogs(t) {const debugLogger = require("../../src/helpers/debugLogger");const entries = [];@@ -661,7 +677,7 @@ test("dictation-style connect (inputRate 16000, custom socket factory) declaresstreaming.cleanup();});-test("characterization: a final that lands during disconnect()'s commit window reaches onFinalTranscript WITHOUT its timestamp (Phase 1 forwards it)", async () => {+test("a final that lands during disconnect()'s commit window reaches onFinalTranscript with its timestamp", async () => {const OpenAIRealtimeStreaming = (await load()).default;const streaming = new OpenAIRealtimeStreaming();const socket = makeFakeSocket(WS.CONNECTING);@@ -693,10 +709,7 @@ test("characterization: a final that lands during disconnect()'s commit window rassert.equal(result.text, "tail words", "the tail is returned to the caller");assert.equal(calls.length, 1, "the tail also reaches the live onFinalTranscript handler");assert.equal(calls[0].text, "tail words");- // TODAY the temporary onFinalTranscript wrapper that disconnect() installs while- // awaiting the commit forwards only `text` to the previous handler.- // Phase 1 changes this line to `typeof calls[0].timestamp === "number"`.- assert.equal(calls[0].timestamp, undefined);+ assert.equal(typeof calls[0].timestamp, "number");});// -- diagnostic logging: stream labels, VAD events, failed/empty turns --@@ -867,3 +880,254 @@ test("without a streamLabel (dictation) log metadata carries no stream key", asyassert.ok(line);assert.equal("stream" in line.meta, false, "unlabelled sockets keep today's metadata shape");});++// -- gpt-live-transcribe: no server VAD, the client commits the turn on stop --++const LIVE_UPDATE_BODY = {+ type: "session.update",+ session: {+ type: "transcription",+ audio: {+ input: {+ format: { type: "audio/pcm", rate: 24000 },+ transcription: { model: "gpt-live-transcribe" },+ turn_detection: null,+ },+ },+ },+};++const completedEvent = (transcript) =>+ JSON.stringify({+ type: "conversation.item.input_audio_transcription.completed",+ item_id: "item_1",+ transcript,+ });++test("gpt-live-transcribe session.update sends turn_detection: null (any VAD config is rejected); the rest is the legacy payload", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.CONNECTING);+ await connectByok(streaming, socket, { model: "gpt-live-transcribe", vadThreshold: 0.3 });++ const updates = sentEvents(socket, "session.update");+ assert.equal(updates.length, 1);+ assert.deepEqual(updates[0], LIVE_UPDATE_BODY);+ assert.ok(socket.sent[0].includes('"turn_detection":null'), "null is on the wire, not dropped");+ streaming.cleanup();+});++test("legacy models keep the server_vad block byte-for-byte", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ for (const model of ["gpt-4o-mini-transcribe", "gpt-4o-transcribe"]) {+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.CONNECTING);+ await connectByok(streaming, socket, { model });++ const [update] = sentEvents(socket, "session.update");+ assert.equal(update.session.audio.input.transcription.model, model);+ assert.deepEqual(+ update.session.audio.input.turn_detection,+ { type: "server_vad", threshold: 0.6, silence_duration_ms: 600, prefix_padding_ms: 500 },+ model+ );+ streaming.cleanup();+ }+});++test("gpt-live-transcribe stop: disconnect() commits once, then consumes the .completed final it triggered", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.CONNECTING);+ await connectByok(streaming, socket, { model: "gpt-live-transcribe" });+ const partials = [];+ const finals = [];+ streaming.onPartialTranscript = (text) => partials.push(text);+ streaming.onFinalTranscript = (text, timestamp) => finals.push({ text, timestamp });++ // Partials stream during speech with no speech_started/speech_stopped around them.+ for (const delta of ["Hello", " world"]) {+ socket.emit(+ "message",+ JSON.stringify({ type: "conversation.item.input_audio_transcription.delta", delta })+ );+ }+ streaming.sendAudio(Buffer.alloc(4800, 1));++ const disconnecting = streaming.disconnect();+ await new Promise((resolve) => setImmediate(resolve));+ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 1);+ assert.equal(finals.length, 0, "the final only exists once the server answers the commit");++ socket.emit(+ "message",+ JSON.stringify({ type: "input_audio_buffer.committed", item_id: "item_1" })+ );+ socket.emit("message", completedEvent("Hello world."));+ const result = await disconnecting;++ assert.deepEqual(partials, ["Hello", "Hello world"]);+ assert.equal(result.text, "Hello world.");+ assert.deepEqual(+ finals.map((f) => f.text),+ ["Hello world."]+ );+ assert.equal(typeof finals[0].timestamp, "number", "timestamp falls back without speech_started");+ assert.equal(streaming.speechStartedCount, 0);+ assert.equal(+ sentEvents(socket, "input_audio_buffer.commit").length,+ 1,+ "still exactly one commit"+ );+ assert.equal(socket.readyState, WS.CLOSED);+});++test("gpt-live-transcribe stop with no audio sent never commits (an empty commit is a server error)", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.CONNECTING);+ await connectByok(streaming, socket, { model: "gpt-live-transcribe" });++ const result = await streaming.disconnect();++ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 0);+ assert.equal(result.text, "");+ assert.equal(socket.readyState, WS.CLOSED);+});++test("gpt-live-transcribe stop outlives the legacy 3 s commit budget; legacy keeps it", (t) => {+ t.mock.timers.enable({ apis: ["setTimeout"] });+ return (async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const settle = async () => new Promise((resolve) => setImmediate(resolve));++ const live = new OpenAIRealtimeStreaming();+ const liveSocket = makeFakeSocket(WS.CONNECTING);+ await connectByok(live, liveSocket, { model: "gpt-live-transcribe" });+ live.sendAudio(Buffer.alloc(4800, 1));+ let liveSettled = false;+ const liveDisconnect = live.disconnect().then((result) => {+ liveSettled = true;+ return result;+ });+ await settle();+ t.mock.timers.tick(3000);+ await settle();+ assert.equal(liveSettled, false, "a long dictation's final trails stop by more than 3 s");+ liveSocket.emit("message", completedEvent("late tail"));+ assert.equal((await liveDisconnect).text, "late tail");++ const legacy = new OpenAIRealtimeStreaming();+ const legacySocket = makeFakeSocket(WS.CONNECTING);+ await connectByok(legacy, legacySocket, { model: "gpt-4o-mini-transcribe" });+ legacy.sendAudio(Buffer.alloc(4800, 1));+ let legacySettled = false;+ const legacyDisconnect = legacy.disconnect().then(() => {+ legacySettled = true;+ });+ await settle();+ t.mock.timers.tick(3000);+ await legacyDisconnect;+ assert.equal(legacySettled, true);+ })();+});++test("an empty or failed committed turn ends the stop wait at once instead of stalling for the budget", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const terminalEvents = [+ completedEvent(" "),+ JSON.stringify({+ type: "conversation.item.input_audio_transcription.failed",+ item_id: "item_1",+ error: { code: "audio_unintelligible", message: "audio too noisy" },+ }),+ ];+ for (const terminal of terminalEvents) {+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.CONNECTING);+ await connectByok(streaming, socket, { model: "gpt-live-transcribe" });+ let finalCalled = false;+ streaming.onFinalTranscript = () => {+ finalCalled = true;+ };+ streaming.sendAudio(Buffer.alloc(4800, 1));++ const disconnecting = streaming.disconnect();+ await new Promise((resolve) => setImmediate(resolve));+ socket.emit("message", terminal);+ const result = await Promise.race([+ disconnecting,+ new Promise((_, reject) => setTimeout(() => reject(new Error("stop stalled")), 500)),+ ]);++ assert.equal(result.text, "");+ assert.equal(finalCalled, false);+ assert.equal(streaming._onTurnSettled, null, "hook is released once the wait ends");+ }+});++test("a socket that closes during the commit wait ends the stop at once with the accumulated text", async () => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.CONNECTING);+ await connectByok(streaming, socket, { model: "gpt-live-transcribe" });+ streaming.sendAudio(Buffer.alloc(4800, 1));++ const disconnecting = streaming.disconnect();+ await new Promise((resolve) => setImmediate(resolve));+ socket.readyState = WS.CLOSED;+ socket.emit("close", 1006, Buffer.from(""));+ const result = await Promise.race([+ disconnecting,+ new Promise((_, reject) => setTimeout(() => reject(new Error("stop stalled")), 500)),+ ]);++ assert.equal(result.text, "");+ assert.equal(streaming.ws, null);+});++test("preconfigured sessions adopt the model the server minted (session.created echo) and never send session.update", async (t) => {+ const OpenAIRealtimeStreaming = (await load()).default;+ const logs = captureLogs(t);+ const streaming = new OpenAIRealtimeStreaming();+ const socket = makeFakeSocket(WS.CONNECTING);++ const connected = streaming.connect({+ apiKey: "cs-secret",+ preconfigured: true,+ // The desktop's BYOK default; the server decides the managed model.+ model: "gpt-4o-mini-transcribe",+ createSocket: async () => socket,+ });+ await new Promise((resolve) => setImmediate(resolve));+ socket.readyState = WS.OPEN;+ socket.emit(+ "message",+ JSON.stringify({+ type: "session.created",+ session: {+ audio: {+ input: {+ format: { type: "audio/pcm", rate: 24000 },+ transcription: { model: "gpt-live-transcribe", language: null },+ turn_detection: null,+ },+ },+ },+ })+ );+ await connected;++ assert.equal(streaming.model, "gpt-live-transcribe");+ assert.equal(sentEvents(socket, "session.update").length, 0);+ const line = logs.find((entry) => entry.message.includes("session created (preconfigured)"));+ assert.equal(line.meta.model, "gpt-live-transcribe");+ assert.equal(line.meta.turnDetection, null);+ streaming.cleanup();++ // No echo (older servers, bare mocks): the caller's model stands.+ const bare = new OpenAIRealtimeStreaming();+ await connectPreconfigured(bare, makeFakeSocket(WS.CONNECTING));+ assert.equal(bare.model, "gpt-4o-mini-transcribe");+ bare.cleanup();+});