Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(meetings): harden realtime provider routing
Gabriel Steinf67f2a6b1cdf
diff --git a/preload.js b/preload.jsindex 4e4125228..121c14e9f 100644--- a/preload.js+++ b/preload.js@@ -732,6 +732,10 @@ contextBridge.exposeInMainWorld("electronAPI", {"meeting-transcription-error",(callback) => (_event, data) => callback(data)),+ onMeetingTranscriptionFatalError: registerListener(+ "meeting-transcription-fatal-error",+ (callback) => (_event, data) => callback(data)+ ),// Dictation realtime streamingdictationRealtimeWarmup: (options) => ipcRenderer.invoke("dictation-realtime-warmup", options),diff --git a/src/components/TranscriptionModelPicker.tsx b/src/components/TranscriptionModelPicker.tsxindex 9d1f7641d..08a2ae1e9 100644--- a/src/components/TranscriptionModelPicker.tsx+++ b/src/components/TranscriptionModelPicker.tsx@@ -408,7 +408,7 @@ export default function TranscriptionModelPicker({);const cloudProviderTabs = useMemo(() => {const availableIds = new Set(availableCloudProviders.map((p) => p.id));- availableIds.add("custom");+ if (!streamingOnly) availableIds.add("custom");return CLOUD_PROVIDER_TABS.filter((p) => availableIds.has(p.id)).map((provider) => {const named =provider.id === "custom"@@ -418,7 +418,7 @@ export default function TranscriptionModelPicker({? named: { ...named, disabled: true, disabledLabel: t("common.managedByOrg") };});- }, [availableCloudProviders, providerAllowed, t]);+ }, [availableCloudProviders, providerAllowed, streamingOnly, t]);useEffect(() => {selectedLocalModelRef.current = selectedLocalModel;diff --git a/src/components/settings/MeetingSettings.tsx b/src/components/settings/MeetingSettings.tsxindex dc36dd3be..a7a2a6a25 100644--- a/src/components/settings/MeetingSettings.tsx+++ b/src/components/settings/MeetingSettings.tsx@@ -7,7 +7,6 @@ import { InferenceModeSelector, SettingsRow } from "../ui/SettingsSection";import type { InferenceModeOption } from "../ui/SettingsSection";import { Toggle } from "../ui/toggle";import TranscriptionModelPicker from "../TranscriptionModelPicker";-import SelfHostedPanel from "../SelfHostedPanel";import type { InferenceMode } from "../../types/electron";import { useStartOnboarding } from "../../hooks/useStartOnboarding";@@ -50,8 +49,6 @@ export function MeetingTranscriptionPanel() {meetingCloudTranscriptionBaseUrl,setMeetingCloudTranscriptionBaseUrl,setMeetingCloudTranscriptionMode,- meetingRemoteTranscriptionUrl,- setMeetingRemoteTranscriptionUrl,} = useSettingsStore();const { modes: transcriptionModes, isModeAllowed } = usePolicyModeOptions<InferenceModeOption>([@@ -80,6 +77,8 @@ export function MeetingTranscriptionPanel() {label: t("settingsPage.transcription.modes.selfHosted"),description: t("settingsPage.transcription.modes.selfHostedDesc"),icon: <Network className="w-4 h-4" />,+ disabled: true,+ badge: t("settings.meeting.comingSoon"),},],"transcription"@@ -87,6 +86,7 @@ export function MeetingTranscriptionPanel() {const handleTranscriptionModeSelect = (mode: InferenceMode) => {if (!isModeAllowed(mode)) return;+ if (mode === "self-hosted") return;if (mode === "openwhispr" && !isSignedIn) {startOnboarding();return;@@ -140,18 +140,6 @@ export function MeetingTranscriptionPanel() {{meetingTranscriptionMode === "providers" && renderTranscriptionPicker("cloud")}{meetingTranscriptionMode === "local" && renderTranscriptionPicker("local")}- {meetingTranscriptionMode === "self-hosted" && (- <>- <SelfHostedPanel- service="transcription"- url={meetingRemoteTranscriptionUrl}- onUrlChange={setMeetingRemoteTranscriptionUrl}- />- <p className="text-xs text-muted-foreground/80 px-1">- {t("settingsPage.speechToText.selfHostedStreamingNote")}- </p>- </>- )}<MeetingSpeakerDetectionRow /></div>);diff --git a/src/helpers/assemblyAiStreaming.js b/src/helpers/assemblyAiStreaming.jsindex 0bae9b1d3..3a6b79988 100644--- a/src/helpers/assemblyAiStreaming.js+++ b/src/helpers/assemblyAiStreaming.js@@ -21,6 +21,8 @@ class AssemblyAiStreaming {this.onFinalTranscript = null;this.onError = null;this.onSessionEnd = null;+ this.onConnectionLost = null;+ this.connectionLossNotified = false;this.pendingResolve = null;this.pendingReject = null;this.connectionTimeout = null;@@ -245,6 +247,7 @@ class AssemblyAiStreaming {this.ws = this.warmConnection;this.isConnected = true;+ this.connectionLossNotified = false;this.sessionId = this.warmSessionId || null;this.warmConnection = null;this.warmConnectionReady = false;@@ -257,9 +260,10 @@ class AssemblyAiStreaming {this.ws.removeAllListeners("error");this.ws.on("error", (error) => {+ const wasActive = this.isConnected;debugLogger.error("AssemblyAI WebSocket error", { error: error.message });this.cleanup();- this.onError?.(error);+ if (wasActive && !this.isDisconnecting) this.notifyConnectionLost(error);});this.ws.removeAllListeners("close");@@ -272,7 +276,7 @@ class AssemblyAiStreaming {});this.cleanup();if (wasActive && !this.isDisconnecting) {- this.onError?.(new Error(`Connection lost (code: ${code})`));+ this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));}});@@ -318,6 +322,7 @@ class AssemblyAiStreaming {this.accumulatedText = "";this.lastTurnText = "";this.turns = [];+ this.connectionLossNotified = false;// Try to use pre-warmed connection for instant startif (this.hasWarmConnection()) {@@ -350,6 +355,7 @@ class AssemblyAiStreaming {});this.ws.on("error", (error) => {+ const wasActive = this.isConnected;debugLogger.error("AssemblyAI WebSocket error", { error: error.message });this.cleanup();if (this.pendingReject) {@@ -357,7 +363,11 @@ class AssemblyAiStreaming {this.pendingReject = null;this.pendingResolve = null;}- this.onError?.(error);+ if (wasActive && !this.isDisconnecting) {+ this.notifyConnectionLost(error);+ } else if (!this.isDisconnecting) {+ this.onError?.(error);+ }});this.ws.on("close", (code, reason) => {@@ -374,12 +384,22 @@ class AssemblyAiStreaming {}this.cleanup();if (wasActive && !this.isDisconnecting) {- this.onError?.(new Error(`Connection lost (code: ${code})`));+ this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));}});});}+ notifyConnectionLost(error) {+ if (this.connectionLossNotified) return;+ this.connectionLossNotified = true;+ if (this.onConnectionLost) {+ this.onConnectionLost(error);+ } else {+ this.onError?.(error);+ }+ }+handleMessage(data) {try {const message = JSON.parse(data.toString());diff --git a/src/helpers/cortiStreaming.js b/src/helpers/cortiStreaming.jsindex d7dd8b21f..d835581ac 100644--- a/src/helpers/cortiStreaming.js+++ b/src/helpers/cortiStreaming.js@@ -17,6 +17,8 @@ class CortiStreaming {this.onFinalTranscript = null;this.onError = null;this.onSessionEnd = null;+ this.onConnectionLost = null;+ this.connectionLossNotified = false;this.pendingResolve = null;this.pendingReject = null;this.connectionTimeout = null;@@ -73,6 +75,7 @@ class CortiStreaming {this.accumulatedText = "";this.completedSegments = [];this.configAccepted = false;+ this.connectionLossNotified = false;this.preConfigBuffer = [];this.preConfigBufferSize = 0;this.audioBytesSent = 0;@@ -113,6 +116,7 @@ class CortiStreaming {});ws.on("error", (error) => {+ const wasActive = this.isConnected;debugLogger.error("Corti WebSocket error", { error: error.message });this.cleanup();if (this.pendingReject) {@@ -120,7 +124,11 @@ class CortiStreaming {this.pendingReject = null;this.pendingResolve = null;}- this.onError?.(error);+ if (wasActive && !this.isDisconnecting) {+ this.notifyConnectionLost(error);+ } else if (!this.isDisconnecting) {+ this.onError?.(error);+ }});ws.on("close", (code, reason) => {@@ -138,11 +146,21 @@ class CortiStreaming {this.resolvePendingAck();this.cleanup();if (wasActive && !this.isDisconnecting) {- this.onError?.(new Error(`Connection lost (code: ${code})`));+ this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));}});}+ notifyConnectionLost(error) {+ if (this.connectionLossNotified) return;+ this.connectionLossNotified = true;+ if (this.onConnectionLost) {+ this.onConnectionLost(error);+ } else {+ this.onError?.(error);+ }+ }+// Pre-open the socket and finish the config handshake before recording, so the// first words aren't lost waiting for CONFIG_ACCEPTED. Mirrors Deepgram/AssemblyAI.async warmup(options = {}) {diff --git a/src/helpers/deepgramStreaming.js b/src/helpers/deepgramStreaming.jsindex e4a0f9731..3275c107a 100644--- a/src/helpers/deepgramStreaming.js+++ b/src/helpers/deepgramStreaming.js@@ -77,6 +77,8 @@ class DeepgramStreaming {this.onFinalTranscript = null;this.onError = null;this.onSessionEnd = null;+ this.onConnectionLost = null;+ this.connectionLossNotified = false;this.pendingResolve = null;this.pendingReject = null;this.connectionTimeout = null;@@ -419,6 +421,7 @@ class DeepgramStreaming {this.ws = this.warmConnection;this.isConnected = true;+ this.connectionLossNotified = false;this.sessionId = this.warmSessionId || null;this.warmConnection = null;this.warmConnectionReady = false;@@ -431,9 +434,10 @@ class DeepgramStreaming {this.ws.removeAllListeners("error");this.ws.on("error", (error) => {+ const wasActive = this.isConnected;debugLogger.error("Deepgram WebSocket error", { error: error.message });this.cleanup();- this.onError?.(error);+ if (wasActive && !this.isDisconnecting) this.notifyConnectionLost(error);});this.ws.removeAllListeners("close");@@ -449,7 +453,7 @@ class DeepgramStreaming {}this.cleanup();if (wasActive && !this.isDisconnecting) {- this.onError?.(new Error(`Connection lost (code: ${code})`));+ this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));}});@@ -562,6 +566,7 @@ class DeepgramStreaming {this.finalSegments = [];this.audioBytesSent = 0;this.resultsReceived = 0;+ this.connectionLossNotified = false;if (replayBuffer && replayBuffer.length > 0) {this.coldStartBuffer = replayBuffer;@@ -608,6 +613,7 @@ class DeepgramStreaming {});this.ws.on("error", (error) => {+ const wasActive = this.isConnected;debugLogger.error("Deepgram WebSocket error", { error: error.message });// Invalidate cached token on auth failure so next attempt fetches freshif (error.message && error.message.includes("401")) {@@ -620,7 +626,11 @@ class DeepgramStreaming {this.pendingReject = null;this.pendingResolve = null;}- this.onError?.(error);+ if (wasActive && !this.isDisconnecting) {+ this.notifyConnectionLost(error);+ } else if (!this.isDisconnecting) {+ this.onError?.(error);+ }});this.ws.on("close", (code, reason) => {@@ -640,12 +650,22 @@ class DeepgramStreaming {}this.cleanup();if (wasActive && !this.isDisconnecting) {- this.onError?.(new Error(`Connection lost (code: ${code})`));+ this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));}});});}+ notifyConnectionLost(error) {+ if (this.connectionLossNotified) return;+ this.connectionLossNotified = true;+ if (this.onConnectionLost) {+ this.onConnectionLost(error);+ } else {+ this.onError?.(error);+ }+ }+handleMessage(data) {try {const message = JSON.parse(data.toString());diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex caa064539..99e2004e1 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -64,13 +64,15 @@ const {} = require("./whisperVadConfig");const {- STREAMING_CLIENT_BY_PROVIDER,ALLOWED_MEETING_PROVIDERS,+ getMeetingStreamingClient,+ getMeetingConnectionKey,} = require("./meetingStreamingProviders");// Meeting capture runs at 24 kHz (see meetingRecordingStore AudioContext); cloud// streaming providers must be told the true PCM rate or they misread the audio.const MEETING_STREAM_SAMPLE_RATE = 24000;+const MEETING_RECONNECT_BUFFER_MAX_BYTES = MEETING_STREAM_SAMPLE_RATE * 2 * 30;function parseAttendees(raw) {if (!raw) return [];@@ -5424,119 +5426,202 @@ class IPCHandlers {streaming.onError = (error) => {send("meeting-transcription-error", error.message);};- streaming.onSessionExpired = () => reconnectMeetingStreams();+ const recoverConnection = async (error, restoreOldOnFailure) => {+ let recovered = false;+ try {+ recovered = await reconnectMeetingStreams({ restoreOldOnFailure });+ } catch (reconnectError) {+ debugLogger.error("Meeting stream recovery failed unexpectedly", {+ error: reconnectError.message,+ });+ }+ if (!recovered && !meetingFatalErrorSent) {+ meetingFatalErrorSent = true;+ send(+ "meeting-transcription-fatal-error",+ error?.message || "Meeting transcription connection could not be restored."+ );+ }+ };+ streaming.onConnectionLost = (error) => {+ void recoverConnection(error, false);+ };+ streaming.onSessionExpired = ({ proactive = false } = {}) => {+ void recoverConnection(+ new Error("Meeting transcription session could not be renewed."),+ proactive+ );+ };};- const reconnectMeetingStreams = async () => {- if (meetingReconnecting || meetingLocalMode) return;+ const resetMeetingReconnectAudio = () => {+ meetingReconnectAudioBuffers = { mic: [], system: [] };+ meetingReconnectAudioBytes = { mic: 0, system: 0 };+ meetingReconnectReplaySources = new Set();+ };- const options = meetingConnectionOptions;- const win = meetingConnectionWin;- if (!options || !win || win.isDestroyed()) {- debugLogger.error("Cannot reconnect meeting streams: missing connection context");- return;+ const queueMeetingReconnectAudio = (source, buffer) => {+ if (!meetingReconnectReplaySources.has(source)) return;+ const copy = Buffer.from(buffer);+ const queue = meetingReconnectAudioBuffers[source];+ queue.push(copy);+ meetingReconnectAudioBytes[source] += copy.length;+ while (+ meetingReconnectAudioBytes[source] > MEETING_RECONNECT_BUFFER_MAX_BYTES &&+ queue.length > 1+ ) {+ meetingReconnectAudioBytes[source] -= queue.shift().length;}+ };- if (meetingReconnectCount >= MAX_MEETING_RECONNECTS) {- debugLogger.error("Meeting reconnect limit reached", { count: meetingReconnectCount });- win.webContents.send(- "meeting-transcription-error",- "Session reconnect limit reached. Please stop and restart the recording."- );- return;- }+ const replayMeetingReconnectAudio = (source, streaming) => {+ if (!meetingReconnectReplaySources.has(source)) return true;+ const queue = meetingReconnectAudioBuffers[source];+ const replayed = queue.every((buffer) => streaming.sendAudio(buffer));+ debugLogger.info("Replayed meeting audio after reconnect", {+ source,+ chunks: queue.length,+ bytes: meetingReconnectAudioBytes[source],+ });+ return replayed;+ };- meetingReconnecting = true;- meetingReconnectCount++;+ const reconnectMeetingStreams = ({ restoreOldOnFailure = false } = {}) => {+ if (meetingReconnectPromise) return meetingReconnectPromise;- const StreamingClass =- STREAMING_CLIENT_BY_PROVIDER[options.provider] ?? OpenAIRealtimeStreaming;+ const pending = (async () => {+ if (meetingLocalMode) return false;- const oldMic = this._meetingMicStreaming;- const oldSystem = this._meetingSystemStreaming;+ const options = meetingConnectionOptions;+ const win = meetingConnectionWin;+ if (!options || !win || win.isDestroyed()) {+ debugLogger.error("Cannot reconnect meeting streams: missing connection context");+ return false;+ }- // Swap fresh instances in before the token fetch so audio arriving during- // the swap lands in their pre-connect buffers instead of a dead socket.- const newMic = new StreamingClass();- newMic.beginConnecting?.();- attachMeetingStreamingHandlers(newMic, win, "mic");- this._meetingMicStreaming = newMic;- let newSystem = null;- if (oldSystem) {- newSystem = new StreamingClass();- newSystem.beginConnecting?.();- attachMeetingStreamingHandlers(newSystem, win, "system");- this._meetingSystemStreaming = newSystem;- }+ if (meetingReconnectCount >= MAX_MEETING_RECONNECTS) {+ debugLogger.error("Meeting reconnect limit reached", { count: meetingReconnectCount });+ return false;+ }- debugLogger.info("Reconnecting meeting streams", {- attempt: meetingReconnectCount,- maxAttempts: MAX_MEETING_RECONNECTS,- });+ meetingReconnectCount++;- const tokenEvent = { sender: win.webContents };- try {- const connectOpts = {- model: options.model,- language: options.language,- preconfigured: options.mode !== "byok",- environment: options.environment,- tenant: options.tenant,- keyterms: options.keyterms,- sampleRate: MEETING_STREAM_SAMPLE_RATE,- };+ const oldMic = this._meetingMicStreaming;+ const oldSystem = this._meetingSystemStreaming;+ meetingReconnectReplaySources = new Set([+ ...(!oldMic?.isConnected ? ["mic"] : []),+ ...(oldSystem && !oldSystem.isConnected ? ["system"] : []),+ ]);+ let newMic = null;+ let newSystem = null;- let pairs;- if (newSystem) {- const secrets = await fetchRealtimeToken(tokenEvent, options, { streams: 2 });- pairs = [- { streaming: newMic, secret: secrets[0] },- { streaming: newSystem, secret: secrets[1] },- ];- } else {- pairs = [{ streaming: newMic, secret: await fetchRealtimeToken(tokenEvent, options) }];- }+ try {+ const StreamingClass = getMeetingStreamingClient(options.provider);+ newMic = new StreamingClass();+ attachMeetingStreamingHandlers(newMic, win, "mic");+ if (oldSystem) {+ newSystem = new StreamingClass();+ attachMeetingStreamingHandlers(newSystem, win, "system");+ }- await Promise.all(- pairs.map(({ streaming, secret }) =>- streaming.connect({ apiKey: secret, token: secret, ...connectOpts })- )- );+ debugLogger.info("Reconnecting meeting streams", {+ attempt: meetingReconnectCount,+ maxAttempts: MAX_MEETING_RECONNECTS,+ });++ const tokenEvent = { sender: win.webContents };+ const connectOpts = {+ model: options.model,+ language: options.language,+ preconfigured: options.mode !== "byok",+ environment: options.environment,+ tenant: options.tenant,+ keyterms: options.keyterms,+ sampleRate: MEETING_STREAM_SAMPLE_RATE,+ };- if (meetingConnectionOptions !== options) {- // Recording stopped while the reconnect was in flight.- for (const { streaming } of pairs) streaming.disconnect().catch(() => {});+ let pairs;+ if (newSystem) {+ const secrets = await fetchRealtimeToken(tokenEvent, options, { streams: 2 });+ pairs = [+ { streaming: newMic, secret: secrets[0] },+ { streaming: newSystem, secret: secrets[1] },+ ];+ } else {+ pairs = [{ streaming: newMic, secret: await fetchRealtimeToken(tokenEvent, options) }];+ }++ await Promise.all(+ pairs.map(({ streaming, secret }) =>+ streaming.connect({ apiKey: secret, token: secret, ...connectOpts })+ )+ );++ if (pairs.some(({ streaming }) => !streaming.isConnected)) {+ throw new Error("Meeting transcription connection closed during reconnect.");+ }++ if (meetingConnectionOptions !== options) {+ for (const { streaming } of pairs) streaming.disconnect().catch(() => {});+ oldMic?.disconnect().catch(() => {});+ oldSystem?.disconnect().catch(() => {});+ resetMeetingReconnectAudio();+ return true;+ }++ const replayedMic = replayMeetingReconnectAudio("mic", newMic);+ const replayedSystem = !newSystem || replayMeetingReconnectAudio("system", newSystem);+ if (!replayedMic || !replayedSystem) {+ throw new Error("Meeting audio could not be restored after reconnect.");+ }+ this._meetingMicStreaming = newMic;+ this._meetingSystemStreaming = newSystem;+ resetMeetingReconnectAudio();oldMic?.disconnect().catch(() => {});oldSystem?.disconnect().catch(() => {});- return;- }+ meetingConnectionKey = getMeetingConnectionKey(options);- oldMic?.disconnect().catch(() => {});- oldSystem?.disconnect().catch(() => {});+ debugLogger.info("Meeting streams reconnected", { attempt: meetingReconnectCount });+ meetingReconnectCount = 0;+ return true;+ } catch (error) {+ debugLogger.error("Meeting stream reconnect failed", {+ error: error.message,+ attempt: meetingReconnectCount,+ });+ newMic?.disconnect().catch(() => {});+ newSystem?.disconnect().catch(() => {});+ if (meetingConnectionOptions !== options) {+ oldMic?.disconnect().catch(() => {});+ oldSystem?.disconnect().catch(() => {});+ resetMeetingReconnectAudio();+ return true;+ }- debugLogger.info("Meeting streams reconnected", { attempt: meetingReconnectCount });- } catch (error) {- debugLogger.error("Meeting stream reconnect failed", {- error: error.message,- attempt: meetingReconnectCount,- });- newMic.disconnect().catch(() => {});- newSystem?.disconnect().catch(() => {});- if (meetingConnectionOptions === options) {- // A proactive reconnect leaves the old connections open; restore them- // so transcription continues until the hard limit retries this path.- this._meetingMicStreaming = oldMic;- this._meetingSystemStreaming = oldSystem;+ const canRestoreOld =+ restoreOldOnFailure && !!oldMic?.isConnected && (!oldSystem || oldSystem.isConnected);+ if (canRestoreOld) {+ this._meetingMicStreaming = oldMic;+ this._meetingSystemStreaming = oldSystem;+ } else {+ oldMic?.disconnect().catch(() => {});+ oldSystem?.disconnect().catch(() => {});+ this._meetingMicStreaming = null;+ this._meetingSystemStreaming = null;+ meetingConnectionKey = null;+ }+ resetMeetingReconnectAudio();if (!win.isDestroyed()) {win.webContents.send("meeting-transcription-error", error.message);}- } else {- oldMic?.disconnect().catch(() => {});- oldSystem?.disconnect().catch(() => {});+ return canRestoreOld;}- } finally {- meetingReconnecting = false;- }+ })();++ meetingReconnectPromise = pending;+ return pending.finally(() => {+ if (meetingReconnectPromise === pending) meetingReconnectPromise = null;+ });};const fetchRealtimeToken = async (event, options, { streams } = {}) => {@@ -5633,6 +5718,10 @@ class IPCHandlers {return streams === 2 ? [apiKey, apiKey] : apiKey;}+ if (options.provider !== "openai-realtime") {+ throw new Error(`Unsupported realtime token provider: ${options.provider}`);+ }+if (options.mode === "byok") {const apiKey = this.environmentManager.getOpenAIKey();if (!apiKey) throw new Error("No OpenAI API key configured. Add your key in Settings.");@@ -5698,6 +5787,8 @@ class IPCHandlers {(systemAudioMode === "unsupported" || !!this._meetingSystemStreaming?.isConnected);const connectRealtimeStreaming = async (event, options) => {+ const connectionKey = getMeetingConnectionKey(options);+ const StreamingClass = getMeetingStreamingClient(options.provider);if (this._meetingMicStreaming?.isConnected) {await this._meetingMicStreaming.disconnect();}@@ -5735,18 +5826,30 @@ class IPCHandlers {];}- const StreamingClass =- STREAMING_CLIENT_BY_PROVIDER[options.provider] ?? OpenAIRealtimeStreaming;for (const { ref, source } of pairs) {this[ref] = new StreamingClass();attachMeetingStreamingHandlers(this[ref], win, source);}- await Promise.all(- pairs.map(({ ref, secret }) =>- this[ref].connect({ apiKey: secret, token: secret, ...connectOpts })- )- );+ try {+ await Promise.all(+ pairs.map(({ ref, secret }) =>+ this[ref].connect({ apiKey: secret, token: secret, ...connectOpts })+ )+ );+ if (pairs.some(({ ref }) => !this[ref]?.isConnected)) {+ throw new Error("Meeting transcription connection closed during startup.");+ }+ meetingConnectionKey = connectionKey;+ } catch (error) {+ await Promise.all(+ pairs.map(({ ref }) => this[ref]?.disconnect().catch(() => ({ text: "" })))+ );+ this._meetingMicStreaming = null;+ this._meetingSystemStreaming = null;+ meetingConnectionKey = null;+ throw error;+ }return win;};@@ -5761,11 +5864,16 @@ class IPCHandlers {let meetingStartedAt = null;let meetingSendCounts = { mic: 0, system: 0 };const meetingEchoLeakDetector = new MeetingEchoLeakDetector();- let meetingReconnecting = false;+ let meetingReconnectPromise = null;+ let meetingFatalErrorSent = false;let meetingReconnectCount = 0;const MAX_MEETING_RECONNECTS = 5;let meetingConnectionOptions = null;let meetingConnectionWin = null;+ let meetingConnectionKey = null;+ let meetingReconnectAudioBuffers = { mic: [], system: [] };+ let meetingReconnectAudioBytes = { mic: 0, system: 0 };+ let meetingReconnectReplaySources = new Set();const fs = require("fs");let meetingDiarizationStream = null;@@ -5923,6 +6031,7 @@ class IPCHandlers {}}+ queueMeetingReconnectAudio(source, outbound);const sent = streaming.sendAudio(outbound);meetingSendCounts[source]++;if (meetingSendCounts[source] <= 5 || meetingSendCounts[source] % 100 === 0) {@@ -6488,10 +6597,13 @@ class IPCHandlers {resetPendingMicFinals();meetingAecEnabled = false;meetingEchoLeakDetector.reset();- meetingReconnecting = false;+ meetingReconnectPromise = null;+ meetingFatalErrorSent = false;meetingReconnectCount = 0;meetingConnectionOptions = null;meetingConnectionWin = null;+ meetingConnectionKey = null;+ resetMeetingReconnectAudio();};const disconnectMeetingStreaming = async ({ flushPending = false } = {}) => {@@ -6642,8 +6754,12 @@ class IPCHandlers {}const { mode: systemAudioMode } = await getMeetingSystemAudioPlan();+ const requestedConnectionKey = getMeetingConnectionKey(options);- if (isMeetingStreamingConnected(systemAudioMode)) {+ if (+ isMeetingStreamingConnected(systemAudioMode) &&+ meetingConnectionKey === requestedConnectionKey+ ) {debugLogger.debug("Meeting transcription already prepared (warm connections)");return { success: true, alreadyPrepared: true };}@@ -6700,10 +6816,12 @@ class IPCHandlers {meetingConnectionOptions = options;meetingConnectionWin = BrowserWindow.fromWebContents(event.sender);meetingReconnectCount = 0;+ meetingFatalErrorSent = false;this.meetingDetectionEngine?.setUserRecording(true);try {const systemAudioPlan = await getMeetingSystemAudioPlan();let { mode: systemAudioMode, strategy: systemAudioStrategy } = systemAudioPlan;+ const requestedConnectionKey = getMeetingConnectionKey(options);meetingEchoLeakDetector.reset();meetingOneOnOneAttendee = resolveOneOnOneAttendeeForNote(options.noteId);meetingOneOnOneProfileBound = false;@@ -6721,7 +6839,11 @@ class IPCHandlers {}// If already prepared (warm connections from prepare), just re-attach handlers- if (!meetingLocalMode && isMeetingStreamingConnected(systemAudioMode)) {+ if (+ !meetingLocalMode &&+ isMeetingStreamingConnected(systemAudioMode) &&+ meetingConnectionKey === requestedConnectionKey+ ) {debugLogger.debug("Meeting transcription start: reusing warm connections");const win = BrowserWindow.fromWebContents(event.sender);attachMeetingStreamingHandlers(this._meetingMicStreaming, win, "mic");diff --git a/src/helpers/meetingStreamingProviders.js b/src/helpers/meetingStreamingProviders.jsindex f14cf91f7..b88a35e40 100644--- a/src/helpers/meetingStreamingProviders.js+++ b/src/helpers/meetingStreamingProviders.js@@ -16,4 +16,26 @@ const STREAMING_CLIENT_BY_PROVIDER = {// class and silently fall through to the OpenAI default.const ALLOWED_MEETING_PROVIDERS = new Set(["local", ...Object.keys(STREAMING_CLIENT_BY_PROVIDER)]);-module.exports = { STREAMING_CLIENT_BY_PROVIDER, ALLOWED_MEETING_PROVIDERS };+const getMeetingStreamingClient = (provider) => {+ const StreamingClient = STREAMING_CLIENT_BY_PROVIDER[provider];+ if (!StreamingClient) throw new Error(`Unsupported meeting streaming provider: ${provider}`);+ return StreamingClient;+};++const getMeetingConnectionKey = (options = {}) =>+ JSON.stringify({+ provider: options.provider,+ model: options.model,+ language: options.language,+ mode: options.mode,+ environment: options.environment,+ tenant: options.tenant,+ keyterms: options.keyterms,+ });++module.exports = {+ STREAMING_CLIENT_BY_PROVIDER,+ ALLOWED_MEETING_PROVIDERS,+ getMeetingStreamingClient,+ getMeetingConnectionKey,+};diff --git a/src/helpers/meetingTranscriptionRouting.js b/src/helpers/meetingTranscriptionRouting.jsnew file mode 100644index 000000000..b060162a9--- /dev/null+++ b/src/helpers/meetingTranscriptionRouting.js@@ -0,0 +1,79 @@+const DEFAULT_MANAGED_PROVIDER = {+ id: "openai",+ models: [{ id: "gpt-4o-mini-transcribe", default: true }],+};++const resolveModel = (provider, selectedModel) =>+ provider.models.find((model) => model.id === selectedModel)?.id ??+ provider.models.find((model) => model.default)?.id ??+ provider.models[0]?.id;++export function resolveMeetingTranscriptionOptions({+ transcriptionMode,+ language,+ localProvider,+ whisperModel,+ parakeetModel,+ selectedProvider,+ selectedModel,+ byokProviders,+ managedProviders,+ cortiEnvironment,+ cortiTenant,+ keyterms,+}) {+ if (transcriptionMode === "local") {+ return {+ provider: "local",+ localProvider,+ localModel:+ localProvider === "nvidia"+ ? parakeetModel || "parakeet-tdt-0.6b-v3"+ : whisperModel || "base",+ language,+ };+ }++ if (transcriptionMode === "openwhispr") {+ const provider = managedProviders?.[0] ?? DEFAULT_MANAGED_PROVIDER;+ return {+ provider: `${provider.id}-realtime`,+ model: resolveModel(provider, selectedModel),+ mode: "openwhispr",+ language,+ };+ }++ if (transcriptionMode === "self-hosted") {+ throw new Error(+ "Self-hosted realtime transcription is not supported for Note Recording. Choose Local or Cloud Providers."+ );+ }++ if (transcriptionMode !== "providers") {+ throw new Error(`Unsupported Note Recording transcription mode: ${transcriptionMode}`);+ }++ const provider = byokProviders.find((candidate) => candidate.id === selectedProvider);+ if (!provider) {+ throw new Error(`Unsupported Note Recording provider: ${selectedProvider || "none selected"}`);+ }++ const options = {+ provider: `${provider.id}-realtime`,+ model: resolveModel(provider, selectedModel),+ mode: "byok",+ language,+ };++ if (provider.id === "corti") {+ return {+ ...options,+ environment: cortiEnvironment,+ tenant: cortiTenant,+ keyterms,+ };+ }++ return options;+}diff --git a/src/helpers/openaiRealtimeStreaming.js b/src/helpers/openaiRealtimeStreaming.jsindex e4aefcb68..88f6b2965 100644--- a/src/helpers/openaiRealtimeStreaming.js+++ b/src/helpers/openaiRealtimeStreaming.js@@ -39,7 +39,9 @@ class OpenAIRealtimeStreaming {this.onError = null;this.onSessionEnd = null;this.onSessionExpired = null;+ this.onConnectionLost = null;this._sessionExpired = false;+ this._connectionLossNotified = false;this._sessionTimer = null;this.pendingResolve = null;this.pendingReject = null;@@ -92,6 +94,7 @@ class OpenAIRealtimeStreaming {this.audioBytesSent = 0;this.speechStartedAt = null;this._sessionExpired = false;+ this._connectionLossNotified = false;const url = "wss://api.openai.com/v1/realtime?intent=transcription";debugLogger.debug("OpenAI Realtime connecting", { model: this.model });@@ -129,6 +132,7 @@ class OpenAIRealtimeStreaming {});this.ws.on("error", (error) => {+ const wasActive = this.isConnected;debugLogger.error("OpenAI Realtime WebSocket error", { error: error.message });this.isConnecting = false;this.cleanup();@@ -137,7 +141,11 @@ class OpenAIRealtimeStreaming {this.pendingReject = null;this.pendingResolve = null;}- this.onError?.(error);+ if (wasActive && !this.isDisconnecting) {+ this._notifyConnectionLost(error);+ } else if (!this.isDisconnecting) {+ this.onError?.(error);+ }});this.ws.on("close", (code, reason) => {@@ -156,6 +164,7 @@ class OpenAIRealtimeStreaming {this.cleanup();if (wasActive && !this.isDisconnecting && !this._sessionExpired) {this.onSessionEnd?.({ text: this.getFullTranscript() });+ this._notifyConnectionLost(new Error(`Connection lost (code: ${code})`));}});});@@ -257,7 +266,7 @@ class OpenAIRealtimeStreaming {if (errCode === "session_expired" && this.onSessionExpired) {debugLogger.warn("OpenAI Realtime session expired", { message: errMsg });this._sessionExpired = true;- this.onSessionExpired();+ this.onSessionExpired({ proactive: false });break;}const isEmptyBuffer =@@ -299,12 +308,22 @@ class OpenAIRealtimeStreaming {}}+ _notifyConnectionLost(error) {+ if (this._connectionLossNotified) return;+ this._connectionLossNotified = true;+ if (this.onConnectionLost) {+ this.onConnectionLost(error);+ } else {+ this.onError?.(error);+ }+ }+_startSessionTimer() {clearTimeout(this._sessionTimer);this._sessionTimer = setTimeout(() => {if (!this.isConnected) return;debugLogger.debug("OpenAI Realtime session approaching 60min limit, requesting reconnect");- this.onSessionExpired?.();+ this.onSessionExpired?.({ proactive: true });}, SESSION_PREEMPT_MS);}@@ -401,7 +420,7 @@ class OpenAIRealtimeStreaming {return true;}- async disconnect() {+ async disconnect({ commit = true } = {}) {debugLogger.debug("OpenAI Realtime disconnect", {audioBytesSent: this.audioBytesSent,segments: this.completedSegments.length,@@ -421,7 +440,7 @@ class OpenAIRealtimeStreaming {}if (this.ws.readyState === WebSocket.OPEN) {- if (this.audioBytesSent > 0) {+ if (commit && this.audioBytesSent > 0) {const prevOnFinal = this.onFinalTranscript;const prevOnError = this.onError;diff --git a/src/helpers/tinfoilRealtimeStreaming.js b/src/helpers/tinfoilRealtimeStreaming.jsindex c1f4312e8..c0e34fa75 100644--- a/src/helpers/tinfoilRealtimeStreaming.js+++ b/src/helpers/tinfoilRealtimeStreaming.js@@ -4,24 +4,38 @@ const { createTinfoilRealtimeSocket } = require("./tinfoilSecureClient");const debugLogger = require("./debugLogger");const TINFOIL_REALTIME_MODEL = "voxtral-mini-4b-realtime";-const COMMIT_POLL_MS = 250;-// Mirrors the silence_duration_ms the OpenAI path configures in server_vad.const COMMIT_IDLE_MS = 600;-// Bounds the text lost on abrupt socket death and keeps segments flowing to diarization.-const MAX_UTTERANCE_MS = 30000;-const COMMIT_ACK_TIMEOUT_MS = 10000;-// Mirrors the base class's disconnect timeout so a dead server can't hang the stop.+const MAX_TURN_MS = 30000;+const COMMIT_ACK_TIMEOUT_MS = 20000;const COMMIT_DRAIN_TIMEOUT_MS = 3000;+const SPEECH_RMS_THRESHOLD = 0.003;+const SPEECH_PEAK_THRESHOLD = 0.02;-// Matches the empty-buffer signatures the base class tolerates at disconnect.const isEmptyBufferError = (error) =>error?.code === "input_audio_buffer_commit_empty" ||(error?.message || "").includes("buffer too small") ||(error?.message || "").includes("commit_empty");-// Tinfoil's transcription intent has no server VAD, so utterance boundaries-// are decided client-side from the delta stream and committed explicitly.+const hasSpeechEnergy = (pcmBuffer) => {+ if (!pcmBuffer?.length) return false;+ const samples = new Int16Array(+ pcmBuffer.buffer,+ pcmBuffer.byteOffset,+ Math.floor(pcmBuffer.byteLength / 2)+ );+ let sumSquares = 0;+ let peak = 0;+ for (let index = 0; index < samples.length; index += 1) {+ const sample = samples[index] / 0x7fff;+ sumSquares += sample * sample;+ peak = Math.max(peak, Math.abs(sample));+ }+ return (+ Math.sqrt(sumSquares / samples.length) >= SPEECH_RMS_THRESHOLD && peak >= SPEECH_PEAK_THRESHOLD+ );+};+class TinfoilRealtimeStreaming extends OpenAIRealtimeStreaming {constructor(createSocketImpl = createTinfoilRealtimeSocket) {super();@@ -31,12 +45,13 @@ class TinfoilRealtimeStreaming extends OpenAIRealtimeStreaming {this._commitSentAt = 0;this._commitDrain = null;this._commitDrainResolve = null;- this._lastDeltaAt = 0;- this._emptyCommitStreak = 0;+ this._turnStartedAt = 0;+ this._lastSpeechAt = 0;+ this._turnSpeechStartedAt = 0;+ this._turnHasAudio = false;+ this._recoveryRequested = false;}- // Audio must only ever reach the attested Tinfoil socket, so a- // caller-supplied factory is overridden, never honored.async connect(options = {}) {const { apiKey } = options;const model = options.model || TINFOIL_REALTIME_MODEL;@@ -47,54 +62,58 @@ class TinfoilRealtimeStreaming extends OpenAIRealtimeStreaming {});}+ sendAudio(pcmBuffer) {+ const buffer = Buffer.isBuffer(pcmBuffer) ? pcmBuffer : Buffer.from(pcmBuffer);+ const accepted = super.sendAudio(buffer);+ if ((!accepted && !this.bufferingAudio) || !buffer.length) return accepted;++ const now = Date.now();+ this._turnHasAudio = true;+ if (!this._turnStartedAt) this._turnStartedAt = now;+ if (hasSpeechEnergy(buffer)) {+ this._lastSpeechAt = now;+ if (!this._turnSpeechStartedAt) this._turnSpeechStartedAt = now;+ if (!this._commitInFlight && !this.speechStartedAt) {+ this.speechStartedAt = this._turnSpeechStartedAt;+ }+ }+ return accepted;+ }+handleMessage(data) {let event;try {event = JSON.parse(data.toString());} catch {- // Malformed frames are logged by the base handler.super.handleMessage(data);return;}- if (event.type === "conversation.item.input_audio_transcription.delta") {- // Only real text moves the idle clock; without server VAD the first delta- // of an utterance is also the speech-start signal for mic suppression.- if (event.delta) {- this._lastDeltaAt = Date.now();- if (!this.speechStartedAt) this.speechStartedAt = this._lastDeltaAt;- }- } else if (event.type === "conversation.item.input_audio_transcription.completed") {- this._emptyCommitStreak = 0;- this._settleCommit();- } else if (event.type === "error") {- const wasTimerCommit = this._commitInFlight;++ if (event.type === "conversation.item.input_audio_transcription.completed") {+ const nextTurnSpeechStartedAt = this._turnSpeechStartedAt;+ this._recoveryRequested = false;this._settleCommit();- if (wasTimerCommit && isEmptyBufferError(event.error)) {- this._handleEmptyCommitReply();- return;+ super.handleMessage(data);+ if (nextTurnSpeechStartedAt && !this.speechStartedAt) {+ this.speechStartedAt = nextTurnSpeechStartedAt;}+ return;}- super.handleMessage(data);- }- // An empty-buffer reply to a timer commit is benign (a lost completed or a- // double commit); surfacing it would flash an error mid-meeting.- _handleEmptyCommitReply() {- this._emptyCommitStreak += 1;- // Restart the idle window so a late completed can land before any retry.- this._lastDeltaAt = Date.now();- if (this._emptyCommitStreak >= 2) {- // A second empty reply means this text can't finalize; drop it rather- // than loop commits forever. A late completed would still deliver it.- debugLogger.error("Tinfoil Realtime utterance could not be finalized, dropping partial", {- partialLength: this.currentPartial.length,- });- this.currentPartial = "";- this.speechStartedAt = null;- this._emptyCommitStreak = 0;+ if (event.type === "error" && this._commitInFlight) {+ const error = event.error || {};+ this._settleCommit();+ if (isEmptyBufferError(error)) {+ this.currentPartial = "";+ this.speechStartedAt = this._turnSpeechStartedAt || null;+ return;+ }+ super.handleMessage(data);+ this._requestRecovery(new Error(error.message || "Tinfoil commit failed"), false);return;}- debugLogger.debug("Tinfoil Realtime empty commit reply, awaiting late completed");++ super.handleMessage(data);}_markConnected() {@@ -108,70 +127,108 @@ class TinfoilRealtimeStreaming extends OpenAIRealtimeStreaming {}_stopCommitTimer() {- if (this._commitTimer) {- clearInterval(this._commitTimer);- this._commitTimer = null;- }+ if (!this._commitTimer) return;+ clearInterval(this._commitTimer);+ this._commitTimer = null;}_commitTick() {if (this.isDisconnecting || this.ws?.readyState !== WebSocket.OPEN) return;const now = Date.now();if (this._commitInFlight) {- // A commit the server never answered must not block the session forever.if (now - this._commitSentAt >= COMMIT_ACK_TIMEOUT_MS) {- debugLogger.debug("Tinfoil Realtime commit unanswered, re-arming");- this._settleCommit();+ this._requestRecovery(new Error("Tinfoil did not finish the current transcript segment."));}return;}- // Committing before any delta would finalize an empty or silent buffer.- if (!this.currentPartial) return;- const idleFor = now - this._lastDeltaAt;- const utteranceAge = now - (this.speechStartedAt || now);- if (idleFor < COMMIT_IDLE_MS && utteranceAge < MAX_UTTERANCE_MS) return;+ if (!this._turnHasAudio) return;++ const turnAge = now - this._turnStartedAt;+ const speechEnded = this._lastSpeechAt > 0 && now - this._lastSpeechAt >= COMMIT_IDLE_MS;+ if (!speechEnded && turnAge < MAX_TURN_MS) return;+ this._sendCommit();+ }++ _sendCommit() {+ if (this._commitInFlight || !this._turnHasAudio || this.ws?.readyState !== WebSocket.OPEN) {+ return false;+ }+this._commitInFlight = true;- this._commitSentAt = now;+ this._commitSentAt = Date.now();this._commitDrain = new Promise((resolve) => {this._commitDrainResolve = resolve;});try {this.ws.send(JSON.stringify({ type: "input_audio_buffer.commit" }));- } catch (err) {+ this._resetTurn();+ return true;+ } catch (error) {this._settleCommit();- debugLogger.debug("Tinfoil Realtime commit send failed", { error: err.message });+ debugLogger.debug("Tinfoil Realtime commit send failed", { error: error.message });+ return false;}}+ _requestRecovery(error, surfaceError = true) {+ if (this._recoveryRequested) return;+ this._recoveryRequested = true;+ debugLogger.warn("Tinfoil Realtime connection recovery requested", {+ error: error.message,+ });+ if (this.onConnectionLost) {+ this._notifyConnectionLost(error);+ } else if (surfaceError) {+ this.onError?.(error);+ }+ }++ _resetTurn() {+ this._turnStartedAt = 0;+ this._lastSpeechAt = 0;+ this._turnSpeechStartedAt = 0;+ this._turnHasAudio = false;+ }+_settleCommit() {this._commitInFlight = false;- if (this._commitDrainResolve) {- this._commitDrainResolve();- this._commitDrainResolve = null;- this._commitDrain = null;- }+ this._commitSentAt = 0;+ this._commitDrainResolve?.();+ this._commitDrainResolve = null;+ this._commitDrain = null;}- // The base flush resolves on the first completed, so a commit the timer- // already sent must finalize first or its tail transcript is lost.async disconnect() {this._stopCommitTimer();- if (this._commitInFlight && this._commitDrain) {- let timer;- await Promise.race([- this._commitDrain,- new Promise((resolve) => {- timer = setTimeout(resolve, COMMIT_DRAIN_TIMEOUT_MS);- }),- ]);- clearTimeout(timer);+ const deadline = Date.now() + COMMIT_DRAIN_TIMEOUT_MS;+ if (!(await this._waitForCommitDrain(deadline))) {+ return super.disconnect({ commit: false });}- return super.disconnect();+ if (!this._commitInFlight) this._sendCommit();+ await this._waitForCommitDrain(deadline);+ return super.disconnect({ commit: false });+ }++ async _waitForCommitDrain(deadline) {+ if (!this._commitInFlight || !this._commitDrain) return true;+ const remaining = Math.max(0, deadline - Date.now());+ if (remaining === 0) return false;+ let timeout;+ const drained = await Promise.race([+ this._commitDrain.then(() => true),+ new Promise((resolve) => {+ timeout = setTimeout(() => resolve(false), remaining);+ }),+ ]);+ clearTimeout(timeout);+ return drained;}cleanup() {this._stopCommitTimer();this._settleCommit();+ this._resetTurn();+ this._recoveryRequested = false;super.cleanup();}}diff --git a/src/stores/meetingRecordingStore.ts b/src/stores/meetingRecordingStore.tsindex 713690d0e..ef68d67b4 100644--- a/src/stores/meetingRecordingStore.ts+++ b/src/stores/meetingRecordingStore.ts@@ -1,6 +1,8 @@import { create } from "zustand";import { getSettings, selectResolvedMeetingTranscription } from "./settingsStore";import { useStreamingProvidersStore } from "./streamingProvidersStore";+import { getStreamingTranscriptionProviders } from "../models/ModelRegistry";+import { resolveMeetingTranscriptionOptions } from "../helpers/meetingTranscriptionRouting";import { isBuiltInMicrophone } from "../utils/audioDeviceUtils";import {followsSystemDefaultMic,@@ -122,62 +124,20 @@ const getMeetingTranscriptionOptions = () => {const resolved = selectResolvedMeetingTranscription(state);const language = getBaseLanguageCode(state.preferredLanguage);- if (resolved.useLocalWhisper) {- return {- provider: "local" as const,- localProvider: resolved.localTranscriptionProvider,- localModel:- resolved.localTranscriptionProvider === "nvidia"- ? resolved.parakeetModel || "parakeet-tdt-0.6b-v3"- : resolved.whisperModel || "base",- language,- };- }-- // Corti (BYOK) streams over its own WSS — independent of the server-driven catalog.- const selectedProvider =- state.meetingCloudTranscriptionProvider || state.cloudTranscriptionProvider;- if (resolved.cloudTranscriptionMode === "byok" && selectedProvider === "corti") {- return {- provider: "corti-realtime" as const,- model: "corti-transcribe",- mode: "byok" as const,- language,- environment: state.cortiEnvironment,- tenant: state.cortiTenant,- keyterms: (state.customDictionary ?? []).filter(Boolean),- };- }-- // Tinfoil (BYOK) streams over its own attested WSS — same catalog independence as Corti.- if (resolved.cloudTranscriptionMode === "byok" && selectedProvider === "tinfoil") {- return {- provider: "tinfoil-realtime" as const,- model: "voxtral-mini-4b-realtime",- mode: "byok" as const,- language,- };- }-- const catalog = useStreamingProvidersStore.getState().providers;- const provider =- catalog?.find((p) => p.id === resolved.cloudTranscriptionProvider) ?? catalog?.[0];- const byokKeyAvailable = provider?.id === "openai" ? !!state.openaiApiKey : true;- const mode =- resolved.cloudTranscriptionMode === "byok" && byokKeyAvailable ? "byok" : "openwhispr";- if (!provider) {- logger.debug(- "Streaming providers catalog not loaded, falling back to OpenAI default",- {},- "meeting"- );- return { provider: "openai-realtime" as const, model: "gpt-4o-mini-transcribe", mode };- }- const model =- provider.models.find((m) => m.id === resolved.cloudTranscriptionModel)?.id ??- provider.models.find((m) => m.default)?.id ??- provider.models[0]?.id;- return { provider: `${provider.id}-realtime` as const, model, mode };+ return resolveMeetingTranscriptionOptions({+ transcriptionMode: resolved.transcriptionMode,+ language,+ localProvider: resolved.localTranscriptionProvider,+ whisperModel: resolved.whisperModel,+ parakeetModel: resolved.parakeetModel,+ selectedProvider: resolved.cloudTranscriptionProvider,+ selectedModel: resolved.cloudTranscriptionModel,+ byokProviders: getStreamingTranscriptionProviders(),+ managedProviders: useStreamingProvidersStore.getState().providers,+ cortiEnvironment: state.cortiEnvironment,+ cortiTenant: state.cortiTenant,+ keyterms: (state.customDictionary ?? []).filter(Boolean),+ });};const stopMediaStream = (stream: MediaStream | null) => {@@ -1075,6 +1035,17 @@ export async function startRecording(args: StartRecordingArgs): Promise<boolean>});if (errorCleanup) ipcCleanups.push(errorCleanup);+ const fatalErrorCleanup = window.electronAPI?.onMeetingTranscriptionFatalError?.((err) => {+ reportMeetingError(err);+ logger.error(+ "Meeting transcription stopped after connection loss",+ { error: err },+ "meeting"+ );+ if (isRecordingFlag) void stopRecording();+ });+ if (fatalErrorCleanup) ipcCleanups.push(fatalErrorCleanup);+if (startResult.oneOnOneAttendee) {const synthetic: SpeakerIdentification = {speakerId: "speaker_0",diff --git a/src/stores/settingsStore.ts b/src/stores/settingsStore.tsindex 59e00ca43..878057ebc 100644--- a/src/stores/settingsStore.ts+++ b/src/stores/settingsStore.ts@@ -3,7 +3,6 @@ import { API_ENDPOINTS } from "../config/constants";import i18n, { normalizeUiLanguage } from "../i18n";import { ensureAgentNameInDictionary } from "../utils/agentName";import { chooseDictionaryStartupAction } from "../helpers/dictionaryStartup";-import { useStreamingProvidersStore } from "./streamingProvidersStore";import logger from "../utils/logger";import whisperVadConstants from "../constants/whisperVad.json";import type {@@ -2025,27 +2024,21 @@ export interface ResolvedMeetingTranscription {export const selectResolvedMeetingTranscription = (state: SettingsState-): ResolvedMeetingTranscription => {- const catalog = useStreamingProvidersStore.getState().providers;- // TODO(1.8.0): Catalog has one cloud entry today (OpenAI Realtime).- // When a second is added, resolve as `meetingCloudTranscriptionProvider || cloudTranscriptionProvider || catalog[0]?.id`, then validate against catalog.- const cloudTranscriptionProvider = catalog?.[0]?.id ?? "";-- return {- useLocalWhisper: state.meetingUseLocalWhisper,- whisperModel: state.meetingWhisperModel || state.whisperModel,- localTranscriptionProvider: state.meetingLocalTranscriptionProvider,- parakeetModel: state.meetingParakeetModel || state.parakeetModel,- cloudTranscriptionProvider,- cloudTranscriptionModel: state.meetingCloudTranscriptionModel || state.cloudTranscriptionModel,- cloudTranscriptionBaseUrl:- state.meetingCloudTranscriptionBaseUrl || state.cloudTranscriptionBaseUrl || "",- cloudTranscriptionMode: state.meetingCloudTranscriptionMode || state.cloudTranscriptionMode,- transcriptionMode: state.meetingTranscriptionMode,- remoteTranscriptionType: state.meetingRemoteTranscriptionType,- remoteTranscriptionUrl: state.meetingRemoteTranscriptionUrl || state.remoteTranscriptionUrl,- };-};+): ResolvedMeetingTranscription => ({+ useLocalWhisper: state.meetingUseLocalWhisper,+ whisperModel: state.meetingWhisperModel || state.whisperModel,+ localTranscriptionProvider: state.meetingLocalTranscriptionProvider,+ parakeetModel: state.meetingParakeetModel || state.parakeetModel,+ cloudTranscriptionProvider:+ state.meetingCloudTranscriptionProvider || state.cloudTranscriptionProvider,+ cloudTranscriptionModel: state.meetingCloudTranscriptionModel || state.cloudTranscriptionModel,+ cloudTranscriptionBaseUrl:+ state.meetingCloudTranscriptionBaseUrl || state.cloudTranscriptionBaseUrl || "",+ cloudTranscriptionMode: state.meetingCloudTranscriptionMode || state.cloudTranscriptionMode,+ transcriptionMode: state.meetingTranscriptionMode,+ remoteTranscriptionType: state.meetingRemoteTranscriptionType,+ remoteTranscriptionUrl: state.meetingRemoteTranscriptionUrl || state.remoteTranscriptionUrl,+});export interface ResolvedUploadTranscription {useLocalWhisper: boolean;diff --git a/src/types/electron.ts b/src/types/electron.tsindex 5b230c404..6369a3c28 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -2214,6 +2214,7 @@ declare global {) => void) => () => void;onMeetingTranscriptionError?: (callback: (error: string) => void) => () => void;+ onMeetingTranscriptionFatalError?: (callback: (error: string) => void) => () => void;// Speaker diarizationdownloadDiarizationModels?: () => Promise<{ success: boolean; error?: string }>;diff --git a/test/helpers/meetingStreamingProviders.test.js b/test/helpers/meetingStreamingProviders.test.jsindex ccbd4a263..801b7d4f9 100644--- a/test/helpers/meetingStreamingProviders.test.js+++ b/test/helpers/meetingStreamingProviders.test.js@@ -5,7 +5,8 @@ const load = () => import("../../src/helpers/meetingStreamingProviders.js");test("tinfoil-realtime resolves to the Tinfoil client, never the OpenAI default", async () => {const { STREAMING_CLIENT_BY_PROVIDER } = await load();- const { TinfoilRealtimeStreaming } = await import("../../src/helpers/tinfoilRealtimeStreaming.js");+ const { TinfoilRealtimeStreaming } =+ await import("../../src/helpers/tinfoilRealtimeStreaming.js");const OpenAIRealtimeStreaming = (await import("../../src/helpers/openaiRealtimeStreaming.js")).default;@@ -40,3 +41,32 @@ test("allow-list rejects unknown and batch-only providers", async () => {assert.equal(ALLOWED_MEETING_PROVIDERS.has(provider), false, `${provider} must be rejected`);}});++test("client lookup fails closed instead of defaulting to OpenAI", async () => {+ const { getMeetingStreamingClient } = await load();++ assert.throws(() => getMeetingStreamingClient("unknown-realtime"), /Unsupported meeting/);+});++test("connection identity includes every provider-specific option", async () => {+ const { getMeetingConnectionKey } = await load();+ const options = {+ provider: "corti-realtime",+ model: "corti-transcribe",+ language: "en",+ mode: "byok",+ environment: "us",+ tenant: "tenant-a",+ keyterms: ["OpenWhispr"],+ };++ assert.equal(getMeetingConnectionKey(options), getMeetingConnectionKey({ ...options }));+ assert.notEqual(+ getMeetingConnectionKey(options),+ getMeetingConnectionKey({ ...options, provider: "tinfoil-realtime" })+ );+ assert.notEqual(+ getMeetingConnectionKey(options),+ getMeetingConnectionKey({ ...options, tenant: "tenant-b" })+ );+});diff --git a/test/helpers/meetingTranscriptionRouting.test.js b/test/helpers/meetingTranscriptionRouting.test.jsnew file mode 100644index 000000000..35d17a229--- /dev/null+++ b/test/helpers/meetingTranscriptionRouting.test.js@@ -0,0 +1,165 @@+const test = require("node:test");+const assert = require("node:assert/strict");++const load = () => import("../../src/helpers/meetingTranscriptionRouting.js");++const byokProviders = [+ {+ id: "openai",+ models: [{ id: "gpt-4o-mini-transcribe" }, { id: "gpt-4o-transcribe" }],+ },+ { id: "corti", models: [{ id: "corti-transcribe" }] },+ { id: "tinfoil", models: [{ id: "voxtral-mini-4b-realtime" }] },+];++const baseOptions = {+ transcriptionMode: "providers",+ language: "en",+ localProvider: "whisper",+ whisperModel: "small",+ parakeetModel: "parakeet-tdt-0.6b-v3",+ selectedProvider: "tinfoil",+ selectedModel: "voxtral-mini-4b-realtime",+ byokProviders,+ managedProviders: [+ {+ id: "assemblyai",+ models: [{ id: "universal-streaming", default: true }],+ },+ ],+ cortiEnvironment: "us",+ cortiTenant: "tenant",+ keyterms: ["OpenWhispr"],+};++test("providers mode routes Tinfoil through its realtime client", async () => {+ const { resolveMeetingTranscriptionOptions } = await load();++ assert.deepEqual(resolveMeetingTranscriptionOptions(baseOptions), {+ provider: "tinfoil-realtime",+ model: "voxtral-mini-4b-realtime",+ mode: "byok",+ language: "en",+ });+});++test("BYOK OpenAI never downgrades to managed cloud when its key is unavailable", async () => {+ const { resolveMeetingTranscriptionOptions } = await load();++ assert.deepEqual(+ resolveMeetingTranscriptionOptions({+ ...baseOptions,+ selectedProvider: "openai",+ selectedModel: "gpt-4o-mini-transcribe",+ }),+ {+ provider: "openai-realtime",+ model: "gpt-4o-mini-transcribe",+ mode: "byok",+ language: "en",+ }+ );+});++test("self-hosted mode never follows a stale Tinfoil provider", async () => {+ const { resolveMeetingTranscriptionOptions } = await load();++ assert.throws(+ () =>+ resolveMeetingTranscriptionOptions({+ ...baseOptions,+ transcriptionMode: "self-hosted",+ }),+ /Self-hosted realtime transcription is not supported/+ );+});++test("local mode wins over stale cloud provider state", async () => {+ const { resolveMeetingTranscriptionOptions } = await load();++ assert.deepEqual(+ resolveMeetingTranscriptionOptions({+ ...baseOptions,+ transcriptionMode: "local",+ localProvider: "nvidia",+ parakeetModel: "nemotron-speech-streaming-en-0.6b",+ }),+ {+ provider: "local",+ localProvider: "nvidia",+ localModel: "nemotron-speech-streaming-en-0.6b",+ language: "en",+ }+ );+});++test("managed mode ignores stale BYOK state", async () => {+ const { resolveMeetingTranscriptionOptions } = await load();++ assert.deepEqual(+ resolveMeetingTranscriptionOptions({+ ...baseOptions,+ transcriptionMode: "openwhispr",+ }),+ {+ provider: "assemblyai-realtime",+ model: "universal-streaming",+ mode: "openwhispr",+ language: "en",+ }+ );+});++test("managed mode keeps its established OpenAI default before the catalog loads", async () => {+ const { resolveMeetingTranscriptionOptions } = await load();++ assert.deepEqual(+ resolveMeetingTranscriptionOptions({+ ...baseOptions,+ transcriptionMode: "openwhispr",+ managedProviders: null,+ }),+ {+ provider: "openai-realtime",+ model: "gpt-4o-mini-transcribe",+ mode: "openwhispr",+ language: "en",+ }+ );+});++test("Corti keeps the meeting-specific connection settings", async () => {+ const { resolveMeetingTranscriptionOptions } = await load();++ assert.deepEqual(+ resolveMeetingTranscriptionOptions({+ ...baseOptions,+ selectedProvider: "corti",+ selectedModel: "corti-transcribe",+ }),+ {+ provider: "corti-realtime",+ model: "corti-transcribe",+ mode: "byok",+ language: "en",+ environment: "us",+ tenant: "tenant",+ keyterms: ["OpenWhispr"],+ }+ );+});++test("unknown and custom providers fail closed", async () => {+ const { resolveMeetingTranscriptionOptions } = await load();++ for (const selectedProvider of ["custom", "groq", "", undefined]) {+ assert.throws(+ () =>+ resolveMeetingTranscriptionOptions({+ ...baseOptions,+ selectedProvider,+ }),+ /Unsupported Note Recording provider/+ );+ }+});diff --git a/test/helpers/streamingConnectionLifecycle.test.js b/test/helpers/streamingConnectionLifecycle.test.jsnew file mode 100644index 000000000..50f1c1e15--- /dev/null+++ b/test/helpers/streamingConnectionLifecycle.test.js@@ -0,0 +1,40 @@+const test = require("node:test");+const assert = require("node:assert/strict");++const AssemblyAiStreaming = require("../../src/helpers/assemblyAiStreaming");+const CortiStreaming = require("../../src/helpers/cortiStreaming");+const DeepgramStreaming = require("../../src/helpers/deepgramStreaming");+const OpenAIRealtimeStreaming = require("../../src/helpers/openaiRealtimeStreaming");++const clients = [+ ["OpenAI", OpenAIRealtimeStreaming, "_notifyConnectionLost"],+ ["AssemblyAI", AssemblyAiStreaming, "notifyConnectionLost"],+ ["Deepgram", DeepgramStreaming, "notifyConnectionLost"],+ ["Corti", CortiStreaming, "notifyConnectionLost"],+];++for (const [name, StreamingClient, notifyMethod] of clients) {+ test(`${name} reports one recoverable connection loss to meetings`, () => {+ const streaming = new StreamingClient();+ const recoveries = [];+ const errors = [];+ streaming.onConnectionLost = (error) => recoveries.push(error.message);+ streaming.onError = (error) => errors.push(error.message);++ streaming[notifyMethod](new Error("socket failed"));+ streaming[notifyMethod](new Error("duplicate close"));++ assert.deepEqual(recoveries, ["socket failed"]);+ assert.deepEqual(errors, []);+ });++ test(`${name} preserves the existing error path outside meetings`, () => {+ const streaming = new StreamingClient();+ const errors = [];+ streaming.onError = (error) => errors.push(error.message);++ streaming[notifyMethod](new Error("socket failed"));++ assert.deepEqual(errors, ["socket failed"]);+ });+}diff --git a/test/helpers/tinfoilRealtimeStreaming.test.js b/test/helpers/tinfoilRealtimeStreaming.test.jsindex afee9b46f..5075feec3 100644--- a/test/helpers/tinfoilRealtimeStreaming.test.js+++ b/test/helpers/tinfoilRealtimeStreaming.test.js@@ -4,8 +4,6 @@ const { EventEmitter } = require("node:events");const WS = require("ws");const load = () => import("../../src/helpers/tinfoilRealtimeStreaming.js");--const DELTA_EVENT = "conversation.item.input_audio_transcription.delta";const COMPLETED_EVENT = "conversation.item.input_audio_transcription.completed";function makeFakeSocket(readyState) {@@ -13,7 +11,6 @@ function makeFakeSocket(readyState) {socket.readyState = readyState;socket.sent = [];socket.send = (data) => socket.sent.push(data);- // A healthy connection answers pings, or the keep-alive terminates it mid-test.socket.ping = () => socket.emit("pong");socket.terminate = () => {socket.readyState = WS.CLOSED;@@ -27,19 +24,17 @@ function makeFakeSocket(readyState) {function makeRecordingFactory(socket) {const calls = [];- const factory = async (args) => {- calls.push(args);- return socket;+ return {+ calls,+ factory: async (args) => {+ calls.push(args);+ return socket;+ },};- return { factory, calls };}-function sentCommits(socket) {- return socket.sent.filter((raw) => JSON.parse(raw).type === "input_audio_buffer.commit");-}--function delta(text) {- return JSON.stringify({ type: DELTA_EVENT, delta: text });+function sentEvents(socket, type) {+ return socket.sent.map((raw) => JSON.parse(raw)).filter((event) => event.type === type);}function completed(transcript) {@@ -53,8 +48,14 @@ function commitEmptyError() {});}-// Wires a connected instance directly, mirroring the existing realtime test-// style: cadence behavior is driven through handleMessage + mocked timers.+function speechPcm(sampleCount = 480, amplitude = 2000) {+ const samples = new Int16Array(sampleCount);+ for (let index = 0; index < samples.length; index += 1) {+ samples[index] = index % 2 === 0 ? amplitude : -amplitude;+ }+ return Buffer.from(samples.buffer);+}+async function makeConnected() {const { TinfoilRealtimeStreaming } = await load();const streaming = new TinfoilRealtimeStreaming();@@ -63,116 +64,92 @@ async function makeConnected() {return streaming;}-// -- connect: attested socket factory --+async function finishConnect(streaming, socket, preconfigured = false) {+ await new Promise((resolve) => setImmediate(resolve));+ socket.readyState = WS.OPEN;+ socket.emit("message", JSON.stringify({ type: "session.created" }));+ if (!preconfigured) {+ socket.emit("message", JSON.stringify({ type: "session.updated" }));+ }+}-test("connect() passes the configured model and key to the attested socket factory", async () => {+test("connect uses only the attested socket factory with the selected model and key", async () => {const { TinfoilRealtimeStreaming } = await load();const socket = makeFakeSocket(WS.CONNECTING);const { factory, calls } = makeRecordingFactory(socket);const streaming = new TinfoilRealtimeStreaming(factory);+ let rogueCalled = false;- const connected = streaming.connect({ apiKey: "tk-secret", model: "custom-rt-model" });- await new Promise((resolve) => setImmediate(resolve));- socket.readyState = WS.OPEN;- socket.emit("message", JSON.stringify({ type: "session.created" }));- socket.emit("message", JSON.stringify({ type: "session.updated" }));+ const connected = streaming.connect({+ apiKey: "tk-secret",+ model: "custom-rt-model",+ createSocket: async () => {+ rogueCalled = true;+ return makeFakeSocket(WS.OPEN);+ },+ });+ await finishConnect(streaming, socket);await connected;+ assert.equal(rogueCalled, false);assert.deepEqual(calls, [{ model: "custom-rt-model", apiKey: "tk-secret" }]);streaming.cleanup();});-test("connect() defaults the model to voxtral-mini-4b-realtime when the caller omits one", async () => {+test("connect defaults to the supported Tinfoil realtime model", async () => {const { TinfoilRealtimeStreaming, TINFOIL_REALTIME_MODEL } = await load();const socket = makeFakeSocket(WS.CONNECTING);const { factory, calls } = makeRecordingFactory(socket);const streaming = new TinfoilRealtimeStreaming(factory);const connected = streaming.connect({ apiKey: "tk-secret" });- await new Promise((resolve) => setImmediate(resolve));- socket.readyState = WS.OPEN;- socket.emit("message", JSON.stringify({ type: "session.created" }));- socket.emit("message", JSON.stringify({ type: "session.updated" }));+ await finishConnect(streaming, socket);await connected;- assert.equal(calls[0].model, TINFOIL_REALTIME_MODEL);assert.equal(TINFOIL_REALTIME_MODEL, "voxtral-mini-4b-realtime");+ assert.equal(calls[0].model, TINFOIL_REALTIME_MODEL);streaming.cleanup();});-test("connect() overrides a caller-supplied createSocket with the attested factory", async () => {- const { TinfoilRealtimeStreaming } = await load();- const socket = makeFakeSocket(WS.CONNECTING);- const { factory, calls } = makeRecordingFactory(socket);- const streaming = new TinfoilRealtimeStreaming(factory);- let rogueCalled = false;-- const connected = streaming.connect({- apiKey: "tk-secret",- createSocket: async () => {- rogueCalled = true;- return makeFakeSocket(WS.OPEN);- },- });- await new Promise((resolve) => setImmediate(resolve));- socket.readyState = WS.OPEN;- socket.emit("message", JSON.stringify({ type: "session.created" }));- socket.emit("message", JSON.stringify({ type: "session.updated" }));- await connected;-- assert.equal(rogueCalled, false, "audio must only travel over the attested socket");- assert.equal(calls.length, 1);- streaming.cleanup();-});--test("the default socket factory is the attested Tinfoil transport", async () => {+test("the default factory is the pinned Tinfoil transport", async () => {const { TinfoilRealtimeStreaming } = await load();const { createTinfoilRealtimeSocket } = require("../../src/helpers/tinfoilSecureClient");- const streaming = new TinfoilRealtimeStreaming();-- assert.equal(streaming._createSocketImpl, createTinfoilRealtimeSocket);+ assert.equal(new TinfoilRealtimeStreaming()._createSocketImpl, createTinfoilRealtimeSocket);});-test("factory failure rejects connect() without constructing any fallback socket", async () => {+test("attestation failure rejects without opening a fallback socket", async () => {const { TinfoilRealtimeStreaming } = await load();const streaming = new TinfoilRealtimeStreaming(async () => {throw new Error("attestation failed");});await assert.rejects(() => streaming.connect({ apiKey: "tk-secret" }), /attestation failed/);-- assert.equal(streaming.ws, null, "no socket of any kind may exist after a factory failure");- assert.equal(streaming.isConnecting, false);+ assert.equal(streaming.ws, null);assert.equal(streaming.isConnected, false);});-test("each fresh instance re-invokes the factory (meeting reconnect swaps instances)", async () => {+test("every fresh instance invokes the attested factory", async () => {const { TinfoilRealtimeStreaming } = await load();const calls = [];- for (let i = 0; i < 2; i++) {+ for (let index = 0; index < 2; index += 1) {const socket = makeFakeSocket(WS.CONNECTING);const streaming = new TinfoilRealtimeStreaming(async (args) => {calls.push(args);return socket;});const connected = streaming.connect({ apiKey: "tk-secret" });- await new Promise((resolve) => setImmediate(resolve));- socket.readyState = WS.OPEN;- socket.emit("message", JSON.stringify({ type: "session.created" }));- socket.emit("message", JSON.stringify({ type: "session.updated" }));+ await finishConnect(streaming, socket);await connected;streaming.cleanup();}- assert.equal(calls.length, 2, "every connect must dial a fresh attested socket");+ assert.equal(calls.length, 2);assert.deepEqual(calls[0], calls[1]);});-// -- session configuration ----test("connect() sends the GA transcription session.update declaring the 24kHz meeting rate", async () => {+test("meeting sessions declare the 24kHz PCM input format", async () => {const { TinfoilRealtimeStreaming } = await load();const socket = makeFakeSocket(WS.CONNECTING);const { factory } = makeRecordingFactory(socket);@@ -183,10 +160,7 @@ test("connect() sends the GA transcription session.update declaring the 24kHz mesocket.readyState = WS.OPEN;socket.emit("message", JSON.stringify({ type: "session.created" }));- assert.equal(socket.sent.length, 1);- const update = JSON.parse(socket.sent[0]);- assert.equal(update.type, "session.update");- assert.equal(update.session.type, "transcription");+ const [update] = sentEvents(socket, "session.update");assert.deepEqual(update.session.audio.input.format, { type: "audio/pcm", rate: 24000 });assert.equal(update.session.audio.input.transcription.model, "voxtral-mini-4b-realtime");@@ -195,224 +169,194 @@ test("connect() sends the GA transcription session.update declaring the 24kHz mestreaming.cleanup();});-test("preconfigured sessions get no session.update (dictation semantics survive subclassing)", async () => {+test("preconfigured sessions do not send a client session update", async () => {const { TinfoilRealtimeStreaming } = await load();const socket = makeFakeSocket(WS.CONNECTING);const { factory } = makeRecordingFactory(socket);const streaming = new TinfoilRealtimeStreaming(factory);const connected = streaming.connect({ apiKey: "tk-secret", preconfigured: true });- await new Promise((resolve) => setImmediate(resolve));- socket.readyState = WS.OPEN;- socket.emit("message", JSON.stringify({ type: "session.created" }));+ await finishConnect(streaming, socket, true);await connected;- assert.equal(socket.sent.length, 0, "session.update must not override a preconfigured session");+ assert.equal(sentEvents(socket, "session.update").length, 0);streaming.cleanup();});-// -- client-side utterance commits (Tinfoil has no server VAD) ----test("commits after the delta stream goes idle for the silence window", (t) => {+test("speech followed by 600ms of PCM silence commits the turn", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();+ streaming.sendAudio(speechPcm());- streaming.handleMessage(delta("hello"));t.mock.timers.tick(500);- assert.equal(sentCommits(streaming.ws).length, 0, "no commit while under the idle window");+ streaming.sendAudio(Buffer.alloc(960));+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 0);t.mock.timers.tick(250);- const commits = sentCommits(streaming.ws);- assert.equal(commits.length, 1);- assert.deepEqual(JSON.parse(commits[0]), { type: "input_audio_buffer.commit" });+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);})();});-test("keeps streaming without commits while deltas keep arriving", (t) => {+test("continuous speech is bounded by a 30-second turn", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();- for (let i = 0; i < 8; i++) {- streaming.handleMessage(delta("word "));+ for (let index = 0; index < 119; index += 1) {+ streaming.sendAudio(speechPcm());t.mock.timers.tick(250);}+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 0);- assert.equal(sentCommits(streaming.ws).length, 0, "an active utterance must not be split");+ streaming.sendAudio(speechPcm());+ t.mock.timers.tick(250);+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);})();});-test("commits a continuous utterance at the max-utterance cap", (t) => {+test("a silent backend turn is recycled before the provider zombie window", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();+ streaming.sendAudio(Buffer.alloc(960));- for (let i = 0; i < 118; i++) {- streaming.handleMessage(delta("word "));- t.mock.timers.tick(250);- }- assert.equal(sentCommits(streaming.ws).length, 0, "cap must not fire before 30s");-- for (let i = 0; i < 4; i++) {- streaming.handleMessage(delta("word "));- t.mock.timers.tick(250);- }- assert.equal(sentCommits(streaming.ws).length, 1, "cap must bound an endless utterance");+ t.mock.timers.tick(29750);+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 0);+ t.mock.timers.tick(250);+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);})();});-test("never commits while nothing has been transcribed (empty-buffer guard)", (t) => {+test("no audio produces no commits", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();-- t.mock.timers.tick(5000);-- assert.equal(streaming.ws.sent.length, 0, "silence must never produce a commit");+ t.mock.timers.tick(60000);+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 0);})();});-test("one commit in flight at a time; completed re-arms the next utterance", (t) => {+test("completed re-arms the next buffered turn", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();const finals = [];streaming.onFinalTranscript = (text) => finals.push(text);- streaming.handleMessage(delta("first utterance"));- t.mock.timers.tick(850);- assert.equal(sentCommits(streaming.ws).length, 1);-- t.mock.timers.tick(850);- assert.equal(sentCommits(streaming.ws).length, 1, "no second commit before the server answers");-- streaming.handleMessage(completed("first utterance"));- streaming.handleMessage(delta("second utterance"));- t.mock.timers.tick(850);- assert.equal(sentCommits(streaming.ws).length, 2, "next utterance must commit again");- assert.deepEqual(finals, ["first utterance"], "finalized text must still reach the consumer");- })();-});--test("an unanswered commit re-arms after the ack deadline instead of stalling the session", (t) => {- t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });- return (async () => {- const streaming = await makeConnected();+ streaming.sendAudio(speechPcm());+ t.mock.timers.tick(750);+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);- streaming.handleMessage(delta("hello"));- t.mock.timers.tick(850);- assert.equal(sentCommits(streaming.ws).length, 1);+ streaming.sendAudio(speechPcm());+ t.mock.timers.tick(750);+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);- t.mock.timers.tick(10250);- assert.equal(sentCommits(streaming.ws).length, 2, "session must self-heal after a lost commit");+ streaming.handleMessage(completed("first"));+ t.mock.timers.tick(250);+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 2);+ assert.deepEqual(finals, ["first"]);})();});-test("first delta stamps speechStartedAt for the mic-suppression window", (t) => {+test("commit timeout requests recovery without sending a duplicate commit", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();- assert.equal(streaming.speechStartedAt, null);+ const errors = [];+ streaming.onConnectionLost = (error) => errors.push(error.message);- t.mock.timers.tick(1000);- streaming.handleMessage(delta("hello"));- assert.equal(streaming.speechStartedAt, 101000);+ streaming.sendAudio(speechPcm());+ t.mock.timers.tick(750);+ t.mock.timers.tick(20250);- streaming.handleMessage(completed("hello"));- assert.equal(streaming.speechStartedAt, null, "utterance boundary must reset the stamp");+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);+ assert.deepEqual(errors, ["Tinfoil did not finish the current transcript segment."]);})();});-test("cleanup() stops the commit timer so a dead instance never commits", (t) => {+test("speech timing comes from PCM activity rather than transcript timing", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();- const socket = streaming.ws;-- streaming.handleMessage(delta("hello"));- streaming.cleanup();- assert.equal(streaming._commitTimer, null);- t.mock.timers.tick(5000);- assert.equal(sentCommits(socket).length, 0);+ streaming.sendAudio(Buffer.alloc(960));+ assert.equal(streaming.speechStartedAt, null);+ t.mock.timers.tick(250);+ streaming.sendAudio(speechPcm());+ assert.equal(streaming.speechStartedAt, 100250);})();});-// -- disconnect drain: an in-flight timer commit must finalize before the flush ----test("disconnect() drains the in-flight timer commit so the tail transcript still lands", (t) => {+test("disconnect drains an in-flight commit before committing the tail", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();const socket = streaming.ws;- streaming.sendAudio(Buffer.alloc(480));- streaming.handleMessage(delta("first utterance"));- t.mock.timers.tick(850);- assert.equal(sentCommits(socket).length, 1, "timer commit in flight");- streaming.handleMessage(delta("tail spoken after the commit"));+ streaming.sendAudio(speechPcm());+ t.mock.timers.tick(750);+ streaming.sendAudio(speechPcm());const disconnected = streaming.disconnect();await new Promise((resolve) => setImmediate(resolve));- assert.equal(sentCommits(socket).length, 1, "flush must wait for the in-flight commit");+ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 1);- streaming.handleMessage(completed("first utterance"));+ streaming.handleMessage(completed("first"));await new Promise((resolve) => setImmediate(resolve));- assert.equal(sentCommits(socket).length, 2, "flush commit follows the drained completed");+ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 2);- streaming.handleMessage(completed("tail spoken after the commit"));- const result = await disconnected;- assert.equal(result.text, "first utterance tail spoken after the commit");+ streaming.handleMessage(completed("tail"));+ assert.deepEqual(await disconnected, { text: "first tail" });})();});-test("disconnect() with an unanswered commit resolves within the bounded deadlines", (t) => {+test("disconnect remains bounded when the provider never completes", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();- streaming.sendAudio(Buffer.alloc(480));- streaming.handleMessage(delta("hello"));- t.mock.timers.tick(850);- assert.equal(sentCommits(streaming.ws).length, 1);+ streaming.sendAudio(speechPcm());+ t.mock.timers.tick(750);const disconnected = streaming.disconnect();await new Promise((resolve) => setImmediate(resolve));t.mock.timers.tick(3000);await new Promise((resolve) => setImmediate(resolve));- t.mock.timers.tick(3000);- const result = await disconnected;- assert.equal(result.text, "", "resolves with accumulated text instead of hanging");++ assert.deepEqual(await disconnected, { text: "" });})();});-// -- empty-commit replies: benign for timer commits, bounded against loops ----test("empty-buffer reply to a timer commit is swallowed and retried after a fresh idle window", (t) => {+test("disconnect uses one drain deadline across the active turn and its tail", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();- let surfaced = 0;- streaming.onError = () => {- surfaced += 1;- };+ const socket = streaming.ws;+ streaming.sendAudio(speechPcm());+ t.mock.timers.tick(750);+ streaming.sendAudio(speechPcm());++ let finished = false;+ const disconnected = streaming.disconnect().then((result) => {+ finished = true;+ return result;+ });- streaming.handleMessage(delta("hello"));- t.mock.timers.tick(850);- assert.equal(sentCommits(streaming.ws).length, 1);+ t.mock.timers.tick(2000);+ streaming.handleMessage(completed("first"));+ await new Promise((resolve) => setImmediate(resolve));+ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 2);- streaming.handleMessage(commitEmptyError());- assert.equal(surfaced, 0, "benign empty reply must not reach the meeting error channel");- assert.equal(streaming._commitInFlight, false);+ t.mock.timers.tick(999);+ await new Promise((resolve) => setImmediate(resolve));+ assert.equal(finished, false);- t.mock.timers.tick(250);- assert.equal(sentCommits(streaming.ws).length, 1, "no instant re-commit before the idle window");- t.mock.timers.tick(600);- assert.equal(sentCommits(streaming.ws).length, 2, "retry follows a full idle window");+ t.mock.timers.tick(1);+ assert.deepEqual(await disconnected, { text: "first" });})();});-test("a second empty-buffer reply drops the unfinalizable partial instead of looping commits", (t) => {+test("empty commit replies are swallowed without a retry loop", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();@@ -421,85 +365,46 @@ test("a second empty-buffer reply drops the unfinalizable partial instead of loosurfaced += 1;};- streaming.handleMessage(delta("hello"));- t.mock.timers.tick(850);+ streaming.sendAudio(speechPcm());+ t.mock.timers.tick(750);streaming.handleMessage(commitEmptyError());- t.mock.timers.tick(850);- assert.equal(sentCommits(streaming.ws).length, 2, "one retry after the first empty reply");-- streaming.handleMessage(commitEmptyError());- assert.equal(surfaced, 0);- assert.equal(streaming.currentPartial, "", "unfinalizable partial dropped");- assert.equal(streaming.speechStartedAt, null);-t.mock.timers.tick(5000);- assert.equal(sentCommits(streaming.ws).length, 2, "no commit loop after dropping the partial");- })();-});--test("empty-buffer error with no timer commit in flight still reaches onError", (t) => {- t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });- return (async () => {- const streaming = await makeConnected();- let errMsg = null;- streaming.onError = (err) => {- errMsg = err.message;- };-- streaming.handleMessage(commitEmptyError());- assert.equal(errMsg, "buffer too small", "base disconnect-flush tolerance must still see it");+ assert.equal(surfaced, 0);+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);})();});-test("server errors during an in-flight commit still surface", (t) => {+test("provider errors surface and request connection recovery", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();- let errMsg = null;- streaming.onError = (err) => {- errMsg = err.message;- };+ const errors = [];+ const recoveries = [];+ streaming.onError = (error) => errors.push(error.message);+ streaming.onConnectionLost = (error) => recoveries.push(error.message);- streaming.handleMessage(delta("hello"));- t.mock.timers.tick(850);+ streaming.sendAudio(speechPcm());+ t.mock.timers.tick(750);streaming.handleMessage(- JSON.stringify({ type: "error", error: { code: "server_error", message: "something broke" } })+ JSON.stringify({ type: "error", error: { code: "server_error", message: "failed" } }));- assert.equal(errMsg, "something broke");- assert.equal(streaming._commitInFlight, false);+ assert.deepEqual(errors, ["failed"]);+ assert.deepEqual(recoveries, ["failed"]);})();});-// -- empty delta frames must not touch the utterance timers ----test("an empty delta frame does not stamp speechStartedAt", (t) => {+test("cleanup stops the turn timer", (t) => {t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });return (async () => {const streaming = await makeConnected();+ const socket = streaming.ws;+ streaming.sendAudio(speechPcm());+ streaming.cleanup();- streaming.handleMessage(delta(""));- assert.equal(streaming.speechStartedAt, null, "an empty frame is not speech");-- t.mock.timers.tick(250);- streaming.handleMessage(delta("real"));- assert.equal(streaming.speechStartedAt, 100250);- })();-});--test("empty delta frames do not starve the idle commit", (t) => {- t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });- return (async () => {- const streaming = await makeConnected();-- streaming.handleMessage(delta("hi"));- t.mock.timers.tick(250);- streaming.handleMessage(delta(""));- t.mock.timers.tick(250);- streaming.handleMessage(delta(""));- t.mock.timers.tick(250);-- assert.equal(sentCommits(streaming.ws).length, 1, "empty frames must not reset the idle clock");+ t.mock.timers.tick(5000);+ assert.equal(streaming._commitTimer, null);+ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 0);})();});