Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(meetings): harden realtime provider routing

Gabriel Steinf67f2a6b1cdf
diff --git a/preload.js b/preload.js
index 4e4125228..121c14e9f 100644
--- a/preload.js
+++ b/preload.js
@@ -732,6 +732,10 @@ contextBridge.exposeInMainWorld("electronAPI", {
"meeting-transcription-error",
(callback) => (_event, data) => callback(data)
),
+ onMeetingTranscriptionFatalError: registerListener(
+ "meeting-transcription-fatal-error",
+ (callback) => (_event, data) => callback(data)
+ ),
// Dictation realtime streaming
dictationRealtimeWarmup: (options) => ipcRenderer.invoke("dictation-realtime-warmup", options),
diff --git a/src/components/TranscriptionModelPicker.tsx b/src/components/TranscriptionModelPicker.tsx
index 9d1f7641d..08a2ae1e9 100644
--- a/src/components/TranscriptionModelPicker.tsx
+++ b/src/components/TranscriptionModelPicker.tsx
@@ -408,7 +408,7 @@ export default function TranscriptionModelPicker({
);
const cloudProviderTabs = useMemo(() => {
const availableIds = new Set(availableCloudProviders.map((p) => p.id));
- availableIds.add("custom");
+ if (!streamingOnly) availableIds.add("custom");
return CLOUD_PROVIDER_TABS.filter((p) => availableIds.has(p.id)).map((provider) => {
const named =
provider.id === "custom"
@@ -418,7 +418,7 @@ export default function TranscriptionModelPicker({
? named
: { ...named, disabled: true, disabledLabel: t("common.managedByOrg") };
});
- }, [availableCloudProviders, providerAllowed, t]);
+ }, [availableCloudProviders, providerAllowed, streamingOnly, t]);
useEffect(() => {
selectedLocalModelRef.current = selectedLocalModel;
diff --git a/src/components/settings/MeetingSettings.tsx b/src/components/settings/MeetingSettings.tsx
index dc36dd3be..a7a2a6a25 100644
--- a/src/components/settings/MeetingSettings.tsx
+++ b/src/components/settings/MeetingSettings.tsx
@@ -7,7 +7,6 @@ import { InferenceModeSelector, SettingsRow } from "../ui/SettingsSection";
import type { InferenceModeOption } from "../ui/SettingsSection";
import { Toggle } from "../ui/toggle";
import TranscriptionModelPicker from "../TranscriptionModelPicker";
-import SelfHostedPanel from "../SelfHostedPanel";
import type { InferenceMode } from "../../types/electron";
import { useStartOnboarding } from "../../hooks/useStartOnboarding";
@@ -50,8 +49,6 @@ export function MeetingTranscriptionPanel() {
meetingCloudTranscriptionBaseUrl,
setMeetingCloudTranscriptionBaseUrl,
setMeetingCloudTranscriptionMode,
- meetingRemoteTranscriptionUrl,
- setMeetingRemoteTranscriptionUrl,
} = useSettingsStore();
const { modes: transcriptionModes, isModeAllowed } = usePolicyModeOptions<InferenceModeOption>(
[
@@ -80,6 +77,8 @@ export function MeetingTranscriptionPanel() {
label: t("settingsPage.transcription.modes.selfHosted"),
description: t("settingsPage.transcription.modes.selfHostedDesc"),
icon: <Network className="w-4 h-4" />,
+ disabled: true,
+ badge: t("settings.meeting.comingSoon"),
},
],
"transcription"
@@ -87,6 +86,7 @@ export function MeetingTranscriptionPanel() {
const handleTranscriptionModeSelect = (mode: InferenceMode) => {
if (!isModeAllowed(mode)) return;
+ if (mode === "self-hosted") return;
if (mode === "openwhispr" && !isSignedIn) {
startOnboarding();
return;
@@ -140,18 +140,6 @@ export function MeetingTranscriptionPanel() {
{meetingTranscriptionMode === "providers" && renderTranscriptionPicker("cloud")}
{meetingTranscriptionMode === "local" && renderTranscriptionPicker("local")}
- {meetingTranscriptionMode === "self-hosted" && (
- <>
- <SelfHostedPanel
- service="transcription"
- url={meetingRemoteTranscriptionUrl}
- onUrlChange={setMeetingRemoteTranscriptionUrl}
- />
- <p className="text-xs text-muted-foreground/80 px-1">
- {t("settingsPage.speechToText.selfHostedStreamingNote")}
- </p>
- </>
- )}
<MeetingSpeakerDetectionRow />
</div>
);
diff --git a/src/helpers/assemblyAiStreaming.js b/src/helpers/assemblyAiStreaming.js
index 0bae9b1d3..3a6b79988 100644
--- a/src/helpers/assemblyAiStreaming.js
+++ b/src/helpers/assemblyAiStreaming.js
@@ -21,6 +21,8 @@ class AssemblyAiStreaming {
this.onFinalTranscript = null;
this.onError = null;
this.onSessionEnd = null;
+ this.onConnectionLost = null;
+ this.connectionLossNotified = false;
this.pendingResolve = null;
this.pendingReject = null;
this.connectionTimeout = null;
@@ -245,6 +247,7 @@ class AssemblyAiStreaming {
this.ws = this.warmConnection;
this.isConnected = true;
+ this.connectionLossNotified = false;
this.sessionId = this.warmSessionId || null;
this.warmConnection = null;
this.warmConnectionReady = false;
@@ -257,9 +260,10 @@ class AssemblyAiStreaming {
this.ws.removeAllListeners("error");
this.ws.on("error", (error) => {
+ const wasActive = this.isConnected;
debugLogger.error("AssemblyAI WebSocket error", { error: error.message });
this.cleanup();
- this.onError?.(error);
+ if (wasActive && !this.isDisconnecting) this.notifyConnectionLost(error);
});
this.ws.removeAllListeners("close");
@@ -272,7 +276,7 @@ class AssemblyAiStreaming {
});
this.cleanup();
if (wasActive && !this.isDisconnecting) {
- this.onError?.(new Error(`Connection lost (code: ${code})`));
+ this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));
}
});
@@ -318,6 +322,7 @@ class AssemblyAiStreaming {
this.accumulatedText = "";
this.lastTurnText = "";
this.turns = [];
+ this.connectionLossNotified = false;
// Try to use pre-warmed connection for instant start
if (this.hasWarmConnection()) {
@@ -350,6 +355,7 @@ class AssemblyAiStreaming {
});
this.ws.on("error", (error) => {
+ const wasActive = this.isConnected;
debugLogger.error("AssemblyAI WebSocket error", { error: error.message });
this.cleanup();
if (this.pendingReject) {
@@ -357,7 +363,11 @@ class AssemblyAiStreaming {
this.pendingReject = null;
this.pendingResolve = null;
}
- this.onError?.(error);
+ if (wasActive && !this.isDisconnecting) {
+ this.notifyConnectionLost(error);
+ } else if (!this.isDisconnecting) {
+ this.onError?.(error);
+ }
});
this.ws.on("close", (code, reason) => {
@@ -374,12 +384,22 @@ class AssemblyAiStreaming {
}
this.cleanup();
if (wasActive && !this.isDisconnecting) {
- this.onError?.(new Error(`Connection lost (code: ${code})`));
+ this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));
}
});
});
}
+ notifyConnectionLost(error) {
+ if (this.connectionLossNotified) return;
+ this.connectionLossNotified = true;
+ if (this.onConnectionLost) {
+ this.onConnectionLost(error);
+ } else {
+ this.onError?.(error);
+ }
+ }
+
handleMessage(data) {
try {
const message = JSON.parse(data.toString());
diff --git a/src/helpers/cortiStreaming.js b/src/helpers/cortiStreaming.js
index d7dd8b21f..d835581ac 100644
--- a/src/helpers/cortiStreaming.js
+++ b/src/helpers/cortiStreaming.js
@@ -17,6 +17,8 @@ class CortiStreaming {
this.onFinalTranscript = null;
this.onError = null;
this.onSessionEnd = null;
+ this.onConnectionLost = null;
+ this.connectionLossNotified = false;
this.pendingResolve = null;
this.pendingReject = null;
this.connectionTimeout = null;
@@ -73,6 +75,7 @@ class CortiStreaming {
this.accumulatedText = "";
this.completedSegments = [];
this.configAccepted = false;
+ this.connectionLossNotified = false;
this.preConfigBuffer = [];
this.preConfigBufferSize = 0;
this.audioBytesSent = 0;
@@ -113,6 +116,7 @@ class CortiStreaming {
});
ws.on("error", (error) => {
+ const wasActive = this.isConnected;
debugLogger.error("Corti WebSocket error", { error: error.message });
this.cleanup();
if (this.pendingReject) {
@@ -120,7 +124,11 @@ class CortiStreaming {
this.pendingReject = null;
this.pendingResolve = null;
}
- this.onError?.(error);
+ if (wasActive && !this.isDisconnecting) {
+ this.notifyConnectionLost(error);
+ } else if (!this.isDisconnecting) {
+ this.onError?.(error);
+ }
});
ws.on("close", (code, reason) => {
@@ -138,11 +146,21 @@ class CortiStreaming {
this.resolvePendingAck();
this.cleanup();
if (wasActive && !this.isDisconnecting) {
- this.onError?.(new Error(`Connection lost (code: ${code})`));
+ this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));
}
});
}
+ notifyConnectionLost(error) {
+ if (this.connectionLossNotified) return;
+ this.connectionLossNotified = true;
+ if (this.onConnectionLost) {
+ this.onConnectionLost(error);
+ } else {
+ this.onError?.(error);
+ }
+ }
+
// Pre-open the socket and finish the config handshake before recording, so the
// first words aren't lost waiting for CONFIG_ACCEPTED. Mirrors Deepgram/AssemblyAI.
async warmup(options = {}) {
diff --git a/src/helpers/deepgramStreaming.js b/src/helpers/deepgramStreaming.js
index e4a0f9731..3275c107a 100644
--- a/src/helpers/deepgramStreaming.js
+++ b/src/helpers/deepgramStreaming.js
@@ -77,6 +77,8 @@ class DeepgramStreaming {
this.onFinalTranscript = null;
this.onError = null;
this.onSessionEnd = null;
+ this.onConnectionLost = null;
+ this.connectionLossNotified = false;
this.pendingResolve = null;
this.pendingReject = null;
this.connectionTimeout = null;
@@ -419,6 +421,7 @@ class DeepgramStreaming {
this.ws = this.warmConnection;
this.isConnected = true;
+ this.connectionLossNotified = false;
this.sessionId = this.warmSessionId || null;
this.warmConnection = null;
this.warmConnectionReady = false;
@@ -431,9 +434,10 @@ class DeepgramStreaming {
this.ws.removeAllListeners("error");
this.ws.on("error", (error) => {
+ const wasActive = this.isConnected;
debugLogger.error("Deepgram WebSocket error", { error: error.message });
this.cleanup();
- this.onError?.(error);
+ if (wasActive && !this.isDisconnecting) this.notifyConnectionLost(error);
});
this.ws.removeAllListeners("close");
@@ -449,7 +453,7 @@ class DeepgramStreaming {
}
this.cleanup();
if (wasActive && !this.isDisconnecting) {
- this.onError?.(new Error(`Connection lost (code: ${code})`));
+ this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));
}
});
@@ -562,6 +566,7 @@ class DeepgramStreaming {
this.finalSegments = [];
this.audioBytesSent = 0;
this.resultsReceived = 0;
+ this.connectionLossNotified = false;
if (replayBuffer && replayBuffer.length > 0) {
this.coldStartBuffer = replayBuffer;
@@ -608,6 +613,7 @@ class DeepgramStreaming {
});
this.ws.on("error", (error) => {
+ const wasActive = this.isConnected;
debugLogger.error("Deepgram WebSocket error", { error: error.message });
// Invalidate cached token on auth failure so next attempt fetches fresh
if (error.message && error.message.includes("401")) {
@@ -620,7 +626,11 @@ class DeepgramStreaming {
this.pendingReject = null;
this.pendingResolve = null;
}
- this.onError?.(error);
+ if (wasActive && !this.isDisconnecting) {
+ this.notifyConnectionLost(error);
+ } else if (!this.isDisconnecting) {
+ this.onError?.(error);
+ }
});
this.ws.on("close", (code, reason) => {
@@ -640,12 +650,22 @@ class DeepgramStreaming {
}
this.cleanup();
if (wasActive && !this.isDisconnecting) {
- this.onError?.(new Error(`Connection lost (code: ${code})`));
+ this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));
}
});
});
}
+ notifyConnectionLost(error) {
+ if (this.connectionLossNotified) return;
+ this.connectionLossNotified = true;
+ if (this.onConnectionLost) {
+ this.onConnectionLost(error);
+ } else {
+ this.onError?.(error);
+ }
+ }
+
handleMessage(data) {
try {
const message = JSON.parse(data.toString());
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index caa064539..99e2004e1 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -64,13 +64,15 @@ const {
} = require("./whisperVadConfig");
const {
- STREAMING_CLIENT_BY_PROVIDER,
ALLOWED_MEETING_PROVIDERS,
+ getMeetingStreamingClient,
+ getMeetingConnectionKey,
} = require("./meetingStreamingProviders");
// Meeting capture runs at 24 kHz (see meetingRecordingStore AudioContext); cloud
// streaming providers must be told the true PCM rate or they misread the audio.
const MEETING_STREAM_SAMPLE_RATE = 24000;
+const MEETING_RECONNECT_BUFFER_MAX_BYTES = MEETING_STREAM_SAMPLE_RATE * 2 * 30;
function parseAttendees(raw) {
if (!raw) return [];
@@ -5424,119 +5426,202 @@ class IPCHandlers {
streaming.onError = (error) => {
send("meeting-transcription-error", error.message);
};
- streaming.onSessionExpired = () => reconnectMeetingStreams();
+ const recoverConnection = async (error, restoreOldOnFailure) => {
+ let recovered = false;
+ try {
+ recovered = await reconnectMeetingStreams({ restoreOldOnFailure });
+ } catch (reconnectError) {
+ debugLogger.error("Meeting stream recovery failed unexpectedly", {
+ error: reconnectError.message,
+ });
+ }
+ if (!recovered && !meetingFatalErrorSent) {
+ meetingFatalErrorSent = true;
+ send(
+ "meeting-transcription-fatal-error",
+ error?.message || "Meeting transcription connection could not be restored."
+ );
+ }
+ };
+ streaming.onConnectionLost = (error) => {
+ void recoverConnection(error, false);
+ };
+ streaming.onSessionExpired = ({ proactive = false } = {}) => {
+ void recoverConnection(
+ new Error("Meeting transcription session could not be renewed."),
+ proactive
+ );
+ };
};
- const reconnectMeetingStreams = async () => {
- if (meetingReconnecting || meetingLocalMode) return;
+ const resetMeetingReconnectAudio = () => {
+ meetingReconnectAudioBuffers = { mic: [], system: [] };
+ meetingReconnectAudioBytes = { mic: 0, system: 0 };
+ meetingReconnectReplaySources = new Set();
+ };
- const options = meetingConnectionOptions;
- const win = meetingConnectionWin;
- if (!options || !win || win.isDestroyed()) {
- debugLogger.error("Cannot reconnect meeting streams: missing connection context");
- return;
+ const queueMeetingReconnectAudio = (source, buffer) => {
+ if (!meetingReconnectReplaySources.has(source)) return;
+ const copy = Buffer.from(buffer);
+ const queue = meetingReconnectAudioBuffers[source];
+ queue.push(copy);
+ meetingReconnectAudioBytes[source] += copy.length;
+ while (
+ meetingReconnectAudioBytes[source] > MEETING_RECONNECT_BUFFER_MAX_BYTES &&
+ queue.length > 1
+ ) {
+ meetingReconnectAudioBytes[source] -= queue.shift().length;
}
+ };
- if (meetingReconnectCount >= MAX_MEETING_RECONNECTS) {
- debugLogger.error("Meeting reconnect limit reached", { count: meetingReconnectCount });
- win.webContents.send(
- "meeting-transcription-error",
- "Session reconnect limit reached. Please stop and restart the recording."
- );
- return;
- }
+ const replayMeetingReconnectAudio = (source, streaming) => {
+ if (!meetingReconnectReplaySources.has(source)) return true;
+ const queue = meetingReconnectAudioBuffers[source];
+ const replayed = queue.every((buffer) => streaming.sendAudio(buffer));
+ debugLogger.info("Replayed meeting audio after reconnect", {
+ source,
+ chunks: queue.length,
+ bytes: meetingReconnectAudioBytes[source],
+ });
+ return replayed;
+ };
- meetingReconnecting = true;
- meetingReconnectCount++;
+ const reconnectMeetingStreams = ({ restoreOldOnFailure = false } = {}) => {
+ if (meetingReconnectPromise) return meetingReconnectPromise;
- const StreamingClass =
- STREAMING_CLIENT_BY_PROVIDER[options.provider] ?? OpenAIRealtimeStreaming;
+ const pending = (async () => {
+ if (meetingLocalMode) return false;
- const oldMic = this._meetingMicStreaming;
- const oldSystem = this._meetingSystemStreaming;
+ const options = meetingConnectionOptions;
+ const win = meetingConnectionWin;
+ if (!options || !win || win.isDestroyed()) {
+ debugLogger.error("Cannot reconnect meeting streams: missing connection context");
+ return false;
+ }
- // Swap fresh instances in before the token fetch so audio arriving during
- // the swap lands in their pre-connect buffers instead of a dead socket.
- const newMic = new StreamingClass();
- newMic.beginConnecting?.();
- attachMeetingStreamingHandlers(newMic, win, "mic");
- this._meetingMicStreaming = newMic;
- let newSystem = null;
- if (oldSystem) {
- newSystem = new StreamingClass();
- newSystem.beginConnecting?.();
- attachMeetingStreamingHandlers(newSystem, win, "system");
- this._meetingSystemStreaming = newSystem;
- }
+ if (meetingReconnectCount >= MAX_MEETING_RECONNECTS) {
+ debugLogger.error("Meeting reconnect limit reached", { count: meetingReconnectCount });
+ return false;
+ }
- debugLogger.info("Reconnecting meeting streams", {
- attempt: meetingReconnectCount,
- maxAttempts: MAX_MEETING_RECONNECTS,
- });
+ meetingReconnectCount++;
- const tokenEvent = { sender: win.webContents };
- try {
- const connectOpts = {
- model: options.model,
- language: options.language,
- preconfigured: options.mode !== "byok",
- environment: options.environment,
- tenant: options.tenant,
- keyterms: options.keyterms,
- sampleRate: MEETING_STREAM_SAMPLE_RATE,
- };
+ const oldMic = this._meetingMicStreaming;
+ const oldSystem = this._meetingSystemStreaming;
+ meetingReconnectReplaySources = new Set([
+ ...(!oldMic?.isConnected ? ["mic"] : []),
+ ...(oldSystem && !oldSystem.isConnected ? ["system"] : []),
+ ]);
+ let newMic = null;
+ let newSystem = null;
- let pairs;
- if (newSystem) {
- const secrets = await fetchRealtimeToken(tokenEvent, options, { streams: 2 });
- pairs = [
- { streaming: newMic, secret: secrets[0] },
- { streaming: newSystem, secret: secrets[1] },
- ];
- } else {
- pairs = [{ streaming: newMic, secret: await fetchRealtimeToken(tokenEvent, options) }];
- }
+ try {
+ const StreamingClass = getMeetingStreamingClient(options.provider);
+ newMic = new StreamingClass();
+ attachMeetingStreamingHandlers(newMic, win, "mic");
+ if (oldSystem) {
+ newSystem = new StreamingClass();
+ attachMeetingStreamingHandlers(newSystem, win, "system");
+ }
- await Promise.all(
- pairs.map(({ streaming, secret }) =>
- streaming.connect({ apiKey: secret, token: secret, ...connectOpts })
- )
- );
+ debugLogger.info("Reconnecting meeting streams", {
+ attempt: meetingReconnectCount,
+ maxAttempts: MAX_MEETING_RECONNECTS,
+ });
+
+ const tokenEvent = { sender: win.webContents };
+ const connectOpts = {
+ model: options.model,
+ language: options.language,
+ preconfigured: options.mode !== "byok",
+ environment: options.environment,
+ tenant: options.tenant,
+ keyterms: options.keyterms,
+ sampleRate: MEETING_STREAM_SAMPLE_RATE,
+ };
- if (meetingConnectionOptions !== options) {
- // Recording stopped while the reconnect was in flight.
- for (const { streaming } of pairs) streaming.disconnect().catch(() => {});
+ let pairs;
+ if (newSystem) {
+ const secrets = await fetchRealtimeToken(tokenEvent, options, { streams: 2 });
+ pairs = [
+ { streaming: newMic, secret: secrets[0] },
+ { streaming: newSystem, secret: secrets[1] },
+ ];
+ } else {
+ pairs = [{ streaming: newMic, secret: await fetchRealtimeToken(tokenEvent, options) }];
+ }
+
+ await Promise.all(
+ pairs.map(({ streaming, secret }) =>
+ streaming.connect({ apiKey: secret, token: secret, ...connectOpts })
+ )
+ );
+
+ if (pairs.some(({ streaming }) => !streaming.isConnected)) {
+ throw new Error("Meeting transcription connection closed during reconnect.");
+ }
+
+ if (meetingConnectionOptions !== options) {
+ for (const { streaming } of pairs) streaming.disconnect().catch(() => {});
+ oldMic?.disconnect().catch(() => {});
+ oldSystem?.disconnect().catch(() => {});
+ resetMeetingReconnectAudio();
+ return true;
+ }
+
+ const replayedMic = replayMeetingReconnectAudio("mic", newMic);
+ const replayedSystem = !newSystem || replayMeetingReconnectAudio("system", newSystem);
+ if (!replayedMic || !replayedSystem) {
+ throw new Error("Meeting audio could not be restored after reconnect.");
+ }
+ this._meetingMicStreaming = newMic;
+ this._meetingSystemStreaming = newSystem;
+ resetMeetingReconnectAudio();
oldMic?.disconnect().catch(() => {});
oldSystem?.disconnect().catch(() => {});
- return;
- }
+ meetingConnectionKey = getMeetingConnectionKey(options);
- oldMic?.disconnect().catch(() => {});
- oldSystem?.disconnect().catch(() => {});
+ debugLogger.info("Meeting streams reconnected", { attempt: meetingReconnectCount });
+ meetingReconnectCount = 0;
+ return true;
+ } catch (error) {
+ debugLogger.error("Meeting stream reconnect failed", {
+ error: error.message,
+ attempt: meetingReconnectCount,
+ });
+ newMic?.disconnect().catch(() => {});
+ newSystem?.disconnect().catch(() => {});
+ if (meetingConnectionOptions !== options) {
+ oldMic?.disconnect().catch(() => {});
+ oldSystem?.disconnect().catch(() => {});
+ resetMeetingReconnectAudio();
+ return true;
+ }
- debugLogger.info("Meeting streams reconnected", { attempt: meetingReconnectCount });
- } catch (error) {
- debugLogger.error("Meeting stream reconnect failed", {
- error: error.message,
- attempt: meetingReconnectCount,
- });
- newMic.disconnect().catch(() => {});
- newSystem?.disconnect().catch(() => {});
- if (meetingConnectionOptions === options) {
- // A proactive reconnect leaves the old connections open; restore them
- // so transcription continues until the hard limit retries this path.
- this._meetingMicStreaming = oldMic;
- this._meetingSystemStreaming = oldSystem;
+ const canRestoreOld =
+ restoreOldOnFailure && !!oldMic?.isConnected && (!oldSystem || oldSystem.isConnected);
+ if (canRestoreOld) {
+ this._meetingMicStreaming = oldMic;
+ this._meetingSystemStreaming = oldSystem;
+ } else {
+ oldMic?.disconnect().catch(() => {});
+ oldSystem?.disconnect().catch(() => {});
+ this._meetingMicStreaming = null;
+ this._meetingSystemStreaming = null;
+ meetingConnectionKey = null;
+ }
+ resetMeetingReconnectAudio();
if (!win.isDestroyed()) {
win.webContents.send("meeting-transcription-error", error.message);
}
- } else {
- oldMic?.disconnect().catch(() => {});
- oldSystem?.disconnect().catch(() => {});
+ return canRestoreOld;
}
- } finally {
- meetingReconnecting = false;
- }
+ })();
+
+ meetingReconnectPromise = pending;
+ return pending.finally(() => {
+ if (meetingReconnectPromise === pending) meetingReconnectPromise = null;
+ });
};
const fetchRealtimeToken = async (event, options, { streams } = {}) => {
@@ -5633,6 +5718,10 @@ class IPCHandlers {
return streams === 2 ? [apiKey, apiKey] : apiKey;
}
+ if (options.provider !== "openai-realtime") {
+ throw new Error(`Unsupported realtime token provider: ${options.provider}`);
+ }
+
if (options.mode === "byok") {
const apiKey = this.environmentManager.getOpenAIKey();
if (!apiKey) throw new Error("No OpenAI API key configured. Add your key in Settings.");
@@ -5698,6 +5787,8 @@ class IPCHandlers {
(systemAudioMode === "unsupported" || !!this._meetingSystemStreaming?.isConnected);
const connectRealtimeStreaming = async (event, options) => {
+ const connectionKey = getMeetingConnectionKey(options);
+ const StreamingClass = getMeetingStreamingClient(options.provider);
if (this._meetingMicStreaming?.isConnected) {
await this._meetingMicStreaming.disconnect();
}
@@ -5735,18 +5826,30 @@ class IPCHandlers {
];
}
- const StreamingClass =
- STREAMING_CLIENT_BY_PROVIDER[options.provider] ?? OpenAIRealtimeStreaming;
for (const { ref, source } of pairs) {
this[ref] = new StreamingClass();
attachMeetingStreamingHandlers(this[ref], win, source);
}
- await Promise.all(
- pairs.map(({ ref, secret }) =>
- this[ref].connect({ apiKey: secret, token: secret, ...connectOpts })
- )
- );
+ try {
+ await Promise.all(
+ pairs.map(({ ref, secret }) =>
+ this[ref].connect({ apiKey: secret, token: secret, ...connectOpts })
+ )
+ );
+ if (pairs.some(({ ref }) => !this[ref]?.isConnected)) {
+ throw new Error("Meeting transcription connection closed during startup.");
+ }
+ meetingConnectionKey = connectionKey;
+ } catch (error) {
+ await Promise.all(
+ pairs.map(({ ref }) => this[ref]?.disconnect().catch(() => ({ text: "" })))
+ );
+ this._meetingMicStreaming = null;
+ this._meetingSystemStreaming = null;
+ meetingConnectionKey = null;
+ throw error;
+ }
return win;
};
@@ -5761,11 +5864,16 @@ class IPCHandlers {
let meetingStartedAt = null;
let meetingSendCounts = { mic: 0, system: 0 };
const meetingEchoLeakDetector = new MeetingEchoLeakDetector();
- let meetingReconnecting = false;
+ let meetingReconnectPromise = null;
+ let meetingFatalErrorSent = false;
let meetingReconnectCount = 0;
const MAX_MEETING_RECONNECTS = 5;
let meetingConnectionOptions = null;
let meetingConnectionWin = null;
+ let meetingConnectionKey = null;
+ let meetingReconnectAudioBuffers = { mic: [], system: [] };
+ let meetingReconnectAudioBytes = { mic: 0, system: 0 };
+ let meetingReconnectReplaySources = new Set();
const fs = require("fs");
let meetingDiarizationStream = null;
@@ -5923,6 +6031,7 @@ class IPCHandlers {
}
}
+ queueMeetingReconnectAudio(source, outbound);
const sent = streaming.sendAudio(outbound);
meetingSendCounts[source]++;
if (meetingSendCounts[source] <= 5 || meetingSendCounts[source] % 100 === 0) {
@@ -6488,10 +6597,13 @@ class IPCHandlers {
resetPendingMicFinals();
meetingAecEnabled = false;
meetingEchoLeakDetector.reset();
- meetingReconnecting = false;
+ meetingReconnectPromise = null;
+ meetingFatalErrorSent = false;
meetingReconnectCount = 0;
meetingConnectionOptions = null;
meetingConnectionWin = null;
+ meetingConnectionKey = null;
+ resetMeetingReconnectAudio();
};
const disconnectMeetingStreaming = async ({ flushPending = false } = {}) => {
@@ -6642,8 +6754,12 @@ class IPCHandlers {
}
const { mode: systemAudioMode } = await getMeetingSystemAudioPlan();
+ const requestedConnectionKey = getMeetingConnectionKey(options);
- if (isMeetingStreamingConnected(systemAudioMode)) {
+ if (
+ isMeetingStreamingConnected(systemAudioMode) &&
+ meetingConnectionKey === requestedConnectionKey
+ ) {
debugLogger.debug("Meeting transcription already prepared (warm connections)");
return { success: true, alreadyPrepared: true };
}
@@ -6700,10 +6816,12 @@ class IPCHandlers {
meetingConnectionOptions = options;
meetingConnectionWin = BrowserWindow.fromWebContents(event.sender);
meetingReconnectCount = 0;
+ meetingFatalErrorSent = false;
this.meetingDetectionEngine?.setUserRecording(true);
try {
const systemAudioPlan = await getMeetingSystemAudioPlan();
let { mode: systemAudioMode, strategy: systemAudioStrategy } = systemAudioPlan;
+ const requestedConnectionKey = getMeetingConnectionKey(options);
meetingEchoLeakDetector.reset();
meetingOneOnOneAttendee = resolveOneOnOneAttendeeForNote(options.noteId);
meetingOneOnOneProfileBound = false;
@@ -6721,7 +6839,11 @@ class IPCHandlers {
}
// If already prepared (warm connections from prepare), just re-attach handlers
- if (!meetingLocalMode && isMeetingStreamingConnected(systemAudioMode)) {
+ if (
+ !meetingLocalMode &&
+ isMeetingStreamingConnected(systemAudioMode) &&
+ meetingConnectionKey === requestedConnectionKey
+ ) {
debugLogger.debug("Meeting transcription start: reusing warm connections");
const win = BrowserWindow.fromWebContents(event.sender);
attachMeetingStreamingHandlers(this._meetingMicStreaming, win, "mic");
diff --git a/src/helpers/meetingStreamingProviders.js b/src/helpers/meetingStreamingProviders.js
index f14cf91f7..b88a35e40 100644
--- a/src/helpers/meetingStreamingProviders.js
+++ b/src/helpers/meetingStreamingProviders.js
@@ -16,4 +16,26 @@ const STREAMING_CLIENT_BY_PROVIDER = {
// class and silently fall through to the OpenAI default.
const ALLOWED_MEETING_PROVIDERS = new Set(["local", ...Object.keys(STREAMING_CLIENT_BY_PROVIDER)]);
-module.exports = { STREAMING_CLIENT_BY_PROVIDER, ALLOWED_MEETING_PROVIDERS };
+const getMeetingStreamingClient = (provider) => {
+ const StreamingClient = STREAMING_CLIENT_BY_PROVIDER[provider];
+ if (!StreamingClient) throw new Error(`Unsupported meeting streaming provider: ${provider}`);
+ return StreamingClient;
+};
+
+const getMeetingConnectionKey = (options = {}) =>
+ JSON.stringify({
+ provider: options.provider,
+ model: options.model,
+ language: options.language,
+ mode: options.mode,
+ environment: options.environment,
+ tenant: options.tenant,
+ keyterms: options.keyterms,
+ });
+
+module.exports = {
+ STREAMING_CLIENT_BY_PROVIDER,
+ ALLOWED_MEETING_PROVIDERS,
+ getMeetingStreamingClient,
+ getMeetingConnectionKey,
+};
diff --git a/src/helpers/meetingTranscriptionRouting.js b/src/helpers/meetingTranscriptionRouting.js
new file mode 100644
index 000000000..b060162a9
--- /dev/null
+++ b/src/helpers/meetingTranscriptionRouting.js
@@ -0,0 +1,79 @@
+const DEFAULT_MANAGED_PROVIDER = {
+ id: "openai",
+ models: [{ id: "gpt-4o-mini-transcribe", default: true }],
+};
+
+const resolveModel = (provider, selectedModel) =>
+ provider.models.find((model) => model.id === selectedModel)?.id ??
+ provider.models.find((model) => model.default)?.id ??
+ provider.models[0]?.id;
+
+export function resolveMeetingTranscriptionOptions({
+ transcriptionMode,
+ language,
+ localProvider,
+ whisperModel,
+ parakeetModel,
+ selectedProvider,
+ selectedModel,
+ byokProviders,
+ managedProviders,
+ cortiEnvironment,
+ cortiTenant,
+ keyterms,
+}) {
+ if (transcriptionMode === "local") {
+ return {
+ provider: "local",
+ localProvider,
+ localModel:
+ localProvider === "nvidia"
+ ? parakeetModel || "parakeet-tdt-0.6b-v3"
+ : whisperModel || "base",
+ language,
+ };
+ }
+
+ if (transcriptionMode === "openwhispr") {
+ const provider = managedProviders?.[0] ?? DEFAULT_MANAGED_PROVIDER;
+ return {
+ provider: `${provider.id}-realtime`,
+ model: resolveModel(provider, selectedModel),
+ mode: "openwhispr",
+ language,
+ };
+ }
+
+ if (transcriptionMode === "self-hosted") {
+ throw new Error(
+ "Self-hosted realtime transcription is not supported for Note Recording. Choose Local or Cloud Providers."
+ );
+ }
+
+ if (transcriptionMode !== "providers") {
+ throw new Error(`Unsupported Note Recording transcription mode: ${transcriptionMode}`);
+ }
+
+ const provider = byokProviders.find((candidate) => candidate.id === selectedProvider);
+ if (!provider) {
+ throw new Error(`Unsupported Note Recording provider: ${selectedProvider || "none selected"}`);
+ }
+
+ const options = {
+ provider: `${provider.id}-realtime`,
+ model: resolveModel(provider, selectedModel),
+ mode: "byok",
+ language,
+ };
+
+ if (provider.id === "corti") {
+ return {
+ ...options,
+ environment: cortiEnvironment,
+ tenant: cortiTenant,
+ keyterms,
+ };
+ }
+
+ return options;
+}
diff --git a/src/helpers/openaiRealtimeStreaming.js b/src/helpers/openaiRealtimeStreaming.js
index e4aefcb68..88f6b2965 100644
--- a/src/helpers/openaiRealtimeStreaming.js
+++ b/src/helpers/openaiRealtimeStreaming.js
@@ -39,7 +39,9 @@ class OpenAIRealtimeStreaming {
this.onError = null;
this.onSessionEnd = null;
this.onSessionExpired = null;
+ this.onConnectionLost = null;
this._sessionExpired = false;
+ this._connectionLossNotified = false;
this._sessionTimer = null;
this.pendingResolve = null;
this.pendingReject = null;
@@ -92,6 +94,7 @@ class OpenAIRealtimeStreaming {
this.audioBytesSent = 0;
this.speechStartedAt = null;
this._sessionExpired = false;
+ this._connectionLossNotified = false;
const url = "wss://api.openai.com/v1/realtime?intent=transcription";
debugLogger.debug("OpenAI Realtime connecting", { model: this.model });
@@ -129,6 +132,7 @@ class OpenAIRealtimeStreaming {
});
this.ws.on("error", (error) => {
+ const wasActive = this.isConnected;
debugLogger.error("OpenAI Realtime WebSocket error", { error: error.message });
this.isConnecting = false;
this.cleanup();
@@ -137,7 +141,11 @@ class OpenAIRealtimeStreaming {
this.pendingReject = null;
this.pendingResolve = null;
}
- this.onError?.(error);
+ if (wasActive && !this.isDisconnecting) {
+ this._notifyConnectionLost(error);
+ } else if (!this.isDisconnecting) {
+ this.onError?.(error);
+ }
});
this.ws.on("close", (code, reason) => {
@@ -156,6 +164,7 @@ class OpenAIRealtimeStreaming {
this.cleanup();
if (wasActive && !this.isDisconnecting && !this._sessionExpired) {
this.onSessionEnd?.({ text: this.getFullTranscript() });
+ this._notifyConnectionLost(new Error(`Connection lost (code: ${code})`));
}
});
});
@@ -257,7 +266,7 @@ class OpenAIRealtimeStreaming {
if (errCode === "session_expired" && this.onSessionExpired) {
debugLogger.warn("OpenAI Realtime session expired", { message: errMsg });
this._sessionExpired = true;
- this.onSessionExpired();
+ this.onSessionExpired({ proactive: false });
break;
}
const isEmptyBuffer =
@@ -299,12 +308,22 @@ class OpenAIRealtimeStreaming {
}
}
+ _notifyConnectionLost(error) {
+ if (this._connectionLossNotified) return;
+ this._connectionLossNotified = true;
+ if (this.onConnectionLost) {
+ this.onConnectionLost(error);
+ } else {
+ this.onError?.(error);
+ }
+ }
+
_startSessionTimer() {
clearTimeout(this._sessionTimer);
this._sessionTimer = setTimeout(() => {
if (!this.isConnected) return;
debugLogger.debug("OpenAI Realtime session approaching 60min limit, requesting reconnect");
- this.onSessionExpired?.();
+ this.onSessionExpired?.({ proactive: true });
}, SESSION_PREEMPT_MS);
}
@@ -401,7 +420,7 @@ class OpenAIRealtimeStreaming {
return true;
}
- async disconnect() {
+ async disconnect({ commit = true } = {}) {
debugLogger.debug("OpenAI Realtime disconnect", {
audioBytesSent: this.audioBytesSent,
segments: this.completedSegments.length,
@@ -421,7 +440,7 @@ class OpenAIRealtimeStreaming {
}
if (this.ws.readyState === WebSocket.OPEN) {
- if (this.audioBytesSent > 0) {
+ if (commit && this.audioBytesSent > 0) {
const prevOnFinal = this.onFinalTranscript;
const prevOnError = this.onError;
diff --git a/src/helpers/tinfoilRealtimeStreaming.js b/src/helpers/tinfoilRealtimeStreaming.js
index c1f4312e8..c0e34fa75 100644
--- a/src/helpers/tinfoilRealtimeStreaming.js
+++ b/src/helpers/tinfoilRealtimeStreaming.js
@@ -4,24 +4,38 @@ const { createTinfoilRealtimeSocket } = require("./tinfoilSecureClient");
const debugLogger = require("./debugLogger");
const TINFOIL_REALTIME_MODEL = "voxtral-mini-4b-realtime";
-
const COMMIT_POLL_MS = 250;
-// Mirrors the silence_duration_ms the OpenAI path configures in server_vad.
const COMMIT_IDLE_MS = 600;
-// Bounds the text lost on abrupt socket death and keeps segments flowing to diarization.
-const MAX_UTTERANCE_MS = 30000;
-const COMMIT_ACK_TIMEOUT_MS = 10000;
-// Mirrors the base class's disconnect timeout so a dead server can't hang the stop.
+const MAX_TURN_MS = 30000;
+const COMMIT_ACK_TIMEOUT_MS = 20000;
const COMMIT_DRAIN_TIMEOUT_MS = 3000;
+const SPEECH_RMS_THRESHOLD = 0.003;
+const SPEECH_PEAK_THRESHOLD = 0.02;
-// Matches the empty-buffer signatures the base class tolerates at disconnect.
const isEmptyBufferError = (error) =>
error?.code === "input_audio_buffer_commit_empty" ||
(error?.message || "").includes("buffer too small") ||
(error?.message || "").includes("commit_empty");
-// Tinfoil's transcription intent has no server VAD, so utterance boundaries
-// are decided client-side from the delta stream and committed explicitly.
+const hasSpeechEnergy = (pcmBuffer) => {
+ if (!pcmBuffer?.length) return false;
+ const samples = new Int16Array(
+ pcmBuffer.buffer,
+ pcmBuffer.byteOffset,
+ Math.floor(pcmBuffer.byteLength / 2)
+ );
+ let sumSquares = 0;
+ let peak = 0;
+ for (let index = 0; index < samples.length; index += 1) {
+ const sample = samples[index] / 0x7fff;
+ sumSquares += sample * sample;
+ peak = Math.max(peak, Math.abs(sample));
+ }
+ return (
+ Math.sqrt(sumSquares / samples.length) >= SPEECH_RMS_THRESHOLD && peak >= SPEECH_PEAK_THRESHOLD
+ );
+};
+
class TinfoilRealtimeStreaming extends OpenAIRealtimeStreaming {
constructor(createSocketImpl = createTinfoilRealtimeSocket) {
super();
@@ -31,12 +45,13 @@ class TinfoilRealtimeStreaming extends OpenAIRealtimeStreaming {
this._commitSentAt = 0;
this._commitDrain = null;
this._commitDrainResolve = null;
- this._lastDeltaAt = 0;
- this._emptyCommitStreak = 0;
+ this._turnStartedAt = 0;
+ this._lastSpeechAt = 0;
+ this._turnSpeechStartedAt = 0;
+ this._turnHasAudio = false;
+ this._recoveryRequested = false;
}
- // Audio must only ever reach the attested Tinfoil socket, so a
- // caller-supplied factory is overridden, never honored.
async connect(options = {}) {
const { apiKey } = options;
const model = options.model || TINFOIL_REALTIME_MODEL;
@@ -47,54 +62,58 @@ class TinfoilRealtimeStreaming extends OpenAIRealtimeStreaming {
});
}
+ sendAudio(pcmBuffer) {
+ const buffer = Buffer.isBuffer(pcmBuffer) ? pcmBuffer : Buffer.from(pcmBuffer);
+ const accepted = super.sendAudio(buffer);
+ if ((!accepted && !this.bufferingAudio) || !buffer.length) return accepted;
+
+ const now = Date.now();
+ this._turnHasAudio = true;
+ if (!this._turnStartedAt) this._turnStartedAt = now;
+ if (hasSpeechEnergy(buffer)) {
+ this._lastSpeechAt = now;
+ if (!this._turnSpeechStartedAt) this._turnSpeechStartedAt = now;
+ if (!this._commitInFlight && !this.speechStartedAt) {
+ this.speechStartedAt = this._turnSpeechStartedAt;
+ }
+ }
+ return accepted;
+ }
+
handleMessage(data) {
let event;
try {
event = JSON.parse(data.toString());
} catch {
- // Malformed frames are logged by the base handler.
super.handleMessage(data);
return;
}
- if (event.type === "conversation.item.input_audio_transcription.delta") {
- // Only real text moves the idle clock; without server VAD the first delta
- // of an utterance is also the speech-start signal for mic suppression.
- if (event.delta) {
- this._lastDeltaAt = Date.now();
- if (!this.speechStartedAt) this.speechStartedAt = this._lastDeltaAt;
- }
- } else if (event.type === "conversation.item.input_audio_transcription.completed") {
- this._emptyCommitStreak = 0;
- this._settleCommit();
- } else if (event.type === "error") {
- const wasTimerCommit = this._commitInFlight;
+
+ if (event.type === "conversation.item.input_audio_transcription.completed") {
+ const nextTurnSpeechStartedAt = this._turnSpeechStartedAt;
+ this._recoveryRequested = false;
this._settleCommit();
- if (wasTimerCommit && isEmptyBufferError(event.error)) {
- this._handleEmptyCommitReply();
- return;
+ super.handleMessage(data);
+ if (nextTurnSpeechStartedAt && !this.speechStartedAt) {
+ this.speechStartedAt = nextTurnSpeechStartedAt;
}
+ return;
}
- super.handleMessage(data);
- }
- // An empty-buffer reply to a timer commit is benign (a lost completed or a
- // double commit); surfacing it would flash an error mid-meeting.
- _handleEmptyCommitReply() {
- this._emptyCommitStreak += 1;
- // Restart the idle window so a late completed can land before any retry.
- this._lastDeltaAt = Date.now();
- if (this._emptyCommitStreak >= 2) {
- // A second empty reply means this text can't finalize; drop it rather
- // than loop commits forever. A late completed would still deliver it.
- debugLogger.error("Tinfoil Realtime utterance could not be finalized, dropping partial", {
- partialLength: this.currentPartial.length,
- });
- this.currentPartial = "";
- this.speechStartedAt = null;
- this._emptyCommitStreak = 0;
+ if (event.type === "error" && this._commitInFlight) {
+ const error = event.error || {};
+ this._settleCommit();
+ if (isEmptyBufferError(error)) {
+ this.currentPartial = "";
+ this.speechStartedAt = this._turnSpeechStartedAt || null;
+ return;
+ }
+ super.handleMessage(data);
+ this._requestRecovery(new Error(error.message || "Tinfoil commit failed"), false);
return;
}
- debugLogger.debug("Tinfoil Realtime empty commit reply, awaiting late completed");
+
+ super.handleMessage(data);
}
_markConnected() {
@@ -108,70 +127,108 @@ class TinfoilRealtimeStreaming extends OpenAIRealtimeStreaming {
}
_stopCommitTimer() {
- if (this._commitTimer) {
- clearInterval(this._commitTimer);
- this._commitTimer = null;
- }
+ if (!this._commitTimer) return;
+ clearInterval(this._commitTimer);
+ this._commitTimer = null;
}
_commitTick() {
if (this.isDisconnecting || this.ws?.readyState !== WebSocket.OPEN) return;
const now = Date.now();
if (this._commitInFlight) {
- // A commit the server never answered must not block the session forever.
if (now - this._commitSentAt >= COMMIT_ACK_TIMEOUT_MS) {
- debugLogger.debug("Tinfoil Realtime commit unanswered, re-arming");
- this._settleCommit();
+ this._requestRecovery(new Error("Tinfoil did not finish the current transcript segment."));
}
return;
}
- // Committing before any delta would finalize an empty or silent buffer.
- if (!this.currentPartial) return;
- const idleFor = now - this._lastDeltaAt;
- const utteranceAge = now - (this.speechStartedAt || now);
- if (idleFor < COMMIT_IDLE_MS && utteranceAge < MAX_UTTERANCE_MS) return;
+ if (!this._turnHasAudio) return;
+
+ const turnAge = now - this._turnStartedAt;
+ const speechEnded = this._lastSpeechAt > 0 && now - this._lastSpeechAt >= COMMIT_IDLE_MS;
+ if (!speechEnded && turnAge < MAX_TURN_MS) return;
+ this._sendCommit();
+ }
+
+ _sendCommit() {
+ if (this._commitInFlight || !this._turnHasAudio || this.ws?.readyState !== WebSocket.OPEN) {
+ return false;
+ }
+
this._commitInFlight = true;
- this._commitSentAt = now;
+ this._commitSentAt = Date.now();
this._commitDrain = new Promise((resolve) => {
this._commitDrainResolve = resolve;
});
try {
this.ws.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
- } catch (err) {
+ this._resetTurn();
+ return true;
+ } catch (error) {
this._settleCommit();
- debugLogger.debug("Tinfoil Realtime commit send failed", { error: err.message });
+ debugLogger.debug("Tinfoil Realtime commit send failed", { error: error.message });
+ return false;
}
}
+ _requestRecovery(error, surfaceError = true) {
+ if (this._recoveryRequested) return;
+ this._recoveryRequested = true;
+ debugLogger.warn("Tinfoil Realtime connection recovery requested", {
+ error: error.message,
+ });
+ if (this.onConnectionLost) {
+ this._notifyConnectionLost(error);
+ } else if (surfaceError) {
+ this.onError?.(error);
+ }
+ }
+
+ _resetTurn() {
+ this._turnStartedAt = 0;
+ this._lastSpeechAt = 0;
+ this._turnSpeechStartedAt = 0;
+ this._turnHasAudio = false;
+ }
+
_settleCommit() {
this._commitInFlight = false;
- if (this._commitDrainResolve) {
- this._commitDrainResolve();
- this._commitDrainResolve = null;
- this._commitDrain = null;
- }
+ this._commitSentAt = 0;
+ this._commitDrainResolve?.();
+ this._commitDrainResolve = null;
+ this._commitDrain = null;
}
- // The base flush resolves on the first completed, so a commit the timer
- // already sent must finalize first or its tail transcript is lost.
async disconnect() {
this._stopCommitTimer();
- if (this._commitInFlight && this._commitDrain) {
- let timer;
- await Promise.race([
- this._commitDrain,
- new Promise((resolve) => {
- timer = setTimeout(resolve, COMMIT_DRAIN_TIMEOUT_MS);
- }),
- ]);
- clearTimeout(timer);
+ const deadline = Date.now() + COMMIT_DRAIN_TIMEOUT_MS;
+ if (!(await this._waitForCommitDrain(deadline))) {
+ return super.disconnect({ commit: false });
}
- return super.disconnect();
+ if (!this._commitInFlight) this._sendCommit();
+ await this._waitForCommitDrain(deadline);
+ return super.disconnect({ commit: false });
+ }
+
+ async _waitForCommitDrain(deadline) {
+ if (!this._commitInFlight || !this._commitDrain) return true;
+ const remaining = Math.max(0, deadline - Date.now());
+ if (remaining === 0) return false;
+ let timeout;
+ const drained = await Promise.race([
+ this._commitDrain.then(() => true),
+ new Promise((resolve) => {
+ timeout = setTimeout(() => resolve(false), remaining);
+ }),
+ ]);
+ clearTimeout(timeout);
+ return drained;
}
cleanup() {
this._stopCommitTimer();
this._settleCommit();
+ this._resetTurn();
+ this._recoveryRequested = false;
super.cleanup();
}
}
diff --git a/src/stores/meetingRecordingStore.ts b/src/stores/meetingRecordingStore.ts
index 713690d0e..ef68d67b4 100644
--- a/src/stores/meetingRecordingStore.ts
+++ b/src/stores/meetingRecordingStore.ts
@@ -1,6 +1,8 @@
import { create } from "zustand";
import { getSettings, selectResolvedMeetingTranscription } from "./settingsStore";
import { useStreamingProvidersStore } from "./streamingProvidersStore";
+import { getStreamingTranscriptionProviders } from "../models/ModelRegistry";
+import { resolveMeetingTranscriptionOptions } from "../helpers/meetingTranscriptionRouting";
import { isBuiltInMicrophone } from "../utils/audioDeviceUtils";
import {
followsSystemDefaultMic,
@@ -122,62 +124,20 @@ const getMeetingTranscriptionOptions = () => {
const resolved = selectResolvedMeetingTranscription(state);
const language = getBaseLanguageCode(state.preferredLanguage);
- if (resolved.useLocalWhisper) {
- return {
- provider: "local" as const,
- localProvider: resolved.localTranscriptionProvider,
- localModel:
- resolved.localTranscriptionProvider === "nvidia"
- ? resolved.parakeetModel || "parakeet-tdt-0.6b-v3"
- : resolved.whisperModel || "base",
- language,
- };
- }
-
- // Corti (BYOK) streams over its own WSS — independent of the server-driven catalog.
- const selectedProvider =
- state.meetingCloudTranscriptionProvider || state.cloudTranscriptionProvider;
- if (resolved.cloudTranscriptionMode === "byok" && selectedProvider === "corti") {
- return {
- provider: "corti-realtime" as const,
- model: "corti-transcribe",
- mode: "byok" as const,
- language,
- environment: state.cortiEnvironment,
- tenant: state.cortiTenant,
- keyterms: (state.customDictionary ?? []).filter(Boolean),
- };
- }
-
- // Tinfoil (BYOK) streams over its own attested WSS — same catalog independence as Corti.
- if (resolved.cloudTranscriptionMode === "byok" && selectedProvider === "tinfoil") {
- return {
- provider: "tinfoil-realtime" as const,
- model: "voxtral-mini-4b-realtime",
- mode: "byok" as const,
- language,
- };
- }
-
- const catalog = useStreamingProvidersStore.getState().providers;
- const provider =
- catalog?.find((p) => p.id === resolved.cloudTranscriptionProvider) ?? catalog?.[0];
- const byokKeyAvailable = provider?.id === "openai" ? !!state.openaiApiKey : true;
- const mode =
- resolved.cloudTranscriptionMode === "byok" && byokKeyAvailable ? "byok" : "openwhispr";
- if (!provider) {
- logger.debug(
- "Streaming providers catalog not loaded, falling back to OpenAI default",
- {},
- "meeting"
- );
- return { provider: "openai-realtime" as const, model: "gpt-4o-mini-transcribe", mode };
- }
- const model =
- provider.models.find((m) => m.id === resolved.cloudTranscriptionModel)?.id ??
- provider.models.find((m) => m.default)?.id ??
- provider.models[0]?.id;
- return { provider: `${provider.id}-realtime` as const, model, mode };
+ return resolveMeetingTranscriptionOptions({
+ transcriptionMode: resolved.transcriptionMode,
+ language,
+ localProvider: resolved.localTranscriptionProvider,
+ whisperModel: resolved.whisperModel,
+ parakeetModel: resolved.parakeetModel,
+ selectedProvider: resolved.cloudTranscriptionProvider,
+ selectedModel: resolved.cloudTranscriptionModel,
+ byokProviders: getStreamingTranscriptionProviders(),
+ managedProviders: useStreamingProvidersStore.getState().providers,
+ cortiEnvironment: state.cortiEnvironment,
+ cortiTenant: state.cortiTenant,
+ keyterms: (state.customDictionary ?? []).filter(Boolean),
+ });
};
const stopMediaStream = (stream: MediaStream | null) => {
@@ -1075,6 +1035,17 @@ export async function startRecording(args: StartRecordingArgs): Promise<boolean>
});
if (errorCleanup) ipcCleanups.push(errorCleanup);
+ const fatalErrorCleanup = window.electronAPI?.onMeetingTranscriptionFatalError?.((err) => {
+ reportMeetingError(err);
+ logger.error(
+ "Meeting transcription stopped after connection loss",
+ { error: err },
+ "meeting"
+ );
+ if (isRecordingFlag) void stopRecording();
+ });
+ if (fatalErrorCleanup) ipcCleanups.push(fatalErrorCleanup);
+
if (startResult.oneOnOneAttendee) {
const synthetic: SpeakerIdentification = {
speakerId: "speaker_0",
diff --git a/src/stores/settingsStore.ts b/src/stores/settingsStore.ts
index 59e00ca43..878057ebc 100644
--- a/src/stores/settingsStore.ts
+++ b/src/stores/settingsStore.ts
@@ -3,7 +3,6 @@ import { API_ENDPOINTS } from "../config/constants";
import i18n, { normalizeUiLanguage } from "../i18n";
import { ensureAgentNameInDictionary } from "../utils/agentName";
import { chooseDictionaryStartupAction } from "../helpers/dictionaryStartup";
-import { useStreamingProvidersStore } from "./streamingProvidersStore";
import logger from "../utils/logger";
import whisperVadConstants from "../constants/whisperVad.json";
import type {
@@ -2025,27 +2024,21 @@ export interface ResolvedMeetingTranscription {
export const selectResolvedMeetingTranscription = (
state: SettingsState
-): ResolvedMeetingTranscription => {
- const catalog = useStreamingProvidersStore.getState().providers;
- // TODO(1.8.0): Catalog has one cloud entry today (OpenAI Realtime).
- // When a second is added, resolve as `meetingCloudTranscriptionProvider || cloudTranscriptionProvider || catalog[0]?.id`, then validate against catalog.
- const cloudTranscriptionProvider = catalog?.[0]?.id ?? "";
-
- return {
- useLocalWhisper: state.meetingUseLocalWhisper,
- whisperModel: state.meetingWhisperModel || state.whisperModel,
- localTranscriptionProvider: state.meetingLocalTranscriptionProvider,
- parakeetModel: state.meetingParakeetModel || state.parakeetModel,
- cloudTranscriptionProvider,
- cloudTranscriptionModel: state.meetingCloudTranscriptionModel || state.cloudTranscriptionModel,
- cloudTranscriptionBaseUrl:
- state.meetingCloudTranscriptionBaseUrl || state.cloudTranscriptionBaseUrl || "",
- cloudTranscriptionMode: state.meetingCloudTranscriptionMode || state.cloudTranscriptionMode,
- transcriptionMode: state.meetingTranscriptionMode,
- remoteTranscriptionType: state.meetingRemoteTranscriptionType,
- remoteTranscriptionUrl: state.meetingRemoteTranscriptionUrl || state.remoteTranscriptionUrl,
- };
-};
+): ResolvedMeetingTranscription => ({
+ useLocalWhisper: state.meetingUseLocalWhisper,
+ whisperModel: state.meetingWhisperModel || state.whisperModel,
+ localTranscriptionProvider: state.meetingLocalTranscriptionProvider,
+ parakeetModel: state.meetingParakeetModel || state.parakeetModel,
+ cloudTranscriptionProvider:
+ state.meetingCloudTranscriptionProvider || state.cloudTranscriptionProvider,
+ cloudTranscriptionModel: state.meetingCloudTranscriptionModel || state.cloudTranscriptionModel,
+ cloudTranscriptionBaseUrl:
+ state.meetingCloudTranscriptionBaseUrl || state.cloudTranscriptionBaseUrl || "",
+ cloudTranscriptionMode: state.meetingCloudTranscriptionMode || state.cloudTranscriptionMode,
+ transcriptionMode: state.meetingTranscriptionMode,
+ remoteTranscriptionType: state.meetingRemoteTranscriptionType,
+ remoteTranscriptionUrl: state.meetingRemoteTranscriptionUrl || state.remoteTranscriptionUrl,
+});
export interface ResolvedUploadTranscription {
useLocalWhisper: boolean;
diff --git a/src/types/electron.ts b/src/types/electron.ts
index 5b230c404..6369a3c28 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -2214,6 +2214,7 @@ declare global {
) => void
) => () => void;
onMeetingTranscriptionError?: (callback: (error: string) => void) => () => void;
+ onMeetingTranscriptionFatalError?: (callback: (error: string) => void) => () => void;
// Speaker diarization
downloadDiarizationModels?: () => Promise<{ success: boolean; error?: string }>;
diff --git a/test/helpers/meetingStreamingProviders.test.js b/test/helpers/meetingStreamingProviders.test.js
index ccbd4a263..801b7d4f9 100644
--- a/test/helpers/meetingStreamingProviders.test.js
+++ b/test/helpers/meetingStreamingProviders.test.js
@@ -5,7 +5,8 @@ const load = () => import("../../src/helpers/meetingStreamingProviders.js");
test("tinfoil-realtime resolves to the Tinfoil client, never the OpenAI default", async () => {
const { STREAMING_CLIENT_BY_PROVIDER } = await load();
- const { TinfoilRealtimeStreaming } = await import("../../src/helpers/tinfoilRealtimeStreaming.js");
+ const { TinfoilRealtimeStreaming } =
+ await import("../../src/helpers/tinfoilRealtimeStreaming.js");
const OpenAIRealtimeStreaming = (await import("../../src/helpers/openaiRealtimeStreaming.js"))
.default;
@@ -40,3 +41,32 @@ test("allow-list rejects unknown and batch-only providers", async () => {
assert.equal(ALLOWED_MEETING_PROVIDERS.has(provider), false, `${provider} must be rejected`);
}
});
+
+test("client lookup fails closed instead of defaulting to OpenAI", async () => {
+ const { getMeetingStreamingClient } = await load();
+
+ assert.throws(() => getMeetingStreamingClient("unknown-realtime"), /Unsupported meeting/);
+});
+
+test("connection identity includes every provider-specific option", async () => {
+ const { getMeetingConnectionKey } = await load();
+ const options = {
+ provider: "corti-realtime",
+ model: "corti-transcribe",
+ language: "en",
+ mode: "byok",
+ environment: "us",
+ tenant: "tenant-a",
+ keyterms: ["OpenWhispr"],
+ };
+
+ assert.equal(getMeetingConnectionKey(options), getMeetingConnectionKey({ ...options }));
+ assert.notEqual(
+ getMeetingConnectionKey(options),
+ getMeetingConnectionKey({ ...options, provider: "tinfoil-realtime" })
+ );
+ assert.notEqual(
+ getMeetingConnectionKey(options),
+ getMeetingConnectionKey({ ...options, tenant: "tenant-b" })
+ );
+});
diff --git a/test/helpers/meetingTranscriptionRouting.test.js b/test/helpers/meetingTranscriptionRouting.test.js
new file mode 100644
index 000000000..35d17a229
--- /dev/null
+++ b/test/helpers/meetingTranscriptionRouting.test.js
@@ -0,0 +1,165 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+const load = () => import("../../src/helpers/meetingTranscriptionRouting.js");
+
+const byokProviders = [
+ {
+ id: "openai",
+ models: [{ id: "gpt-4o-mini-transcribe" }, { id: "gpt-4o-transcribe" }],
+ },
+ { id: "corti", models: [{ id: "corti-transcribe" }] },
+ { id: "tinfoil", models: [{ id: "voxtral-mini-4b-realtime" }] },
+];
+
+const baseOptions = {
+ transcriptionMode: "providers",
+ language: "en",
+ localProvider: "whisper",
+ whisperModel: "small",
+ parakeetModel: "parakeet-tdt-0.6b-v3",
+ selectedProvider: "tinfoil",
+ selectedModel: "voxtral-mini-4b-realtime",
+ byokProviders,
+ managedProviders: [
+ {
+ id: "assemblyai",
+ models: [{ id: "universal-streaming", default: true }],
+ },
+ ],
+ cortiEnvironment: "us",
+ cortiTenant: "tenant",
+ keyterms: ["OpenWhispr"],
+};
+
+test("providers mode routes Tinfoil through its realtime client", async () => {
+ const { resolveMeetingTranscriptionOptions } = await load();
+
+ assert.deepEqual(resolveMeetingTranscriptionOptions(baseOptions), {
+ provider: "tinfoil-realtime",
+ model: "voxtral-mini-4b-realtime",
+ mode: "byok",
+ language: "en",
+ });
+});
+
+test("BYOK OpenAI never downgrades to managed cloud when its key is unavailable", async () => {
+ const { resolveMeetingTranscriptionOptions } = await load();
+
+ assert.deepEqual(
+ resolveMeetingTranscriptionOptions({
+ ...baseOptions,
+ selectedProvider: "openai",
+ selectedModel: "gpt-4o-mini-transcribe",
+ }),
+ {
+ provider: "openai-realtime",
+ model: "gpt-4o-mini-transcribe",
+ mode: "byok",
+ language: "en",
+ }
+ );
+});
+
+test("self-hosted mode never follows a stale Tinfoil provider", async () => {
+ const { resolveMeetingTranscriptionOptions } = await load();
+
+ assert.throws(
+ () =>
+ resolveMeetingTranscriptionOptions({
+ ...baseOptions,
+ transcriptionMode: "self-hosted",
+ }),
+ /Self-hosted realtime transcription is not supported/
+ );
+});
+
+test("local mode wins over stale cloud provider state", async () => {
+ const { resolveMeetingTranscriptionOptions } = await load();
+
+ assert.deepEqual(
+ resolveMeetingTranscriptionOptions({
+ ...baseOptions,
+ transcriptionMode: "local",
+ localProvider: "nvidia",
+ parakeetModel: "nemotron-speech-streaming-en-0.6b",
+ }),
+ {
+ provider: "local",
+ localProvider: "nvidia",
+ localModel: "nemotron-speech-streaming-en-0.6b",
+ language: "en",
+ }
+ );
+});
+
+test("managed mode ignores stale BYOK state", async () => {
+ const { resolveMeetingTranscriptionOptions } = await load();
+
+ assert.deepEqual(
+ resolveMeetingTranscriptionOptions({
+ ...baseOptions,
+ transcriptionMode: "openwhispr",
+ }),
+ {
+ provider: "assemblyai-realtime",
+ model: "universal-streaming",
+ mode: "openwhispr",
+ language: "en",
+ }
+ );
+});
+
+test("managed mode keeps its established OpenAI default before the catalog loads", async () => {
+ const { resolveMeetingTranscriptionOptions } = await load();
+
+ assert.deepEqual(
+ resolveMeetingTranscriptionOptions({
+ ...baseOptions,
+ transcriptionMode: "openwhispr",
+ managedProviders: null,
+ }),
+ {
+ provider: "openai-realtime",
+ model: "gpt-4o-mini-transcribe",
+ mode: "openwhispr",
+ language: "en",
+ }
+ );
+});
+
+test("Corti keeps the meeting-specific connection settings", async () => {
+ const { resolveMeetingTranscriptionOptions } = await load();
+
+ assert.deepEqual(
+ resolveMeetingTranscriptionOptions({
+ ...baseOptions,
+ selectedProvider: "corti",
+ selectedModel: "corti-transcribe",
+ }),
+ {
+ provider: "corti-realtime",
+ model: "corti-transcribe",
+ mode: "byok",
+ language: "en",
+ environment: "us",
+ tenant: "tenant",
+ keyterms: ["OpenWhispr"],
+ }
+ );
+});
+
+test("unknown and custom providers fail closed", async () => {
+ const { resolveMeetingTranscriptionOptions } = await load();
+
+ for (const selectedProvider of ["custom", "groq", "", undefined]) {
+ assert.throws(
+ () =>
+ resolveMeetingTranscriptionOptions({
+ ...baseOptions,
+ selectedProvider,
+ }),
+ /Unsupported Note Recording provider/
+ );
+ }
+});
diff --git a/test/helpers/streamingConnectionLifecycle.test.js b/test/helpers/streamingConnectionLifecycle.test.js
new file mode 100644
index 000000000..50f1c1e15
--- /dev/null
+++ b/test/helpers/streamingConnectionLifecycle.test.js
@@ -0,0 +1,40 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+const AssemblyAiStreaming = require("../../src/helpers/assemblyAiStreaming");
+const CortiStreaming = require("../../src/helpers/cortiStreaming");
+const DeepgramStreaming = require("../../src/helpers/deepgramStreaming");
+const OpenAIRealtimeStreaming = require("../../src/helpers/openaiRealtimeStreaming");
+
+const clients = [
+ ["OpenAI", OpenAIRealtimeStreaming, "_notifyConnectionLost"],
+ ["AssemblyAI", AssemblyAiStreaming, "notifyConnectionLost"],
+ ["Deepgram", DeepgramStreaming, "notifyConnectionLost"],
+ ["Corti", CortiStreaming, "notifyConnectionLost"],
+];
+
+for (const [name, StreamingClient, notifyMethod] of clients) {
+ test(`${name} reports one recoverable connection loss to meetings`, () => {
+ const streaming = new StreamingClient();
+ const recoveries = [];
+ const errors = [];
+ streaming.onConnectionLost = (error) => recoveries.push(error.message);
+ streaming.onError = (error) => errors.push(error.message);
+
+ streaming[notifyMethod](new Error("socket failed"));
+ streaming[notifyMethod](new Error("duplicate close"));
+
+ assert.deepEqual(recoveries, ["socket failed"]);
+ assert.deepEqual(errors, []);
+ });
+
+ test(`${name} preserves the existing error path outside meetings`, () => {
+ const streaming = new StreamingClient();
+ const errors = [];
+ streaming.onError = (error) => errors.push(error.message);
+
+ streaming[notifyMethod](new Error("socket failed"));
+
+ assert.deepEqual(errors, ["socket failed"]);
+ });
+}
diff --git a/test/helpers/tinfoilRealtimeStreaming.test.js b/test/helpers/tinfoilRealtimeStreaming.test.js
index afee9b46f..5075feec3 100644
--- a/test/helpers/tinfoilRealtimeStreaming.test.js
+++ b/test/helpers/tinfoilRealtimeStreaming.test.js
@@ -4,8 +4,6 @@ const { EventEmitter } = require("node:events");
const WS = require("ws");
const load = () => import("../../src/helpers/tinfoilRealtimeStreaming.js");
-
-const DELTA_EVENT = "conversation.item.input_audio_transcription.delta";
const COMPLETED_EVENT = "conversation.item.input_audio_transcription.completed";
function makeFakeSocket(readyState) {
@@ -13,7 +11,6 @@ function makeFakeSocket(readyState) {
socket.readyState = readyState;
socket.sent = [];
socket.send = (data) => socket.sent.push(data);
- // A healthy connection answers pings, or the keep-alive terminates it mid-test.
socket.ping = () => socket.emit("pong");
socket.terminate = () => {
socket.readyState = WS.CLOSED;
@@ -27,19 +24,17 @@ function makeFakeSocket(readyState) {
function makeRecordingFactory(socket) {
const calls = [];
- const factory = async (args) => {
- calls.push(args);
- return socket;
+ return {
+ calls,
+ factory: async (args) => {
+ calls.push(args);
+ return socket;
+ },
};
- return { factory, calls };
}
-function sentCommits(socket) {
- return socket.sent.filter((raw) => JSON.parse(raw).type === "input_audio_buffer.commit");
-}
-
-function delta(text) {
- return JSON.stringify({ type: DELTA_EVENT, delta: text });
+function sentEvents(socket, type) {
+ return socket.sent.map((raw) => JSON.parse(raw)).filter((event) => event.type === type);
}
function completed(transcript) {
@@ -53,8 +48,14 @@ function commitEmptyError() {
});
}
-// Wires a connected instance directly, mirroring the existing realtime test
-// style: cadence behavior is driven through handleMessage + mocked timers.
+function speechPcm(sampleCount = 480, amplitude = 2000) {
+ const samples = new Int16Array(sampleCount);
+ for (let index = 0; index < samples.length; index += 1) {
+ samples[index] = index % 2 === 0 ? amplitude : -amplitude;
+ }
+ return Buffer.from(samples.buffer);
+}
+
async function makeConnected() {
const { TinfoilRealtimeStreaming } = await load();
const streaming = new TinfoilRealtimeStreaming();
@@ -63,116 +64,92 @@ async function makeConnected() {
return streaming;
}
-// -- connect: attested socket factory --
+async function finishConnect(streaming, socket, preconfigured = false) {
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.readyState = WS.OPEN;
+ socket.emit("message", JSON.stringify({ type: "session.created" }));
+ if (!preconfigured) {
+ socket.emit("message", JSON.stringify({ type: "session.updated" }));
+ }
+}
-test("connect() passes the configured model and key to the attested socket factory", async () => {
+test("connect uses only the attested socket factory with the selected model and key", async () => {
const { TinfoilRealtimeStreaming } = await load();
const socket = makeFakeSocket(WS.CONNECTING);
const { factory, calls } = makeRecordingFactory(socket);
const streaming = new TinfoilRealtimeStreaming(factory);
+ let rogueCalled = false;
- const connected = streaming.connect({ apiKey: "tk-secret", model: "custom-rt-model" });
- await new Promise((resolve) => setImmediate(resolve));
- socket.readyState = WS.OPEN;
- socket.emit("message", JSON.stringify({ type: "session.created" }));
- socket.emit("message", JSON.stringify({ type: "session.updated" }));
+ const connected = streaming.connect({
+ apiKey: "tk-secret",
+ model: "custom-rt-model",
+ createSocket: async () => {
+ rogueCalled = true;
+ return makeFakeSocket(WS.OPEN);
+ },
+ });
+ await finishConnect(streaming, socket);
await connected;
+ assert.equal(rogueCalled, false);
assert.deepEqual(calls, [{ model: "custom-rt-model", apiKey: "tk-secret" }]);
streaming.cleanup();
});
-test("connect() defaults the model to voxtral-mini-4b-realtime when the caller omits one", async () => {
+test("connect defaults to the supported Tinfoil realtime model", async () => {
const { TinfoilRealtimeStreaming, TINFOIL_REALTIME_MODEL } = await load();
const socket = makeFakeSocket(WS.CONNECTING);
const { factory, calls } = makeRecordingFactory(socket);
const streaming = new TinfoilRealtimeStreaming(factory);
const connected = streaming.connect({ apiKey: "tk-secret" });
- await new Promise((resolve) => setImmediate(resolve));
- socket.readyState = WS.OPEN;
- socket.emit("message", JSON.stringify({ type: "session.created" }));
- socket.emit("message", JSON.stringify({ type: "session.updated" }));
+ await finishConnect(streaming, socket);
await connected;
- assert.equal(calls[0].model, TINFOIL_REALTIME_MODEL);
assert.equal(TINFOIL_REALTIME_MODEL, "voxtral-mini-4b-realtime");
+ assert.equal(calls[0].model, TINFOIL_REALTIME_MODEL);
streaming.cleanup();
});
-test("connect() overrides a caller-supplied createSocket with the attested factory", async () => {
- const { TinfoilRealtimeStreaming } = await load();
- const socket = makeFakeSocket(WS.CONNECTING);
- const { factory, calls } = makeRecordingFactory(socket);
- const streaming = new TinfoilRealtimeStreaming(factory);
- let rogueCalled = false;
-
- const connected = streaming.connect({
- apiKey: "tk-secret",
- createSocket: async () => {
- rogueCalled = true;
- return makeFakeSocket(WS.OPEN);
- },
- });
- await new Promise((resolve) => setImmediate(resolve));
- socket.readyState = WS.OPEN;
- socket.emit("message", JSON.stringify({ type: "session.created" }));
- socket.emit("message", JSON.stringify({ type: "session.updated" }));
- await connected;
-
- assert.equal(rogueCalled, false, "audio must only travel over the attested socket");
- assert.equal(calls.length, 1);
- streaming.cleanup();
-});
-
-test("the default socket factory is the attested Tinfoil transport", async () => {
+test("the default factory is the pinned Tinfoil transport", async () => {
const { TinfoilRealtimeStreaming } = await load();
const { createTinfoilRealtimeSocket } = require("../../src/helpers/tinfoilSecureClient");
- const streaming = new TinfoilRealtimeStreaming();
-
- assert.equal(streaming._createSocketImpl, createTinfoilRealtimeSocket);
+ assert.equal(new TinfoilRealtimeStreaming()._createSocketImpl, createTinfoilRealtimeSocket);
});
-test("factory failure rejects connect() without constructing any fallback socket", async () => {
+test("attestation failure rejects without opening a fallback socket", async () => {
const { TinfoilRealtimeStreaming } = await load();
const streaming = new TinfoilRealtimeStreaming(async () => {
throw new Error("attestation failed");
});
await assert.rejects(() => streaming.connect({ apiKey: "tk-secret" }), /attestation failed/);
-
- assert.equal(streaming.ws, null, "no socket of any kind may exist after a factory failure");
- assert.equal(streaming.isConnecting, false);
+ assert.equal(streaming.ws, null);
assert.equal(streaming.isConnected, false);
});
-test("each fresh instance re-invokes the factory (meeting reconnect swaps instances)", async () => {
+test("every fresh instance invokes the attested factory", async () => {
const { TinfoilRealtimeStreaming } = await load();
const calls = [];
- for (let i = 0; i < 2; i++) {
+ for (let index = 0; index < 2; index += 1) {
const socket = makeFakeSocket(WS.CONNECTING);
const streaming = new TinfoilRealtimeStreaming(async (args) => {
calls.push(args);
return socket;
});
const connected = streaming.connect({ apiKey: "tk-secret" });
- await new Promise((resolve) => setImmediate(resolve));
- socket.readyState = WS.OPEN;
- socket.emit("message", JSON.stringify({ type: "session.created" }));
- socket.emit("message", JSON.stringify({ type: "session.updated" }));
+ await finishConnect(streaming, socket);
await connected;
streaming.cleanup();
}
- assert.equal(calls.length, 2, "every connect must dial a fresh attested socket");
+ assert.equal(calls.length, 2);
assert.deepEqual(calls[0], calls[1]);
});
-// -- session configuration --
-
-test("connect() sends the GA transcription session.update declaring the 24kHz meeting rate", async () => {
+test("meeting sessions declare the 24kHz PCM input format", async () => {
const { TinfoilRealtimeStreaming } = await load();
const socket = makeFakeSocket(WS.CONNECTING);
const { factory } = makeRecordingFactory(socket);
@@ -183,10 +160,7 @@ test("connect() sends the GA transcription session.update declaring the 24kHz me
socket.readyState = WS.OPEN;
socket.emit("message", JSON.stringify({ type: "session.created" }));
- assert.equal(socket.sent.length, 1);
- const update = JSON.parse(socket.sent[0]);
- assert.equal(update.type, "session.update");
- assert.equal(update.session.type, "transcription");
+ const [update] = sentEvents(socket, "session.update");
assert.deepEqual(update.session.audio.input.format, { type: "audio/pcm", rate: 24000 });
assert.equal(update.session.audio.input.transcription.model, "voxtral-mini-4b-realtime");
@@ -195,224 +169,194 @@ test("connect() sends the GA transcription session.update declaring the 24kHz me
streaming.cleanup();
});
-test("preconfigured sessions get no session.update (dictation semantics survive subclassing)", async () => {
+test("preconfigured sessions do not send a client session update", async () => {
const { TinfoilRealtimeStreaming } = await load();
const socket = makeFakeSocket(WS.CONNECTING);
const { factory } = makeRecordingFactory(socket);
const streaming = new TinfoilRealtimeStreaming(factory);
const connected = streaming.connect({ apiKey: "tk-secret", preconfigured: true });
- await new Promise((resolve) => setImmediate(resolve));
- socket.readyState = WS.OPEN;
- socket.emit("message", JSON.stringify({ type: "session.created" }));
+ await finishConnect(streaming, socket, true);
await connected;
- assert.equal(socket.sent.length, 0, "session.update must not override a preconfigured session");
+ assert.equal(sentEvents(socket, "session.update").length, 0);
streaming.cleanup();
});
-// -- client-side utterance commits (Tinfoil has no server VAD) --
-
-test("commits after the delta stream goes idle for the silence window", (t) => {
+test("speech followed by 600ms of PCM silence commits the turn", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
+ streaming.sendAudio(speechPcm());
- streaming.handleMessage(delta("hello"));
t.mock.timers.tick(500);
- assert.equal(sentCommits(streaming.ws).length, 0, "no commit while under the idle window");
+ streaming.sendAudio(Buffer.alloc(960));
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 0);
t.mock.timers.tick(250);
- const commits = sentCommits(streaming.ws);
- assert.equal(commits.length, 1);
- assert.deepEqual(JSON.parse(commits[0]), { type: "input_audio_buffer.commit" });
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);
})();
});
-test("keeps streaming without commits while deltas keep arriving", (t) => {
+test("continuous speech is bounded by a 30-second turn", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
- for (let i = 0; i < 8; i++) {
- streaming.handleMessage(delta("word "));
+ for (let index = 0; index < 119; index += 1) {
+ streaming.sendAudio(speechPcm());
t.mock.timers.tick(250);
}
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 0);
- assert.equal(sentCommits(streaming.ws).length, 0, "an active utterance must not be split");
+ streaming.sendAudio(speechPcm());
+ t.mock.timers.tick(250);
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);
})();
});
-test("commits a continuous utterance at the max-utterance cap", (t) => {
+test("a silent backend turn is recycled before the provider zombie window", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
+ streaming.sendAudio(Buffer.alloc(960));
- for (let i = 0; i < 118; i++) {
- streaming.handleMessage(delta("word "));
- t.mock.timers.tick(250);
- }
- assert.equal(sentCommits(streaming.ws).length, 0, "cap must not fire before 30s");
-
- for (let i = 0; i < 4; i++) {
- streaming.handleMessage(delta("word "));
- t.mock.timers.tick(250);
- }
- assert.equal(sentCommits(streaming.ws).length, 1, "cap must bound an endless utterance");
+ t.mock.timers.tick(29750);
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 0);
+ t.mock.timers.tick(250);
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);
})();
});
-test("never commits while nothing has been transcribed (empty-buffer guard)", (t) => {
+test("no audio produces no commits", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
-
- t.mock.timers.tick(5000);
-
- assert.equal(streaming.ws.sent.length, 0, "silence must never produce a commit");
+ t.mock.timers.tick(60000);
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 0);
})();
});
-test("one commit in flight at a time; completed re-arms the next utterance", (t) => {
+test("completed re-arms the next buffered turn", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
const finals = [];
streaming.onFinalTranscript = (text) => finals.push(text);
- streaming.handleMessage(delta("first utterance"));
- t.mock.timers.tick(850);
- assert.equal(sentCommits(streaming.ws).length, 1);
-
- t.mock.timers.tick(850);
- assert.equal(sentCommits(streaming.ws).length, 1, "no second commit before the server answers");
-
- streaming.handleMessage(completed("first utterance"));
- streaming.handleMessage(delta("second utterance"));
- t.mock.timers.tick(850);
- assert.equal(sentCommits(streaming.ws).length, 2, "next utterance must commit again");
- assert.deepEqual(finals, ["first utterance"], "finalized text must still reach the consumer");
- })();
-});
-
-test("an unanswered commit re-arms after the ack deadline instead of stalling the session", (t) => {
- t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
- return (async () => {
- const streaming = await makeConnected();
+ streaming.sendAudio(speechPcm());
+ t.mock.timers.tick(750);
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);
- streaming.handleMessage(delta("hello"));
- t.mock.timers.tick(850);
- assert.equal(sentCommits(streaming.ws).length, 1);
+ streaming.sendAudio(speechPcm());
+ t.mock.timers.tick(750);
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);
- t.mock.timers.tick(10250);
- assert.equal(sentCommits(streaming.ws).length, 2, "session must self-heal after a lost commit");
+ streaming.handleMessage(completed("first"));
+ t.mock.timers.tick(250);
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 2);
+ assert.deepEqual(finals, ["first"]);
})();
});
-test("first delta stamps speechStartedAt for the mic-suppression window", (t) => {
+test("commit timeout requests recovery without sending a duplicate commit", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
- assert.equal(streaming.speechStartedAt, null);
+ const errors = [];
+ streaming.onConnectionLost = (error) => errors.push(error.message);
- t.mock.timers.tick(1000);
- streaming.handleMessage(delta("hello"));
- assert.equal(streaming.speechStartedAt, 101000);
+ streaming.sendAudio(speechPcm());
+ t.mock.timers.tick(750);
+ t.mock.timers.tick(20250);
- streaming.handleMessage(completed("hello"));
- assert.equal(streaming.speechStartedAt, null, "utterance boundary must reset the stamp");
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);
+ assert.deepEqual(errors, ["Tinfoil did not finish the current transcript segment."]);
})();
});
-test("cleanup() stops the commit timer so a dead instance never commits", (t) => {
+test("speech timing comes from PCM activity rather than transcript timing", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
- const socket = streaming.ws;
-
- streaming.handleMessage(delta("hello"));
- streaming.cleanup();
- assert.equal(streaming._commitTimer, null);
- t.mock.timers.tick(5000);
- assert.equal(sentCommits(socket).length, 0);
+ streaming.sendAudio(Buffer.alloc(960));
+ assert.equal(streaming.speechStartedAt, null);
+ t.mock.timers.tick(250);
+ streaming.sendAudio(speechPcm());
+ assert.equal(streaming.speechStartedAt, 100250);
})();
});
-// -- disconnect drain: an in-flight timer commit must finalize before the flush --
-
-test("disconnect() drains the in-flight timer commit so the tail transcript still lands", (t) => {
+test("disconnect drains an in-flight commit before committing the tail", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
const socket = streaming.ws;
- streaming.sendAudio(Buffer.alloc(480));
- streaming.handleMessage(delta("first utterance"));
- t.mock.timers.tick(850);
- assert.equal(sentCommits(socket).length, 1, "timer commit in flight");
- streaming.handleMessage(delta("tail spoken after the commit"));
+ streaming.sendAudio(speechPcm());
+ t.mock.timers.tick(750);
+ streaming.sendAudio(speechPcm());
const disconnected = streaming.disconnect();
await new Promise((resolve) => setImmediate(resolve));
- assert.equal(sentCommits(socket).length, 1, "flush must wait for the in-flight commit");
+ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 1);
- streaming.handleMessage(completed("first utterance"));
+ streaming.handleMessage(completed("first"));
await new Promise((resolve) => setImmediate(resolve));
- assert.equal(sentCommits(socket).length, 2, "flush commit follows the drained completed");
+ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 2);
- streaming.handleMessage(completed("tail spoken after the commit"));
- const result = await disconnected;
- assert.equal(result.text, "first utterance tail spoken after the commit");
+ streaming.handleMessage(completed("tail"));
+ assert.deepEqual(await disconnected, { text: "first tail" });
})();
});
-test("disconnect() with an unanswered commit resolves within the bounded deadlines", (t) => {
+test("disconnect remains bounded when the provider never completes", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
- streaming.sendAudio(Buffer.alloc(480));
- streaming.handleMessage(delta("hello"));
- t.mock.timers.tick(850);
- assert.equal(sentCommits(streaming.ws).length, 1);
+ streaming.sendAudio(speechPcm());
+ t.mock.timers.tick(750);
const disconnected = streaming.disconnect();
await new Promise((resolve) => setImmediate(resolve));
t.mock.timers.tick(3000);
await new Promise((resolve) => setImmediate(resolve));
- t.mock.timers.tick(3000);
- const result = await disconnected;
- assert.equal(result.text, "", "resolves with accumulated text instead of hanging");
+
+ assert.deepEqual(await disconnected, { text: "" });
})();
});
-// -- empty-commit replies: benign for timer commits, bounded against loops --
-
-test("empty-buffer reply to a timer commit is swallowed and retried after a fresh idle window", (t) => {
+test("disconnect uses one drain deadline across the active turn and its tail", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
- let surfaced = 0;
- streaming.onError = () => {
- surfaced += 1;
- };
+ const socket = streaming.ws;
+ streaming.sendAudio(speechPcm());
+ t.mock.timers.tick(750);
+ streaming.sendAudio(speechPcm());
+
+ let finished = false;
+ const disconnected = streaming.disconnect().then((result) => {
+ finished = true;
+ return result;
+ });
- streaming.handleMessage(delta("hello"));
- t.mock.timers.tick(850);
- assert.equal(sentCommits(streaming.ws).length, 1);
+ t.mock.timers.tick(2000);
+ streaming.handleMessage(completed("first"));
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 2);
- streaming.handleMessage(commitEmptyError());
- assert.equal(surfaced, 0, "benign empty reply must not reach the meeting error channel");
- assert.equal(streaming._commitInFlight, false);
+ t.mock.timers.tick(999);
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.equal(finished, false);
- t.mock.timers.tick(250);
- assert.equal(sentCommits(streaming.ws).length, 1, "no instant re-commit before the idle window");
- t.mock.timers.tick(600);
- assert.equal(sentCommits(streaming.ws).length, 2, "retry follows a full idle window");
+ t.mock.timers.tick(1);
+ assert.deepEqual(await disconnected, { text: "first" });
})();
});
-test("a second empty-buffer reply drops the unfinalizable partial instead of looping commits", (t) => {
+test("empty commit replies are swallowed without a retry loop", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
@@ -421,85 +365,46 @@ test("a second empty-buffer reply drops the unfinalizable partial instead of loo
surfaced += 1;
};
- streaming.handleMessage(delta("hello"));
- t.mock.timers.tick(850);
+ streaming.sendAudio(speechPcm());
+ t.mock.timers.tick(750);
streaming.handleMessage(commitEmptyError());
- t.mock.timers.tick(850);
- assert.equal(sentCommits(streaming.ws).length, 2, "one retry after the first empty reply");
-
- streaming.handleMessage(commitEmptyError());
- assert.equal(surfaced, 0);
- assert.equal(streaming.currentPartial, "", "unfinalizable partial dropped");
- assert.equal(streaming.speechStartedAt, null);
-
t.mock.timers.tick(5000);
- assert.equal(sentCommits(streaming.ws).length, 2, "no commit loop after dropping the partial");
- })();
-});
-
-test("empty-buffer error with no timer commit in flight still reaches onError", (t) => {
- t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
- return (async () => {
- const streaming = await makeConnected();
- let errMsg = null;
- streaming.onError = (err) => {
- errMsg = err.message;
- };
-
- streaming.handleMessage(commitEmptyError());
- assert.equal(errMsg, "buffer too small", "base disconnect-flush tolerance must still see it");
+ assert.equal(surfaced, 0);
+ assert.equal(sentEvents(streaming.ws, "input_audio_buffer.commit").length, 1);
})();
});
-test("server errors during an in-flight commit still surface", (t) => {
+test("provider errors surface and request connection recovery", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
- let errMsg = null;
- streaming.onError = (err) => {
- errMsg = err.message;
- };
+ const errors = [];
+ const recoveries = [];
+ streaming.onError = (error) => errors.push(error.message);
+ streaming.onConnectionLost = (error) => recoveries.push(error.message);
- streaming.handleMessage(delta("hello"));
- t.mock.timers.tick(850);
+ streaming.sendAudio(speechPcm());
+ t.mock.timers.tick(750);
streaming.handleMessage(
- JSON.stringify({ type: "error", error: { code: "server_error", message: "something broke" } })
+ JSON.stringify({ type: "error", error: { code: "server_error", message: "failed" } })
);
- assert.equal(errMsg, "something broke");
- assert.equal(streaming._commitInFlight, false);
+ assert.deepEqual(errors, ["failed"]);
+ assert.deepEqual(recoveries, ["failed"]);
})();
});
-// -- empty delta frames must not touch the utterance timers --
-
-test("an empty delta frame does not stamp speechStartedAt", (t) => {
+test("cleanup stops the turn timer", (t) => {
t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
return (async () => {
const streaming = await makeConnected();
+ const socket = streaming.ws;
+ streaming.sendAudio(speechPcm());
+ streaming.cleanup();
- streaming.handleMessage(delta(""));
- assert.equal(streaming.speechStartedAt, null, "an empty frame is not speech");
-
- t.mock.timers.tick(250);
- streaming.handleMessage(delta("real"));
- assert.equal(streaming.speechStartedAt, 100250);
- })();
-});
-
-test("empty delta frames do not starve the idle commit", (t) => {
- t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
- return (async () => {
- const streaming = await makeConnected();
-
- streaming.handleMessage(delta("hi"));
- t.mock.timers.tick(250);
- streaming.handleMessage(delta(""));
- t.mock.timers.tick(250);
- streaming.handleMessage(delta(""));
- t.mock.timers.tick(250);
-
- assert.equal(sentCommits(streaming.ws).length, 1, "empty frames must not reset the idle clock");
+ t.mock.timers.tick(5000);
+ assert.equal(streaming._commitTimer, null);
+ assert.equal(sentEvents(socket, "input_audio_buffer.commit").length, 0);
})();
});