Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(streaming): pass language and prompt to OpenAI Realtime session (#2050) (#2100)

Aditya kumar singhcf431bc45572
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index d9d50f75a..f9d1f5ebf 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -8790,6 +8790,16 @@ class IPCHandlers {
};
};
+ // A session being replaced must go quiet: a late final or socket error from
+ // it would land in the renderer as if it came from the live session.
+ const retireDictationStreaming = (streaming) => {
+ streaming.onPartialTranscript = null;
+ streaming.onFinalTranscript = null;
+ streaming.onError = null;
+ streaming.onSessionEnd = null;
+ streaming.disconnect({ commit: false }).catch(() => {});
+ };
+
const DICTATION_IDLE_TIMEOUT_MS = 5 * 60 * 1000;
const ORUKEET_IDLE_TIMEOUT_MS = 60 * 1000;
@@ -8817,7 +8827,9 @@ class IPCHandlers {
// What a dictation connection was opened for; a start or warmup reuses one
// only when nothing about the route changed and, for a connection that reads
- // a saved key, no key was saved since.
+ // a saved key, no key was saved since. OpenAI pins the language when the
+ // session is configured (BYOK session.update, or the cloud token request), so
+ // it is part of that route; the other providers never send it at connect.
const dictationConnectionKey = (options) => {
const provider = options.provider || "openai-realtime";
const readsSavedKey = options.mode === "byok" || provider === "tinfoil-realtime";
@@ -8827,6 +8839,9 @@ class IPCHandlers {
options.model,
options.baseUrl,
readsSavedKey ? credentialGeneration : null,
+ provider === "openai-realtime"
+ ? OpenAIRealtimeStreaming.normalizeLanguage(options.language)
+ : null,
]);
};
@@ -8846,8 +8861,13 @@ class IPCHandlers {
clearDictationIdleTimer();
this._dictationPreviewEnabled = !!options.preview;
+ // The renderer is already streaming frames when a start redials (it wires
+ // the worklet before calling start), so the old session is retired without
+ // a wait, and without a commit where the client supports skipping it: a
+ // commit would only surface its head as a final from the wrong session.
+ // Frames that reached the old session before the start are dropped with it.
if (this._dictationStreaming) {
- await this._dictationStreaming.disconnect().catch(() => {});
+ retireDictationStreaming(this._dictationStreaming);
this._dictationStreaming = null;
}
@@ -8889,6 +8909,9 @@ class IPCHandlers {
const apiKey = await fetchRealtimeToken(event, {
mode: options.mode,
provider,
+ // Cloud sessions are configured server-side when the secret is minted;
+ // the server also owns the managed model, so only the language is sent.
+ language: OpenAIRealtimeStreaming.normalizeLanguage(options.language) ?? undefined,
});
if (provider === "tinfoil-realtime") {
const model = options.model || TINFOIL_REALTIME_MODEL;
@@ -8903,6 +8926,7 @@ class IPCHandlers {
await streaming.connect({
apiKey,
model: options.model || "gpt-4o-mini-transcribe",
+ language: options.language,
// OpenAI rejects rates below 24kHz; the 16kHz capture is upsampled instead.
captureRate: 16000,
preconfigured: isCloud,
diff --git a/src/helpers/openaiRealtimeStreaming.js b/src/helpers/openaiRealtimeStreaming.js
index b4468bd60..442408a88 100644
--- a/src/helpers/openaiRealtimeStreaming.js
+++ b/src/helpers/openaiRealtimeStreaming.js
@@ -52,6 +52,15 @@ async function createSocketWithTimeout(createSocket, timeoutMs) {
}
}
+// Realtime transcription takes ISO-639-1 like the batch endpoint, so a
+// registry code such as "zh-TW" is reduced to its base; "auto" means no pin.
+function normalizeLanguage(language) {
+ if (typeof language !== "string") return null;
+ const trimmed = language.trim().toLowerCase();
+ if (!trimmed || trimmed === "auto") return null;
+ return trimmed.split("-")[0];
+}
+
class OpenAIRealtimeStreaming {
constructor() {
this.ws = null;
@@ -83,6 +92,7 @@ class OpenAIRealtimeStreaming {
this.speechStartedCount = 0;
this._vadEventCount = 0;
this.model = "gpt-4o-mini-transcribe";
+ this.language = null;
this.inputRate = SAMPLE_RATE;
this.captureRate = SAMPLE_RATE;
this.coldStartBuffer = [];
@@ -110,6 +120,7 @@ class OpenAIRealtimeStreaming {
const {
apiKey,
model,
+ language,
preconfigured,
inputRate,
captureRate,
@@ -130,6 +141,7 @@ class OpenAIRealtimeStreaming {
this.isConnecting = true;
this.model = model || "gpt-4o-mini-transcribe";
+ this.language = normalizeLanguage(language);
this.preconfigured = !!preconfigured;
this.inputRate = inputRate || SAMPLE_RATE;
this.captureRate = captureRate || this.inputRate;
@@ -254,6 +266,7 @@ class OpenAIRealtimeStreaming {
`${this.providerLabel} session created, sending configuration`,
this._logContext({
model: this.model,
+ language: this.language,
vadThreshold: this.vadThreshold,
})
);
@@ -266,7 +279,9 @@ class OpenAIRealtimeStreaming {
audio: {
input: {
format: { type: "audio/pcm", rate: this.inputRate },
- transcription: { model: this.model },
+ transcription: this.language
+ ? { model: this.model, language: this.language }
+ : { model: this.model },
turn_detection: turnDetectionFor(this.model, this.vadThreshold),
},
},
@@ -666,3 +681,4 @@ class OpenAIRealtimeStreaming {
}
module.exports = OpenAIRealtimeStreaming;
+module.exports.normalizeLanguage = normalizeLanguage;
diff --git a/src/helpers/tinfoilRealtimeStreaming.js b/src/helpers/tinfoilRealtimeStreaming.js
index 3617c3c5a..717031d61 100644
--- a/src/helpers/tinfoilRealtimeStreaming.js
+++ b/src/helpers/tinfoilRealtimeStreaming.js
@@ -58,6 +58,9 @@ class TinfoilRealtimeStreaming extends OpenAIRealtimeStreaming {
const model = options.model || TINFOIL_REALTIME_MODEL;
return super.connect({
...options,
+ // voxtral's support for a pinned language is unverified, so meetings keep
+ // auto-detection like dictation does.
+ language: undefined,
model,
createSocket: () => this._createSocketImpl({ model, apiKey }),
});
diff --git a/test/helpers/apiKeyUpdatesIpc.test.js b/test/helpers/apiKeyUpdatesIpc.test.js
index b8ff2fa8f..a7c6f3ab9 100644
--- a/test/helpers/apiKeyUpdatesIpc.test.js
+++ b/test/helpers/apiKeyUpdatesIpc.test.js
@@ -93,6 +93,8 @@ class FakeStreaming {
return { text: "" };
}
}
+FakeStreaming.normalizeLanguage =
+ require("../../src/helpers/openaiRealtimeStreaming").normalizeLanguage;
class FakeOrukeet extends FakeStreaming {}
const modulePath = require.resolve("../../src/helpers/ipcHandlers");
diff --git a/test/helpers/dictationRealtimeLanguage.test.js b/test/helpers/dictationRealtimeLanguage.test.js
new file mode 100644
index 000000000..f190dbdc4
--- /dev/null
+++ b/test/helpers/dictationRealtimeLanguage.test.js
@@ -0,0 +1,249 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const Module = require("node:module");
+
+// #2050: the dictation language must reach the realtime session on every path
+// the dictation-realtime-* handlers own — the BYOK session.update, the cloud
+// token request, and a warm connection minted for a different language.
+
+const handlersModulePath = require.resolve("../../src/helpers/ipcHandlers");
+const originalLoad = Module._load;
+const handlers = new Map();
+const listeners = new Map();
+const electronStub = {
+ app: {
+ getPath: () => "/tmp",
+ getName: () => "test",
+ getVersion: () => "0.0.0",
+ isPackaged: false,
+ on() {},
+ requestSingleInstanceLock: () => true,
+ },
+ ipcMain: {
+ handle: (channel, handler) => handlers.set(channel, handler),
+ on: (channel, handler) => listeners.set(channel, handler),
+ removeHandler() {},
+ },
+ net: { fetch: async () => ({ ok: true, status: 200, json: async () => ({}) }) },
+ BrowserWindow: class {
+ static getAllWindows() {
+ return [];
+ }
+ static fromWebContents() {
+ return null;
+ }
+ },
+ shell: {},
+ dialog: {},
+ screen: { getPrimaryDisplay: () => ({ workAreaSize: { width: 0, height: 0 } }) },
+ systemPreferences: { getMediaAccessStatus: () => "granted" },
+ session: { fromPartition: () => ({}) },
+ clipboard: {},
+ nativeImage: {},
+ globalShortcut: {},
+ utilityProcess: {},
+ MessageChannelMain: class {},
+};
+
+const calls = { connects: [], disconnects: [], tokens: [], audio: [], sent: [], instances: [] };
+let nextInstance = 0;
+
+class FakeRealtimeStreaming {
+ constructor() {
+ this.id = ++nextInstance;
+ this.isConnected = false;
+ this.options = null;
+ calls.instances.push(this);
+ }
+ beginConnecting() {}
+ async connect(options) {
+ this.options = options;
+ this.isConnected = true;
+ calls.connects.push({ id: this.id, options });
+ }
+ async disconnect({ commit = true } = {}) {
+ this.isConnected = false;
+ calls.disconnects.push({ id: this.id, commit });
+ return { text: "" };
+ }
+ sendAudio() {
+ calls.audio.push(this.id);
+ return true;
+ }
+}
+FakeRealtimeStreaming.normalizeLanguage =
+ require("../../src/helpers/openaiRealtimeStreaming").normalizeLanguage;
+
+Module._load = function loadWithMocks(request, parent, isMain) {
+ if (request === "electron") return electronStub;
+ if (parent?.filename === handlersModulePath) {
+ if (request === "./debugLogger") return new Proxy({}, { get: () => () => {} });
+ if (request === "./openaiRealtimeStreaming") return FakeRealtimeStreaming;
+ if (request === "./realtimeTokenProviders") {
+ return {
+ ...originalLoad.call(this, request, parent, isMain),
+ fetchRealtimeTokenForProvider: async (provider, _deps, options) => {
+ calls.tokens.push({ provider, options });
+ return "secret";
+ },
+ };
+ }
+ }
+ return originalLoad.call(this, request, parent, isMain);
+};
+test.after(() => {
+ Module._load = originalLoad;
+});
+
+function anything() {
+ return new Proxy(function () {}, {
+ get: (_target, property) => {
+ if (property === Symbol.toPrimitive || property === "toString") return () => "";
+ if (property === "then") return undefined;
+ return anything();
+ },
+ apply: () => anything(),
+ });
+}
+
+function scenario(t) {
+ for (const record of Object.values(calls)) record.length = 0;
+ // Seed the constructor's dictation fields: the proxy's catch-all would make
+ // `this._dictationStreaming?.isConnected` truthy on a cold start.
+ const target = {
+ _dictationStreaming: null,
+ _dictationConnectPromise: null,
+ _dictationIdleTimer: null,
+ _dictationPreviewEnabled: false,
+ windowManager: { hideTranscriptionPreview() {}, showTranscriptionPreview() {} },
+ };
+ const IPCHandlers = require(handlersModulePath);
+ IPCHandlers.prototype.setupHandlers.call(
+ new Proxy(target, {
+ get: (value, property) => (property in value ? value[property] : anything()),
+ })
+ );
+ const event = { sender: { send: (channel, payload) => calls.sent.push({ channel, payload }) } };
+ const invoke = (channel, options) => handlers.get(channel)(event, options);
+ const sendAudio = () => listeners.get("dictation-realtime-send")(event, new Uint8Array(4));
+ t.after(() => invoke("dictation-realtime-stop"));
+ return { invoke, sendAudio };
+}
+
+const byok = (overrides) => ({
+ provider: "openai-realtime",
+ mode: "byok",
+ model: "gpt-4o-mini-transcribe",
+ language: "de",
+ ...overrides,
+});
+
+test("a cold BYOK start passes the dictation language to the realtime session", async (t) => {
+ const s = scenario(t);
+ assert.deepEqual(await s.invoke("dictation-realtime-start", byok()), { success: true });
+ assert.equal(calls.connects.length, 1);
+ assert.equal(calls.connects[0].options.language, "de");
+ assert.equal(calls.connects[0].options.model, "gpt-4o-mini-transcribe");
+ assert.equal(calls.connects[0].options.preconfigured, false);
+});
+
+test("a cloud start sends the language with the token request and leaves the model to the server", async (t) => {
+ const s = scenario(t);
+ await s.invoke(
+ "dictation-realtime-start",
+ byok({ mode: "openwhispr", model: "gpt-4o-transcribe", language: "de" })
+ );
+ assert.equal(calls.tokens.length, 1);
+ assert.equal(calls.tokens[0].provider, "openai-realtime");
+ assert.equal(calls.tokens[0].options.language, "de");
+ // The server owns the managed dictation model; a stale non-OpenAI id in
+ // settings must not reach the client-secret request.
+ assert.equal("model" in calls.tokens[0].options, false);
+ assert.equal(calls.connects[0].options.preconfigured, true);
+});
+
+test("the cloud token request carries the base language code and leaves it unset for auto", async (t) => {
+ const s = scenario(t);
+ await s.invoke("dictation-realtime-start", byok({ mode: "openwhispr", language: "zh-TW" }));
+ await s.invoke("dictation-realtime-stop");
+ await s.invoke("dictation-realtime-start", byok({ mode: "openwhispr", language: "auto" }));
+ assert.equal(calls.tokens[0].options.language, "zh");
+ assert.equal("language" in calls.tokens[1].options, true);
+ assert.equal(calls.tokens[1].options.language, undefined);
+});
+
+test("the tinfoil branch keeps its language-free connect (voxtral support is unverified)", async (t) => {
+ const s = scenario(t);
+ await s.invoke("dictation-realtime-start", byok({ provider: "tinfoil-realtime" }));
+ assert.equal(calls.connects.length, 1);
+ assert.equal("language" in calls.connects[0].options, false);
+});
+
+test("a start reuses a warm connection minted for the same language and model", async (t) => {
+ const s = scenario(t);
+ await s.invoke("dictation-realtime-warmup", byok());
+ await s.invoke("dictation-realtime-start", byok());
+ assert.equal(calls.connects.length, 1, "no redial");
+ assert.deepEqual(calls.disconnects, []);
+});
+
+test("a start redials when the warm connection was minted for another language", async (t) => {
+ const s = scenario(t);
+ await s.invoke("dictation-realtime-warmup", byok({ language: "de" }));
+ await s.invoke("dictation-realtime-start", byok({ language: undefined }));
+ assert.equal(calls.connects.length, 2, "warm German session cannot serve an Auto dictation");
+ assert.deepEqual(calls.disconnects, [{ id: calls.connects[0].id, commit: false }]);
+ assert.equal("language" in calls.connects[1].options, true);
+ assert.equal(calls.connects[1].options.language, undefined);
+});
+
+// The renderer streams worklet frames before dictation-realtime-start arrives,
+// so a redial happens with audio already in flight. The retiring session is
+// neither committed (its head would come back as a wrong-language final) nor
+// waited on, so frames sent once the start is handled reach the new session.
+test("frames sent once a mismatch redial starts buffer into the new session, not the retiring one", async (t) => {
+ const s = scenario(t);
+ await s.invoke("dictation-realtime-warmup", byok({ language: "en" }));
+ const started = s.invoke("dictation-realtime-start", byok({ language: "de" }));
+ s.sendAudio();
+ await started;
+ const [warm, fresh] = calls.instances;
+ assert.deepEqual(calls.audio, [fresh.id]);
+ assert.deepEqual(calls.disconnects, [{ id: warm.id, commit: false }]);
+});
+
+test("a retiring warm session's late events never reach the renderer", async (t) => {
+ const s = scenario(t);
+ await s.invoke("dictation-realtime-warmup", byok({ language: "en" }));
+ await s.invoke("dictation-realtime-start", byok({ language: "de" }));
+ const [warm] = calls.instances;
+ warm.onPartialTranscript?.("hel");
+ warm.onFinalTranscript?.("hello");
+ warm.onError?.(new Error("closed"));
+ warm.onSessionEnd?.({ text: "hello" });
+ assert.deepEqual(calls.sent, []);
+});
+
+test("a start redials when the warm connection was minted for another model", async (t) => {
+ const s = scenario(t);
+ await s.invoke("dictation-realtime-warmup", byok({ model: "gpt-4o-mini-transcribe" }));
+ await s.invoke("dictation-realtime-start", byok({ model: "gpt-4o-transcribe" }));
+ assert.equal(calls.connects.length, 2);
+ assert.equal(calls.connects[1].options.model, "gpt-4o-transcribe");
+});
+
+test("a start redials when the warm connection belongs to another mode", async (t) => {
+ const s = scenario(t);
+ await s.invoke("dictation-realtime-warmup", byok({ mode: "openwhispr" }));
+ await s.invoke("dictation-realtime-start", byok({ mode: "byok" }));
+ assert.equal(calls.connects.length, 2);
+ assert.equal(calls.connects[1].options.preconfigured, false);
+});
+
+test("after stop, the next start dials fresh", async (t) => {
+ const s = scenario(t);
+ await s.invoke("dictation-realtime-start", byok());
+ await s.invoke("dictation-realtime-stop");
+ await s.invoke("dictation-realtime-start", byok());
+ assert.equal(calls.connects.length, 2);
+});
diff --git a/test/helpers/openaiRealtimeStreaming.test.js b/test/helpers/openaiRealtimeStreaming.test.js
index c97bef9bf..c7dbd2255 100644
--- a/test/helpers/openaiRealtimeStreaming.test.js
+++ b/test/helpers/openaiRealtimeStreaming.test.js
@@ -556,8 +556,7 @@ test("BYOK session.update is byte-for-byte today's payload when no VAD/language/
const connected = streaming.connect({
apiKey: "sk-test",
model: "gpt-4o-mini-transcribe",
- // Phase 1 will add these; today connect() must ignore them entirely.
- language: "en",
+ // keyterms stay ignored: the dictionary has no realtime channel yet.
keyterms: ["OpenWhispr"],
sampleRate: 24000,
createSocket: async () => socket,
@@ -594,6 +593,61 @@ test("BYOK session.update is byte-for-byte today's payload when no VAD/language/
streaming.cleanup();
});
+// -- #2050: the dictation language reaches the BYOK transcription config --
+
+test("BYOK session.update pins the dictation language the caller selected (#2050)", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe", language: "de" });
+
+ const [update] = sentEvents(socket, "session.update");
+ assert.deepEqual(update.session.audio.input.transcription, {
+ model: "gpt-4o-mini-transcribe",
+ language: "de",
+ });
+ streaming.cleanup();
+});
+
+test("BYOK session.update sends the base ISO-639-1 code and omits language for auto/blank", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const cases = [
+ ["zh-TW", "zh"],
+ [" PT ", "pt"],
+ ["auto", undefined],
+ ["", undefined],
+ [undefined, undefined],
+ ];
+ for (const [language, expected] of cases) {
+ const streaming = new OpenAIRealtimeStreaming();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, socket, { model: "gpt-4o-mini-transcribe", language });
+
+ const [update] = sentEvents(socket, "session.update");
+ assert.equal(update.session.audio.input.transcription.language, expected, String(language));
+ assert.equal(
+ "language" in update.session.audio.input.transcription,
+ expected !== undefined,
+ `${String(language)} must not leave a language key on the wire`
+ );
+ streaming.cleanup();
+ }
+});
+
+test("a later connect() without a language does not inherit the previous session's pin", async () => {
+ const OpenAIRealtimeStreaming = (await load()).default;
+ const streaming = new OpenAIRealtimeStreaming();
+ const first = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, first, { model: "gpt-4o-mini-transcribe", language: "de" });
+ streaming.cleanup();
+
+ const second = makeFakeSocket(WS.CONNECTING);
+ await connectByok(streaming, second, { model: "gpt-4o-mini-transcribe" });
+ const [update] = sentEvents(second, "session.update");
+ assert.equal("language" in update.session.audio.input.transcription, false);
+ streaming.cleanup();
+});
+
test("BYOK session.update is byte-for-byte the overridden payload when vadThreshold is passed (system socket)", async () => {
const OpenAIRealtimeStreaming = (await load()).default;
const streaming = new OpenAIRealtimeStreaming();
@@ -1095,8 +1149,10 @@ test("preconfigured sessions adopt the model the server minted (session.created
const connected = streaming.connect({
apiKey: "cs-secret",
preconfigured: true,
- // The desktop's BYOK default; the server decides the managed model.
+ // The desktop's BYOK default; the server decides the managed model and
+ // pins the language when it mints the secret.
model: "gpt-4o-mini-transcribe",
+ language: "de",
createSocket: async () => socket,
});
await new Promise((resolve) => setImmediate(resolve));
diff --git a/test/helpers/orukeetDictationIpc.test.js b/test/helpers/orukeetDictationIpc.test.js
index eeef716b0..3d300576f 100644
--- a/test/helpers/orukeetDictationIpc.test.js
+++ b/test/helpers/orukeetDictationIpc.test.js
@@ -463,6 +463,25 @@ test("warmups landing together mint one session and share its unused socket", as
await handlers.get("dictation-realtime-stop")();
});
+test("a start in another language still rides the warm socket", async () => {
+ await handlers.get("dictation-realtime-stop")();
+ let mints = 0;
+ backendResponse = async () => {
+ mints += 1;
+ return Response.json(cloudSession);
+ };
+
+ // Orukeet detects the language itself, so a translation dictation or an
+ // Auto setting must not spend a second single-use GPU token.
+ await handlers.get("dictation-realtime-warmup")(event, { ...managedOptions, language: "en" });
+ const warm = target._dictationStreaming;
+ await handlers.get("dictation-realtime-start")(event, { ...managedOptions, language: "de" });
+
+ assert.equal(mints, 1);
+ assert.equal(target._dictationStreaming, warm);
+ await handlers.get("dictation-realtime-stop")();
+});
+
test("a warmup never reuses a socket that has already carried audio", async () => {
await handlers.get("dictation-realtime-stop")();
let mints = 0;
diff --git a/test/helpers/realtimeTokenProviders.test.js b/test/helpers/realtimeTokenProviders.test.js
index 3489c194f..b7a8f9bbb 100644
--- a/test/helpers/realtimeTokenProviders.test.js
+++ b/test/helpers/realtimeTokenProviders.test.js
@@ -204,7 +204,7 @@ test("missing byok keys throw configuration errors, not token errors", async ()
}
});
-test('wire bodies: dictation posts the bare {"streams":1}; meetings post model+language+streams', async () => {
+test("wire bodies: dictation posts language+streams; meetings post model+language+streams", async () => {
const { fetchRealtimeTokenForProvider } = await load();
const wire = [];
const cloudDeps = deps({
@@ -217,8 +217,16 @@ test('wire bodies: dictation posts the bare {"streams":1}; meetings post model+l
},
});
- // Dictation: ipcHandlers' connectDictationStreaming passes only { mode, provider }.
- await fetchRealtimeTokenForProvider("openai-realtime", cloudDeps, { mode: "openwhispr" });
+ // Dictation: ipcHandlers' connectDictationStreaming passes { mode, provider, language }
+ // (base ISO-639-1, undefined for Auto) and never a model — the server owns it (#2050).
+ await fetchRealtimeTokenForProvider("openai-realtime", cloudDeps, {
+ mode: "openwhispr",
+ language: "de",
+ });
+ await fetchRealtimeTokenForProvider("openai-realtime", cloudDeps, {
+ mode: "openwhispr",
+ language: undefined,
+ });
// Meeting with system audio.
await fetchRealtimeTokenForProvider(
"openai-realtime",
@@ -239,6 +247,7 @@ test('wire bodies: dictation posts the bare {"streams":1}; meetings post model+l
await fetchRealtimeTokenForProvider("gemini-realtime", cloudDeps, { mode: "openwhispr" });
assert.deepEqual(wire, [
+ { path: "/api/openai-realtime-token", json: '{"language":"de","streams":1}' },
{ path: "/api/openai-realtime-token", json: '{"streams":1}' },
{
path: "/api/openai-realtime-token",
diff --git a/test/helpers/tinfoilRealtimeStreaming.test.js b/test/helpers/tinfoilRealtimeStreaming.test.js
index 72c37eb4e..437b02897 100644
--- a/test/helpers/tinfoilRealtimeStreaming.test.js
+++ b/test/helpers/tinfoilRealtimeStreaming.test.js
@@ -207,6 +207,27 @@ test("meeting sessions declare the 24kHz PCM input format", async () => {
streaming.cleanup();
});
+test("meeting sessions never pin a language (voxtral support is unverified)", async () => {
+ const { TinfoilRealtimeStreaming } = await load();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ const { factory } = makeRecordingFactory(socket);
+ const streaming = new TinfoilRealtimeStreaming(factory);
+
+ const connected = streaming.connect({ apiKey: "tk-secret", language: "de" });
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.readyState = WS.OPEN;
+ socket.emit("message", JSON.stringify({ type: "session.created" }));
+
+ const [update] = sentEvents(socket, "session.update");
+ assert.deepEqual(update.session.audio.input.transcription, {
+ model: "voxtral-mini-4b-realtime",
+ });
+
+ socket.emit("message", JSON.stringify({ type: "session.updated" }));
+ await connected;
+ streaming.cleanup();
+});
+
test("meeting session.update honors streamLabel + vadThreshold overrides through the subclass", async () => {
const { TinfoilRealtimeStreaming } = await load();
const socket = makeFakeSocket(WS.CONNECTING);