Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

feat(meetings): add Tinfoil realtime streaming client

Alcahest7ed3ca433adb
diff --git a/src/helpers/tinfoilRealtimeStreaming.js b/src/helpers/tinfoilRealtimeStreaming.js
new file mode 100644
index 000000000..e54675722
--- /dev/null
+++ b/src/helpers/tinfoilRealtimeStreaming.js
@@ -0,0 +1,110 @@
+const WebSocket = require("ws");
+const OpenAIRealtimeStreaming = require("./openaiRealtimeStreaming");
+const { createTinfoilRealtimeSocket } = require("./tinfoilSecureClient");
+const debugLogger = require("./debugLogger");
+
+const TINFOIL_REALTIME_MODEL = "voxtral-mini-4b-realtime";
+
+const COMMIT_POLL_MS = 250;
+// Mirrors the silence_duration_ms the OpenAI path configures in server_vad.
+const COMMIT_IDLE_MS = 600;
+// Bounds the text lost on abrupt socket death and keeps segments flowing to diarization.
+const MAX_UTTERANCE_MS = 30000;
+const COMMIT_ACK_TIMEOUT_MS = 10000;
+
+// Tinfoil's transcription intent has no server VAD, so utterance boundaries
+// are decided client-side from the delta stream and committed explicitly.
+class TinfoilRealtimeStreaming extends OpenAIRealtimeStreaming {
+ constructor(createSocketImpl = createTinfoilRealtimeSocket) {
+ super();
+ this._createSocketImpl = createSocketImpl;
+ this._commitTimer = null;
+ this._commitInFlight = false;
+ this._commitSentAt = 0;
+ this._lastDeltaAt = 0;
+ }
+
+ // Audio must only ever reach the attested Tinfoil socket, so a
+ // caller-supplied factory is overridden, never honored.
+ async connect(options = {}) {
+ const { apiKey } = options;
+ const model = options.model || TINFOIL_REALTIME_MODEL;
+ return super.connect({
+ ...options,
+ model,
+ createSocket: () => this._createSocketImpl({ model, apiKey }),
+ });
+ }
+
+ handleMessage(data) {
+ let type;
+ try {
+ type = JSON.parse(data.toString()).type;
+ } catch {
+ // Malformed frames are logged by the base handler.
+ }
+ if (type === "conversation.item.input_audio_transcription.delta") {
+ this._lastDeltaAt = Date.now();
+ // No speech_started events without server VAD; the first delta of an
+ // utterance is the closest signal for the mic-suppression window.
+ if (!this.speechStartedAt) this.speechStartedAt = this._lastDeltaAt;
+ } else if (
+ type === "conversation.item.input_audio_transcription.completed" ||
+ type === "error"
+ ) {
+ this._commitInFlight = false;
+ }
+ super.handleMessage(data);
+ }
+
+ _markConnected() {
+ super._markConnected();
+ this._startCommitTimer();
+ }
+
+ _startCommitTimer() {
+ this._stopCommitTimer();
+ this._commitTimer = setInterval(() => this._commitTick(), COMMIT_POLL_MS);
+ }
+
+ _stopCommitTimer() {
+ if (this._commitTimer) {
+ clearInterval(this._commitTimer);
+ this._commitTimer = null;
+ }
+ }
+
+ _commitTick() {
+ if (this.isDisconnecting || this.ws?.readyState !== WebSocket.OPEN) return;
+ const now = Date.now();
+ if (this._commitInFlight) {
+ // A commit the server never answered must not block the session forever.
+ if (now - this._commitSentAt >= COMMIT_ACK_TIMEOUT_MS) {
+ debugLogger.debug("Tinfoil Realtime commit unanswered, re-arming");
+ this._commitInFlight = false;
+ }
+ return;
+ }
+ // Committing before any delta would finalize an empty or silent buffer.
+ if (!this.currentPartial) return;
+ const idleFor = now - this._lastDeltaAt;
+ const utteranceAge = now - (this.speechStartedAt || now);
+ if (idleFor < COMMIT_IDLE_MS && utteranceAge < MAX_UTTERANCE_MS) return;
+ this._commitInFlight = true;
+ this._commitSentAt = now;
+ try {
+ this.ws.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
+ } catch (err) {
+ this._commitInFlight = false;
+ debugLogger.debug("Tinfoil Realtime commit send failed", { error: err.message });
+ }
+ }
+
+ cleanup() {
+ this._stopCommitTimer();
+ this._commitInFlight = false;
+ super.cleanup();
+ }
+}
+
+module.exports = { TinfoilRealtimeStreaming, TINFOIL_REALTIME_MODEL };
diff --git a/test/helpers/tinfoilRealtimeStreaming.test.js b/test/helpers/tinfoilRealtimeStreaming.test.js
new file mode 100644
index 000000000..d5119e67c
--- /dev/null
+++ b/test/helpers/tinfoilRealtimeStreaming.test.js
@@ -0,0 +1,333 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { EventEmitter } = require("node:events");
+const WS = require("ws");
+
+const load = () => import("../../src/helpers/tinfoilRealtimeStreaming.js");
+
+const DELTA_EVENT = "conversation.item.input_audio_transcription.delta";
+const COMPLETED_EVENT = "conversation.item.input_audio_transcription.completed";
+
+function makeFakeSocket(readyState) {
+ const socket = new EventEmitter();
+ socket.readyState = readyState;
+ socket.sent = [];
+ socket.send = (data) => socket.sent.push(data);
+ // A healthy connection answers pings, or the keep-alive terminates it mid-test.
+ socket.ping = () => socket.emit("pong");
+ socket.terminate = () => {
+ socket.readyState = WS.CLOSED;
+ socket.emit("close", 1006, Buffer.from(""));
+ };
+ socket.close = () => {
+ socket.readyState = WS.CLOSED;
+ };
+ return socket;
+}
+
+function makeRecordingFactory(socket) {
+ const calls = [];
+ const factory = async (args) => {
+ calls.push(args);
+ return socket;
+ };
+ return { factory, calls };
+}
+
+function sentCommits(socket) {
+ return socket.sent.filter((raw) => JSON.parse(raw).type === "input_audio_buffer.commit");
+}
+
+function delta(text) {
+ return JSON.stringify({ type: DELTA_EVENT, delta: text });
+}
+
+function completed(transcript) {
+ return JSON.stringify({ type: COMPLETED_EVENT, transcript });
+}
+
+// Wires a connected instance directly, mirroring the existing realtime test
+// style: cadence behavior is driven through handleMessage + mocked timers.
+async function makeConnected() {
+ const { TinfoilRealtimeStreaming } = await load();
+ const streaming = new TinfoilRealtimeStreaming();
+ streaming.ws = makeFakeSocket(WS.OPEN);
+ streaming._markConnected();
+ return streaming;
+}
+
+// -- connect: attested socket factory --
+
+test("connect() passes the configured model and key to the attested socket factory", async () => {
+ const { TinfoilRealtimeStreaming } = await load();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ const { factory, calls } = makeRecordingFactory(socket);
+ const streaming = new TinfoilRealtimeStreaming(factory);
+
+ const connected = streaming.connect({ apiKey: "tk-secret", model: "custom-rt-model" });
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.readyState = WS.OPEN;
+ socket.emit("message", JSON.stringify({ type: "session.created" }));
+ socket.emit("message", JSON.stringify({ type: "session.updated" }));
+ await connected;
+
+ assert.deepEqual(calls, [{ model: "custom-rt-model", apiKey: "tk-secret" }]);
+ streaming.cleanup();
+});
+
+test("connect() defaults the model to voxtral-mini-4b-realtime when the caller omits one", async () => {
+ const { TinfoilRealtimeStreaming, TINFOIL_REALTIME_MODEL } = await load();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ const { factory, calls } = makeRecordingFactory(socket);
+ const streaming = new TinfoilRealtimeStreaming(factory);
+
+ const connected = streaming.connect({ apiKey: "tk-secret" });
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.readyState = WS.OPEN;
+ socket.emit("message", JSON.stringify({ type: "session.created" }));
+ socket.emit("message", JSON.stringify({ type: "session.updated" }));
+ await connected;
+
+ assert.equal(calls[0].model, TINFOIL_REALTIME_MODEL);
+ assert.equal(TINFOIL_REALTIME_MODEL, "voxtral-mini-4b-realtime");
+ streaming.cleanup();
+});
+
+test("connect() overrides a caller-supplied createSocket with the attested factory", async () => {
+ const { TinfoilRealtimeStreaming } = await load();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ const { factory, calls } = makeRecordingFactory(socket);
+ const streaming = new TinfoilRealtimeStreaming(factory);
+ let rogueCalled = false;
+
+ const connected = streaming.connect({
+ apiKey: "tk-secret",
+ createSocket: async () => {
+ rogueCalled = true;
+ return makeFakeSocket(WS.OPEN);
+ },
+ });
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.readyState = WS.OPEN;
+ socket.emit("message", JSON.stringify({ type: "session.created" }));
+ socket.emit("message", JSON.stringify({ type: "session.updated" }));
+ await connected;
+
+ assert.equal(rogueCalled, false, "audio must only travel over the attested socket");
+ assert.equal(calls.length, 1);
+ streaming.cleanup();
+});
+
+test("the default socket factory is the attested Tinfoil transport", async () => {
+ const { TinfoilRealtimeStreaming } = await load();
+ const { createTinfoilRealtimeSocket } = require("../../src/helpers/tinfoilSecureClient");
+
+ const streaming = new TinfoilRealtimeStreaming();
+
+ assert.equal(streaming._createSocketImpl, createTinfoilRealtimeSocket);
+});
+
+test("factory failure rejects connect() without constructing any fallback socket", async () => {
+ const { TinfoilRealtimeStreaming } = await load();
+ const streaming = new TinfoilRealtimeStreaming(async () => {
+ throw new Error("attestation failed");
+ });
+
+ await assert.rejects(() => streaming.connect({ apiKey: "tk-secret" }), /attestation failed/);
+
+ assert.equal(streaming.ws, null, "no socket of any kind may exist after a factory failure");
+ assert.equal(streaming.isConnecting, false);
+ assert.equal(streaming.isConnected, false);
+});
+
+test("each fresh instance re-invokes the factory (meeting reconnect swaps instances)", async () => {
+ const { TinfoilRealtimeStreaming } = await load();
+ const calls = [];
+
+ for (let i = 0; i < 2; i++) {
+ const socket = makeFakeSocket(WS.CONNECTING);
+ const streaming = new TinfoilRealtimeStreaming(async (args) => {
+ calls.push(args);
+ return socket;
+ });
+ const connected = streaming.connect({ apiKey: "tk-secret" });
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.readyState = WS.OPEN;
+ socket.emit("message", JSON.stringify({ type: "session.created" }));
+ socket.emit("message", JSON.stringify({ type: "session.updated" }));
+ await connected;
+ streaming.cleanup();
+ }
+
+ assert.equal(calls.length, 2, "every connect must dial a fresh attested socket");
+ assert.deepEqual(calls[0], calls[1]);
+});
+
+// -- session configuration --
+
+test("connect() sends the GA transcription session.update declaring the 24kHz meeting rate", async () => {
+ const { TinfoilRealtimeStreaming } = await load();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ const { factory } = makeRecordingFactory(socket);
+ const streaming = new TinfoilRealtimeStreaming(factory);
+
+ const connected = streaming.connect({ apiKey: "tk-secret" });
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.readyState = WS.OPEN;
+ socket.emit("message", JSON.stringify({ type: "session.created" }));
+
+ assert.equal(socket.sent.length, 1);
+ const update = JSON.parse(socket.sent[0]);
+ assert.equal(update.type, "session.update");
+ assert.equal(update.session.type, "transcription");
+ assert.deepEqual(update.session.audio.input.format, { type: "audio/pcm", rate: 24000 });
+ assert.equal(update.session.audio.input.transcription.model, "voxtral-mini-4b-realtime");
+
+ socket.emit("message", JSON.stringify({ type: "session.updated" }));
+ await connected;
+ streaming.cleanup();
+});
+
+test("preconfigured sessions get no session.update (dictation semantics survive subclassing)", async () => {
+ const { TinfoilRealtimeStreaming } = await load();
+ const socket = makeFakeSocket(WS.CONNECTING);
+ const { factory } = makeRecordingFactory(socket);
+ const streaming = new TinfoilRealtimeStreaming(factory);
+
+ const connected = streaming.connect({ apiKey: "tk-secret", preconfigured: true });
+ await new Promise((resolve) => setImmediate(resolve));
+ socket.readyState = WS.OPEN;
+ socket.emit("message", JSON.stringify({ type: "session.created" }));
+ await connected;
+
+ assert.equal(socket.sent.length, 0, "session.update must not override a preconfigured session");
+ streaming.cleanup();
+});
+
+// -- client-side utterance commits (Tinfoil has no server VAD) --
+
+test("commits after the delta stream goes idle for the silence window", (t) => {
+ t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
+ return (async () => {
+ const streaming = await makeConnected();
+
+ streaming.handleMessage(delta("hello"));
+ t.mock.timers.tick(500);
+ assert.equal(sentCommits(streaming.ws).length, 0, "no commit while under the idle window");
+
+ t.mock.timers.tick(250);
+ const commits = sentCommits(streaming.ws);
+ assert.equal(commits.length, 1);
+ assert.deepEqual(JSON.parse(commits[0]), { type: "input_audio_buffer.commit" });
+ })();
+});
+
+test("keeps streaming without commits while deltas keep arriving", (t) => {
+ t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
+ return (async () => {
+ const streaming = await makeConnected();
+
+ for (let i = 0; i < 8; i++) {
+ streaming.handleMessage(delta("word "));
+ t.mock.timers.tick(250);
+ }
+
+ assert.equal(sentCommits(streaming.ws).length, 0, "an active utterance must not be split");
+ })();
+});
+
+test("commits a continuous utterance at the max-utterance cap", (t) => {
+ t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
+ return (async () => {
+ const streaming = await makeConnected();
+
+ for (let i = 0; i < 118; i++) {
+ streaming.handleMessage(delta("word "));
+ t.mock.timers.tick(250);
+ }
+ assert.equal(sentCommits(streaming.ws).length, 0, "cap must not fire before 30s");
+
+ for (let i = 0; i < 4; i++) {
+ streaming.handleMessage(delta("word "));
+ t.mock.timers.tick(250);
+ }
+ assert.equal(sentCommits(streaming.ws).length, 1, "cap must bound an endless utterance");
+ })();
+});
+
+test("never commits while nothing has been transcribed (empty-buffer guard)", (t) => {
+ t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
+ return (async () => {
+ const streaming = await makeConnected();
+
+ t.mock.timers.tick(5000);
+
+ assert.equal(streaming.ws.sent.length, 0, "silence must never produce a commit");
+ })();
+});
+
+test("one commit in flight at a time; completed re-arms the next utterance", (t) => {
+ t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
+ return (async () => {
+ const streaming = await makeConnected();
+ const finals = [];
+ streaming.onFinalTranscript = (text) => finals.push(text);
+
+ streaming.handleMessage(delta("first utterance"));
+ t.mock.timers.tick(850);
+ assert.equal(sentCommits(streaming.ws).length, 1);
+
+ t.mock.timers.tick(850);
+ assert.equal(sentCommits(streaming.ws).length, 1, "no second commit before the server answers");
+
+ streaming.handleMessage(completed("first utterance"));
+ streaming.handleMessage(delta("second utterance"));
+ t.mock.timers.tick(850);
+ assert.equal(sentCommits(streaming.ws).length, 2, "next utterance must commit again");
+ assert.deepEqual(finals, ["first utterance"], "finalized text must still reach the consumer");
+ })();
+});
+
+test("an unanswered commit re-arms after the ack deadline instead of stalling the session", (t) => {
+ t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
+ return (async () => {
+ const streaming = await makeConnected();
+
+ streaming.handleMessage(delta("hello"));
+ t.mock.timers.tick(850);
+ assert.equal(sentCommits(streaming.ws).length, 1);
+
+ t.mock.timers.tick(10250);
+ assert.equal(sentCommits(streaming.ws).length, 2, "session must self-heal after a lost commit");
+ })();
+});
+
+test("first delta stamps speechStartedAt for the mic-suppression window", (t) => {
+ t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
+ return (async () => {
+ const streaming = await makeConnected();
+ assert.equal(streaming.speechStartedAt, null);
+
+ t.mock.timers.tick(1000);
+ streaming.handleMessage(delta("hello"));
+ assert.equal(streaming.speechStartedAt, 101000);
+
+ streaming.handleMessage(completed("hello"));
+ assert.equal(streaming.speechStartedAt, null, "utterance boundary must reset the stamp");
+ })();
+});
+
+test("cleanup() stops the commit timer so a dead instance never commits", (t) => {
+ t.mock.timers.enable({ apis: ["setInterval", "setTimeout", "Date"], now: 100000 });
+ return (async () => {
+ const streaming = await makeConnected();
+ const socket = streaming.ws;
+
+ streaming.handleMessage(delta("hello"));
+ streaming.cleanup();
+ assert.equal(streaming._commitTimer, null);
+
+ t.mock.timers.tick(5000);
+ assert.equal(sentCommits(socket).length, 0);
+ })();
+});