Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
const test = require("node:test");
const assert = require("node:assert/strict");
const { WebSocketServer } = require("ws");
const {
GeminiLiveStreaming,
buildGeminiLiveUrl,
} = require("../../src/helpers/geminiLiveStreaming");
const FRAME = Buffer.alloc(1600); // one 50ms worklet frame at 16kHz s16le (silence)
const LOUD_FRAME = Buffer.alloc(1600, 0x20); // same frame at speech level (~0.25 RMS)
// Loopback Live server. `script(socket, message)` reacts to each client
// message; the default answers `setup` with setupComplete, like the real one.
async function withServer(run, script) {
const server = new WebSocketServer({ host: "127.0.0.1", port: 0 });
await new Promise((resolve) => server.once("listening", resolve));
const received = [];
const urls = [];
const reply = script || ((socket) => socket.send(JSON.stringify({ setupComplete: {} })));
server.on("connection", (socket, request) => {
urls.push(request.url);
socket.on("message", (data) => {
const message = JSON.parse(data.toString());
received.push(message);
reply(socket, message, urls.length);
});
});
const streaming = new GeminiLiveStreaming();
streaming.buildWebSocketUrl = ({ token }) =>
`ws://127.0.0.1:${server.address().port}/?t=${token}`;
try {
await run({ streaming, received, urls });
} finally {
streaming.cleanup();
await new Promise((resolve) => server.close(resolve));
}
}
const audioFrames = (received) => received.filter((message) => message.realtimeInput?.audio);
test("url: the two Live methods are auth-disjoint and pinned per mode", () => {
assert.equal(
buildGeminiLiveUrl({ mode: "byok", token: "AI-KEY" }),
"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=AI-KEY"
);
assert.equal(
buildGeminiLiveUrl({ mode: "openwhispr", token: "auth_tokens/abc" }),
"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained?access_token=auth_tokens/abc"
);
});
test("setup: the custom dictionary always ships; a language only when chosen", () => {
const streaming = new GeminiLiveStreaming();
assert.deepEqual(streaming.buildSetupMessage({ keyterms: ["OpenWhispr", " ", ""] }), {
setup: {
model: "models/gemini-3.5-transcribe-live",
generationConfig: { responseModalities: ["TEXT"] },
inputAudioTranscription: { customVocabulary: ["OpenWhispr"] },
},
});
assert.deepEqual(
streaming.buildSetupMessage({ model: "gemini-next", language: "en", keyterms: [] }),
{
setup: {
model: "models/gemini-next",
generationConfig: { responseModalities: ["TEXT"] },
inputAudioTranscription: { languageCodes: ["en"] },
},
}
);
const many = streaming.buildSetupMessage({
keyterms: Array.from({ length: 150 }, (_, i) => `term-${i}`),
});
assert.equal(many.setup.inputAudioTranscription.customVocabulary.length, 100);
});
test("connect resolves on setupComplete, not on socket open", async () => {
await withServer(
async ({ streaming, received }) => {
streaming.beginConnecting();
// Frames from the token-fetch / handshake window are buffered, not sent.
assert.equal(streaming.sendAudio(FRAME), false);
const connected = streaming.connect({ token: "t", mode: "byok", keyterms: ["OpenWhispr"] });
await new Promise((resolve) => setTimeout(resolve, 30));
assert.equal(streaming.isConnected, false, "open alone must not mark the session ready");
assert.equal(streaming.sendAudio(FRAME), false);
await connected;
assert.equal(streaming.isConnected, true);
assert.deepEqual(received[0].setup.inputAudioTranscription, {
customVocabulary: ["OpenWhispr"],
});
},
(socket, message) => {
if (!message.setup) return;
setTimeout(() => socket.send(JSON.stringify({ setupComplete: {} })), 60);
}
);
});
test("connect rejects when the socket drops before setupComplete", async () => {
await withServer(
async ({ streaming }) => {
await assert.rejects(
streaming.connect({ token: "t", mode: "byok" }),
/closed before ready \(code: 1006\)/
);
assert.equal(streaming.isConnected, false);
},
(socket) => socket.terminate()
);
});
test("audio: buffered frames flush in order, then stream as base64 PCM frames", async () => {
await withServer(async ({ streaming, received }) => {
streaming.beginConnecting();
const first = Buffer.alloc(1600, 1);
const second = Buffer.alloc(1600, 2);
streaming.sendAudio(first);
streaming.sendAudio(second);
await streaming.connect({ token: "t", mode: "byok" });
const live = Buffer.alloc(1600, 3);
assert.equal(streaming.sendAudio(live), true);
await new Promise((resolve) => setTimeout(resolve, 30));
const frames = audioFrames(received);
assert.deepEqual(
frames.map((frame) => frame.realtimeInput.audio.mimeType),
["audio/pcm;rate=16000", "audio/pcm;rate=16000", "audio/pcm;rate=16000"]
);
assert.deepEqual(
frames.map((frame) => Buffer.from(frame.realtimeInput.audio.data, "base64")[0]),
[1, 2, 3]
);
assert.equal(streaming.audioBytesSent, 4800);
});
});
test("audio: a dictation shorter than the handshake still reaches the wire", async () => {
await withServer(async ({ streaming, received }) => {
// Every frame arrives before setupComplete and no frame follows it, so the
// only drain opportunity is readiness itself.
streaming.beginConnecting();
streaming.sendAudio(Buffer.alloc(1600, 7));
streaming.sendAudio(Buffer.alloc(1600, 8));
await streaming.connect({ token: "t", mode: "byok" });
await new Promise((resolve) => setTimeout(resolve, 30));
assert.deepEqual(
audioFrames(received).map(
(frame) => Buffer.from(frame.realtimeInput.audio.data, "base64")[0]
),
[7, 8]
);
assert.equal(streaming.audioBytesSent, 3200);
await streaming.disconnect();
assert.equal(
received.filter((message) => message.realtimeInput?.audioStreamEnd).length,
1,
"the turn must still be closed"
);
});
});
test("setup: a non-Gemini model id never reaches the wire", async () => {
const streaming = new GeminiLiveStreaming();
// Managed dictation carries the BYOK default, which is an OpenAI id.
assert.equal(
streaming.buildSetupMessage({ model: "gpt-4o-mini-transcribe" }).setup.model,
"models/gemini-3.5-transcribe-live"
);
assert.equal(
streaming.buildSetupMessage({ model: "gemini-3.5-transcribe-live" }).setup.model,
"models/gemini-3.5-transcribe-live"
);
});
test("audio: cold-start buffering is bounded at three seconds of PCM", async () => {
const streaming = new GeminiLiveStreaming();
streaming.beginConnecting();
for (let i = 0; i < 100; i++) streaming.sendAudio(Buffer.alloc(1600));
assert.equal(streaming.coldStartBufferSize, 96000);
});
test("characterization: the worklet's flush sentinel is never sent as audio", async () => {
await withServer(async ({ streaming, received }) => {
await streaming.connect({ token: "t", mode: "byok" });
assert.equal(streaming.sendAudio(Buffer.from("flushed")), false);
await new Promise((resolve) => setTimeout(resolve, 30));
assert.equal(audioFrames(received).length, 0);
assert.equal(streaming.audioBytesSent, 0);
});
});
test("transcripts: partials replace, finals accumulate", async () => {
await withServer(
async ({ streaming }) => {
const partials = [];
const finals = [];
streaming.onPartialTranscript = (text) => partials.push(text);
streaming.onFinalTranscript = (text) => finals.push(text);
await streaming.connect({ token: "t", mode: "byok" });
streaming.sendAudio(FRAME);
await new Promise((resolve) => setTimeout(resolve, 60));
// Partials are revised, not appended to — casing included.
assert.deepEqual(partials, ["The quick brown", "the quick brown fox"]);
assert.deepEqual(finals, [
"The quick brown fox.",
"The quick brown fox. OpenWhispr transcription test.",
]);
assert.equal(
streaming.getFullTranscript(),
"The quick brown fox. OpenWhispr transcription test."
);
},
(socket, message) => {
if (message.setup) {
socket.send(JSON.stringify({ setupComplete: {} }));
return;
}
if (!message.realtimeInput?.audio) return;
for (const text of ["The quick brown", "the quick brown fox"]) {
socket.send(JSON.stringify({ serverContent: { interimInputTranscription: { text } } }));
}
socket.send(
JSON.stringify({ serverContent: { inputTranscription: { text: "The quick brown fox." } } })
);
socket.send(
JSON.stringify({
serverContent: { inputTranscription: { text: "OpenWhispr transcription test." } },
})
);
}
);
});
test("disconnect sends audioStreamEnd once and waits for the end of the turn", async () => {
await withServer(
async ({ streaming, received }) => {
await streaming.connect({ token: "t", mode: "byok" });
streaming.sendAudio(FRAME);
// The stop sequence calls finalize(), waits for the text to settle, then
// stops — so disconnect must not send audioStreamEnd a second time.
assert.equal(streaming.finalize(), true);
assert.equal(streaming.finalize(), false, "audioStreamEnd must not be sent twice");
await new Promise((resolve) => setTimeout(resolve, 40));
const result = await streaming.disconnect(true);
assert.equal(result.text, "The quick brown fox.");
assert.equal(received.filter((m) => m.realtimeInput?.audioStreamEnd).length, 1);
// The transcript is handed over, not kept for the next session.
assert.equal(streaming.getFullTranscript(), "");
},
(socket, message) => {
if (message.setup) {
socket.send(JSON.stringify({ setupComplete: {} }));
return;
}
if (!message.realtimeInput?.audioStreamEnd) return;
socket.send(
JSON.stringify({ serverContent: { inputTranscription: { text: "The quick brown fox." } } })
);
socket.send(JSON.stringify({ serverContent: { generationComplete: true } }));
}
);
});
test("disconnect still waits for the final when finalize() already closed the turn", async () => {
await withServer(
async ({ streaming, received }) => {
await streaming.connect({ token: "t", mode: "byok" });
streaming.sendAudio(FRAME);
// The stop sequence finalizes, then stops as soon as its own wait gives
// up — disconnect must not treat the sent audioStreamEnd as "answered".
assert.equal(streaming.finalize(), true);
const result = await streaming.disconnect(true);
assert.equal(result.text, "the slow final");
assert.equal(received.filter((m) => m.realtimeInput?.audioStreamEnd).length, 1);
},
(socket, message) => {
if (message.setup) {
socket.send(JSON.stringify({ setupComplete: {} }));
return;
}
if (!message.realtimeInput?.audioStreamEnd) return;
setTimeout(() => {
socket.send(
JSON.stringify({ serverContent: { inputTranscription: { text: "the slow final" } } })
);
socket.send(JSON.stringify({ serverContent: { generationComplete: true } }));
}, 120);
}
);
});
test("disconnect spends only what is left of the budget since audioStreamEnd", async () => {
await withServer(
async ({ streaming }) => {
await streaming.connect({ token: "t", mode: "byok" });
streaming.sendAudio(FRAME);
await new Promise((resolve) => setTimeout(resolve, 30));
assert.equal(streaming.finalize(), true);
// The renderer already waited out the whole budget on its side.
streaming._audioStreamEndSentAt -= 10000;
const start = Date.now();
const result = await streaming.disconnect(true);
assert.equal(result.text, "already here");
assert.ok(Date.now() - start < 500, "must not re-spend the budget");
},
(socket, message) => {
if (message.setup) {
socket.send(JSON.stringify({ setupComplete: {} }));
return;
}
if (message.realtimeInput?.audio) {
socket.send(
JSON.stringify({ serverContent: { inputTranscription: { text: "already here" } } })
);
}
}
);
});
test("a turn the server closed before the hotkey release answers the stop immediately", async () => {
await withServer(
async ({ streaming, received }) => {
await streaming.connect({ token: "t", mode: "byok" });
streaming.sendAudio(FRAME);
await new Promise((resolve) => setTimeout(resolve, 30));
// Trailing silence let the server finalize on its own; nothing more will come.
assert.equal(streaming._turnEnded, true);
// Room-noise frames keep arriving until the hotkey is released; they are not speech.
streaming.sendAudio(FRAME);
assert.equal(streaming._turnEnded, true);
const startedAt = Date.now();
assert.deepEqual(await streaming.disconnect(true), { text: "already final" });
assert.ok(Date.now() - startedAt < 500, "must not wait out the budget for a second turn end");
await new Promise((resolve) => setTimeout(resolve, 30));
assert.equal(
received.filter((message) => message.realtimeInput?.audioStreamEnd).length,
1,
"the stream is still closed cleanly"
);
},
(socket, message) => {
if (message.setup) {
socket.send(JSON.stringify({ setupComplete: {} }));
return;
}
if (message.realtimeInput?.audio) {
socket.send(
JSON.stringify({ serverContent: { inputTranscription: { text: "already final" } } })
);
socket.send(JSON.stringify({ serverContent: { generationComplete: true } }));
}
}
);
});
test("speech after a closed turn reopens it, so the stop waits for the last turn", async () => {
await withServer(
async ({ streaming }) => {
await streaming.connect({ token: "t", mode: "byok" });
streaming.sendAudio(Buffer.alloc(1600, 1));
await new Promise((resolve) => setTimeout(resolve, 30));
assert.equal(streaming._turnEnded, true);
// The user keeps talking: a speech-level frame reopens the turn and the stop must wait.
streaming.sendAudio(LOUD_FRAME);
assert.equal(streaming._turnEnded, false);
assert.deepEqual(await streaming.disconnect(true), { text: "first segment last words" });
},
(socket, message) => {
if (message.setup) {
socket.send(JSON.stringify({ setupComplete: {} }));
return;
}
const audio = message.realtimeInput?.audio;
if (audio && Buffer.from(audio.data, "base64")[0] !== LOUD_FRAME[0]) {
socket.send(
JSON.stringify({ serverContent: { inputTranscription: { text: "first segment" } } })
);
socket.send(JSON.stringify({ serverContent: { generationComplete: true } }));
}
if (message.realtimeInput?.audioStreamEnd) {
socket.send(
JSON.stringify({ serverContent: { inputTranscription: { text: "last words" } } })
);
socket.send(JSON.stringify({ serverContent: { generationComplete: true } }));
}
}
);
});
test("disconnect returns the accumulated text when the turn never ends cleanly", async () => {
await withServer(
async ({ streaming }) => {
await streaming.connect({ token: "t", mode: "byok" });
streaming.sendAudio(FRAME);
await new Promise((resolve) => setTimeout(resolve, 30));
// No generationComplete: the socket dies instead, which must release the
// finalization wait rather than burn its whole budget.
assert.deepEqual(await streaming.disconnect(true), { text: "partial turn" });
},
(socket, message) => {
if (message.setup) {
socket.send(JSON.stringify({ setupComplete: {} }));
return;
}
if (message.realtimeInput?.audio) {
socket.send(
JSON.stringify({ serverContent: { inputTranscription: { text: "partial turn" } } })
);
}
if (message.realtimeInput?.audioStreamEnd) socket.close(1000, "");
}
);
});
// 1007 is a malformed credential, 1008 a revoked or deleted one. The reason is
// truncated by the server and is never part of the user-facing message.
for (const [code, reason] of [
[1007, "API key not valid. Please pass a valid API key."],
[1008, "Requested entity was not found."],
]) {
test(`errors: a ${code} close frame becomes the coded, fixable key error`, async () => {
await withServer(
async ({ streaming }) => {
await assert.rejects(
streaming.connect({ token: "nope", mode: "byok" }),
(err) =>
err.code === "INVALID_KEY" &&
/Check your key in Settings/.test(err.message) &&
!err.message.includes(reason)
);
},
(socket) => socket.close(code, reason)
);
});
}
test("disconnect keeps a last turn the server never finalized", async () => {
await withServer(
async ({ streaming }) => {
await streaming.connect({ token: "t", mode: "byok" });
streaming.sendAudio(FRAME);
await new Promise((resolve) => setTimeout(resolve, 30));
assert.equal(streaming.finalize(), true);
streaming._audioStreamEndSentAt -= 10000;
// An earlier turn was finalized; the last one only ever had a partial.
assert.deepEqual(await streaming.disconnect(true), {
text: "Turn one. Turn two is still open",
});
},
(socket, message) => {
if (message.setup) {
socket.send(JSON.stringify({ setupComplete: {} }));
return;
}
if (!message.realtimeInput?.audio) return;
socket.send(JSON.stringify({ serverContent: { inputTranscription: { text: "Turn one." } } }));
socket.send(JSON.stringify({ serverContent: { generationComplete: true } }));
socket.send(
JSON.stringify({
serverContent: { interimInputTranscription: { text: "Turn two is still open" } },
})
);
}
);
});
test("a partial that restates the finalized turns adds only the open turn", async () => {
await withServer(
async ({ streaming }) => {
await streaming.connect({ token: "t", mode: "byok" });
streaming.sendAudio(FRAME);
await new Promise((resolve) => setTimeout(resolve, 30));
assert.equal(streaming.getFullTranscript(), "Turn one. Turn two. Turn three");
},
(socket, message) => {
if (message.setup) {
socket.send(JSON.stringify({ setupComplete: {} }));
return;
}
if (!message.realtimeInput?.audio) return;
for (const text of ["Turn one.", "Turn two."]) {
socket.send(JSON.stringify({ serverContent: { inputTranscription: { text } } }));
}
// Real Live sessions sometimes repeat every finalized turn, unspaced.
socket.send(
JSON.stringify({
serverContent: { interimInputTranscription: { text: "Turn one.Turn two.Turn three" } },
})
);
}
);
});
test("errors: a spent managed token is re-minted once and the session survives", async () => {
await withServer(
async ({ streaming, urls }) => {
await streaming.connect({
token: "auth_tokens/spent",
mode: "openwhispr",
refreshToken: async () => "auth_tokens/fresh",
});
assert.equal(streaming.isConnected, true);
assert.deepEqual(urls, ["/?t=auth_tokens/spent", "/?t=auth_tokens/fresh"]);
},
(socket, _message, connectionCount) => {
if (connectionCount === 1) {
socket.close(1011, "Token has been used too many times");
return;
}
socket.send(JSON.stringify({ setupComplete: {} }));
}
);
});
test("errors: byok never retries a 1011 close (there is no token to re-mint)", async () => {
await withServer(
async ({ streaming, urls }) => {
await assert.rejects(
streaming.connect({
token: "key",
mode: "byok",
refreshToken: async () => "unused",
}),
(err) => err.code === "AUTH_EXPIRED"
);
assert.equal(urls.length, 1);
},
(socket) => socket.close(1011, "Token has expired")
);
});
test("an unexpected close hands the transcript over and reports the loss once", async () => {
await withServer(
async ({ streaming }) => {
const errors = [];
const sessionEnds = [];
streaming.onError = (err) => errors.push(err.message);
streaming.onSessionEnd = (data) => sessionEnds.push(data);
await streaming.connect({ token: "t", mode: "byok" });
streaming.sendAudio(FRAME);
await new Promise((resolve) => setTimeout(resolve, 60));
// The open turn's partial is handed over with the finalized text.
assert.deepEqual(sessionEnds, [{ text: "half a sentence and the rest" }]);
// Distinct from the pre-ready failure, and without the raw server reason.
assert.deepEqual(errors, ["Connection lost (code: 1011)"]);
assert.equal(streaming.isConnected, false);
},
(socket, message) => {
if (message.setup) {
socket.send(JSON.stringify({ setupComplete: {} }));
return;
}
if (!message.realtimeInput?.audio) return;
socket.send(
JSON.stringify({ serverContent: { inputTranscription: { text: "half a sentence" } } })
);
socket.send(
JSON.stringify({ serverContent: { interimInputTranscription: { text: "and the rest" } } })
);
setTimeout(() => socket.close(1011, "Token has expired"), 10);
}
);
});