Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
const WebSocket = require("ws");
const debugLogger = require("./debugLogger");
const WEBSOCKET_TIMEOUT_MS = 15000;
const DISCONNECT_TIMEOUT_MS = 3000;
// gpt-live-transcribe acks a commit only once every appended byte is
// transcribed, and it ran at 0.3–0.8x real time in testing, so its final trails
// the stop by a large fraction of the dictation length (partials keep streaming
// meanwhile).
const LIVE_TRANSCRIBE_COMMIT_TIMEOUT_MS = 30000;
const SAMPLE_RATE = 24000;
const COLD_START_BUFFER_MAX = 3 * SAMPLE_RATE * 2; // 3 seconds of 16-bit PCM
const KEEPALIVE_INTERVAL_MS = 15000;
// OpenAI Realtime sessions die at 60 minutes; reconnect proactively before that.
const SESSION_PREEMPT_MS = 55 * 60 * 1000;
// Raised from 0.3 to keep mic ambient noise from opening turns (#630); callers
// on a cleaner channel pass their own vadThreshold.
const DEFAULT_VAD_THRESHOLD = 0.6;
// Matches the server's check, which may pin a snapshot id.
const isLiveTranscribe = (model) => model.startsWith("gpt-live-transcribe");
// gpt-live-transcribe rejects every turn_detection config ("Turn detection is
// not supported for this transcription model"), so it never emits
// speech_started/speech_stopped and a turn only completes once the client
// commits — which disconnect() does on stop. Legacy models keep server VAD.
const turnDetectionFor = (model, vadThreshold) =>
isLiveTranscribe(model)
? null
: {
type: "server_vad",
threshold: vadThreshold,
silence_duration_ms: 600,
prefix_padding_ms: 500,
};
// A socket factory does network work before the socket exists, so the dial
// must be bounded; a socket resolving after the deadline is closed, not leaked.
async function createSocketWithTimeout(createSocket, timeoutMs) {
const socketPromise = createSocket();
let timer;
const timeout = new Promise((_, reject) => {
timer = setTimeout(() => {
socketPromise.then((socket) => socket?.close?.()).catch(() => {});
reject(new Error("Realtime socket setup timeout"));
}, timeoutMs);
});
try {
return await Promise.race([socketPromise, timeout]);
} finally {
clearTimeout(timer);
}
}
// Realtime transcription takes ISO-639-1 like the batch endpoint, so a
// registry code such as "zh-TW" is reduced to its base; "auto" means no pin.
function normalizeLanguage(language) {
if (typeof language !== "string") return null;
const trimmed = language.trim().toLowerCase();
if (!trimmed || trimmed === "auto") return null;
return trimmed.split("-")[0];
}
class OpenAIRealtimeStreaming {
constructor() {
this.ws = null;
this.isConnected = false;
this.isConnecting = false;
this.completedSegments = [];
this.pendingTranscriptions = new Set();
this.serverAcksCommits = false;
this.currentPartial = "";
this.onPartialTranscript = null;
this.onFinalTranscript = null;
this.onError = null;
this.onSessionEnd = null;
this.onSessionExpired = null;
this.onConnectionLost = null;
this._sessionExpired = false;
this._connectionLossNotified = false;
this._sessionTimer = null;
this.pendingResolve = null;
this.pendingReject = null;
this.connectionTimeout = null;
this.isDisconnecting = false;
this.audioBytesSent = 0;
// Subclasses (Tinfoil) override this so logs name the provider actually
// carrying the audio — a Tinfoil session must never log as OpenAI.
this.providerLabel = "OpenAI Realtime";
// Meetings run two instances of this class at once (mic + system); the
// label is the only way field logs can tell the sockets apart.
this.streamLabel = null;
this.vadThreshold = DEFAULT_VAD_THRESHOLD;
this.speechStartedCount = 0;
this._vadEventCount = 0;
this.model = "gpt-4o-mini-transcribe";
this.language = null;
this.inputRate = SAMPLE_RATE;
this.captureRate = SAMPLE_RATE;
this.coldStartBuffer = [];
this.coldStartBufferSize = 0;
this.speechStartedAt = null;
this.bufferingAudio = false;
this.keepAliveInterval = null;
this._onTurnSettled = null;
}
// Starts buffering audio immediately, before the WebSocket even exists —
// covers the token-fetch + handshake window so sendAudio() doesn't drop
// frames while a connection is still being established.
beginConnecting() {
this.bufferingAudio = true;
this.coldStartBuffer = [];
this.coldStartBufferSize = 0;
}
getFullTranscript() {
return this.completedSegments.join(" ");
}
async connect(options = {}) {
const {
apiKey,
model,
language,
preconfigured,
inputRate,
captureRate,
createSocket,
streamLabel,
vadThreshold,
} = options;
if (!apiKey) throw new Error(`${this.providerLabel} API key is required`);
if (this.isConnected || this.isConnecting) {
debugLogger.debug(`${this.providerLabel} already connected/connecting`, this._logContext());
return;
}
// Callers may already be buffering (beginConnecting() called before the
// apiKey was fetched) — don't wipe audio collected during that window.
if (!this.bufferingAudio) this.beginConnecting();
this.isConnecting = true;
this.model = model || "gpt-4o-mini-transcribe";
this.language = normalizeLanguage(language);
this.preconfigured = !!preconfigured;
this.inputRate = inputRate || SAMPLE_RATE;
this.captureRate = captureRate || this.inputRate;
this.streamLabel = streamLabel || null;
this.vadThreshold = vadThreshold ?? DEFAULT_VAD_THRESHOLD;
this.completedSegments = [];
this.pendingTranscriptions = new Set();
this.serverAcksCommits = false;
this.currentPartial = "";
this.audioBytesSent = 0;
this.speechStartedAt = null;
this.speechStartedCount = 0;
this._vadEventCount = 0;
this._sessionExpired = false;
this._connectionLossNotified = false;
const url = "wss://api.openai.com/v1/realtime?intent=transcription";
debugLogger.debug(`${this.providerLabel} connecting`, this._logContext({ model: this.model }));
// Attested providers (Tinfoil) supply their socket via an async factory.
let ws;
try {
ws = createSocket
? await createSocketWithTimeout(createSocket, WEBSOCKET_TIMEOUT_MS)
: new WebSocket(url, { headers: { Authorization: `Bearer ${apiKey}` } });
} catch (err) {
this.isConnecting = false;
this.cleanup();
throw err;
}
return new Promise((resolve, reject) => {
this.pendingResolve = resolve;
this.pendingReject = reject;
this.connectionTimeout = setTimeout(() => {
this.isConnecting = false;
this.cleanup();
reject(new Error(`${this.providerLabel} connection timeout`));
}, WEBSOCKET_TIMEOUT_MS);
this.ws = ws;
this.ws.on("open", () => {
debugLogger.debug(`${this.providerLabel} WebSocket opened`, this._logContext());
});
this.ws.on("message", (data) => {
this.handleMessage(data);
});
this.ws.on("error", (error) => {
const wasActive = this.isConnected;
debugLogger.error(
`${this.providerLabel} WebSocket error`,
this._logContext({ error: error.message })
);
this.isConnecting = false;
this.cleanup();
if (this.pendingReject) {
this.pendingReject(error);
this.pendingReject = null;
this.pendingResolve = null;
}
if (wasActive && !this.isDisconnecting) {
this._notifyConnectionLost(error);
} else if (!this.isDisconnecting) {
this.onError?.(error);
}
});
this.ws.on("close", (code, reason) => {
const wasActive = this.isConnected;
this.isConnecting = false;
debugLogger.debug(
`${this.providerLabel} WebSocket closed`,
this._logContext({
code,
reason: reason?.toString(),
wasActive,
})
);
if (this.pendingReject) {
this.pendingReject(new Error(`WebSocket closed before ready (code: ${code})`));
this.pendingReject = null;
this.pendingResolve = null;
}
this.cleanup();
if (wasActive && !this.isDisconnecting && !this._sessionExpired) {
this.onSessionEnd?.({ text: this.getFullTranscript() });
this._notifyConnectionLost(new Error(`Connection lost (code: ${code})`));
}
});
});
}
handleMessage(data) {
try {
const event = JSON.parse(data.toString());
switch (event.type) {
case "session.created": {
if (this.preconfigured) {
// Server-side ephemeral token already configured the session;
// sending an update would strip language and noise-reduction.
// Echo the server's VAD + format — the only place preconfigured
// (cloud) session settings ever appear in field logs.
const sessionInput = event.session?.audio?.input;
// The server picks the managed model when it mints the secret; the
// session echo is where the client learns which one it got.
this.model = sessionInput?.transcription?.model ?? this.model;
debugLogger.debug(
`${this.providerLabel} session created (preconfigured)`,
this._logContext({
model: this.model,
turnDetection:
sessionInput?.turn_detection ?? event.session?.turn_detection ?? null,
inputFormat: sessionInput?.format ?? event.session?.input_audio_format ?? null,
})
);
this._markConnected();
} else {
debugLogger.debug(
`${this.providerLabel} session created, sending configuration`,
this._logContext({
model: this.model,
language: this.language,
vadThreshold: this.vadThreshold,
})
);
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) break;
this.ws.send(
JSON.stringify({
type: "session.update",
session: {
type: "transcription",
audio: {
input: {
format: { type: "audio/pcm", rate: this.inputRate },
transcription: this.language
? { model: this.model, language: this.language }
: { model: this.model },
turn_detection: turnDetectionFor(this.model, this.vadThreshold),
},
},
},
})
);
}
break;
}
case "session.updated": {
if (this.pendingResolve) {
debugLogger.debug(
`${this.providerLabel} session configured`,
this._logContext({
model: this.model,
})
);
this._markConnected();
}
break;
}
case "conversation.item.input_audio_transcription.delta": {
const delta = event.delta || "";
if (delta) {
this.currentPartial += delta;
this.onPartialTranscript?.(this.currentPartial);
}
break;
}
case "conversation.item.input_audio_transcription.completed": {
const transcript = (event.transcript || "").trim();
if (transcript) {
this.completedSegments.push(transcript);
}
this.currentPartial = "";
const speechTimestamp = this.speechStartedAt || Date.now();
this.speechStartedAt = null;
if (transcript) {
const fullText = this.getFullTranscript();
this.onFinalTranscript?.(fullText, speechTimestamp);
debugLogger.debug(
`${this.providerLabel} turn completed`,
this._logContext({
turnText: transcript.slice(0, 100),
totalLength: fullText.length,
segments: this.completedSegments.length,
})
);
} else {
debugLogger.debug(
`${this.providerLabel} turn completed with empty transcript`,
this._logContext({
itemId: event.item_id,
audioBytesSent: this.audioBytesSent,
})
);
}
this.pendingTranscriptions.delete(event.item_id);
this._onTurnSettled?.();
break;
}
// Log-only by convention: completedSegments holds only server-confirmed
// transcripts and onError is reserved for `error` events, so a failed
// turn counts toward nothing and must not look like a connection problem.
case "conversation.item.input_audio_transcription.failed": {
debugLogger.debug(
`${this.providerLabel} turn transcription failed`,
this._logContext({
itemId: event.item_id,
error: event.error?.message || event.error?.code || null,
})
);
this.pendingTranscriptions.delete(event.item_id);
this._onTurnSettled?.();
break;
}
case "input_audio_buffer.speech_started":
this.speechStartedAt = Date.now();
this.speechStartedCount++;
this._logVadEvent(event.type);
break;
case "input_audio_buffer.committed":
this.serverAcksCommits = true;
if (event.item_id) this.pendingTranscriptions.add(event.item_id);
this._onTurnSettled?.("committed");
this._logVadEvent(event.type);
break;
case "input_audio_buffer.speech_stopped":
this._logVadEvent(event.type);
break;
case "error": {
const errCode = event.error?.code;
const errMsg = event.error?.message || `${this.providerLabel} error`;
// Only consumers that attach onSessionExpired (meetings) get the
// reconnect path; others (dictation) keep the onError/onSessionEnd flow.
if (errCode === "session_expired" && this.onSessionExpired) {
debugLogger.warn(
`${this.providerLabel} session expired`,
this._logContext({ message: errMsg })
);
this._sessionExpired = true;
this.onSessionExpired({ proactive: false });
break;
}
const isEmptyBuffer =
errCode === "input_audio_buffer_commit_empty" ||
errMsg.includes("buffer too small") ||
errMsg.includes("commit_empty");
if (isEmptyBuffer) {
debugLogger.debug(
`${this.providerLabel} empty buffer (server VAD already committed)`,
this._logContext({
code: errCode,
})
);
} else {
debugLogger.error(
`${this.providerLabel} error event`,
this._logContext({
code: errCode,
message: errMsg,
})
);
}
this.onError?.(new Error(errMsg));
break;
}
default:
break;
}
} catch (err) {
debugLogger.error(
`${this.providerLabel} message parse error`,
this._logContext({ error: err.message })
);
}
}
// Unlabelled sockets (dictation) keep their pre-label log shape exactly.
_logContext(extra) {
if (this.streamLabel == null) return extra;
return { stream: this.streamLabel, ...extra };
}
// Throttled to first-3-then-every-50th: a normal meeting produces hundreds of
// these, so ZERO over a whole meeting means server VAD never fired.
_logVadEvent(type) {
this._vadEventCount++;
if (this._vadEventCount <= 3 || this._vadEventCount % 50 === 0) {
debugLogger.debug(
`${this.providerLabel} VAD event`,
this._logContext({
event: type,
count: this._vadEventCount,
audioBytesSent: this.audioBytesSent,
})
);
}
}
_markConnected() {
this.isConnected = true;
this.isConnecting = false;
clearTimeout(this.connectionTimeout);
this.startKeepAlive();
this._startSessionTimer();
if (this.pendingResolve) {
this.pendingResolve();
this.pendingResolve = null;
this.pendingReject = null;
}
}
_notifyConnectionLost(error) {
if (this._connectionLossNotified) return;
this._connectionLossNotified = true;
if (this.onConnectionLost) {
this.onConnectionLost(error);
} else {
this.onError?.(error);
}
}
_startSessionTimer() {
clearTimeout(this._sessionTimer);
this._sessionTimer = setTimeout(() => {
if (!this.isConnected) return;
debugLogger.debug(
`${this.providerLabel} session approaching 60min limit, requesting reconnect`,
this._logContext()
);
this.onSessionExpired?.({ proactive: true });
}, SESSION_PREEMPT_MS);
}
// A warm connection sits idle between dictations for up to 5 minutes and is
// reused with no other liveness check; a network path that dies silently
// (NAT/firewall drop, sleep/wake, VPN toggle) leaves isConnected stuck true
// and the next recording gets sent into a dead socket.
startKeepAlive() {
this.stopKeepAlive();
const socket = this.ws;
if (!socket) return;
socket.isAlive = true;
socket.on("pong", () => {
socket.isAlive = true;
});
this.keepAliveInterval = setInterval(() => {
if (socket !== this.ws || socket.readyState !== WebSocket.OPEN) {
this.stopKeepAlive();
return;
}
if (socket.isAlive === false) {
debugLogger.debug(
`${this.providerLabel} keep-alive missed pong, terminating stale connection`,
this._logContext()
);
socket.terminate();
return;
}
socket.isAlive = false;
try {
socket.ping();
} catch (err) {
debugLogger.debug(
`${this.providerLabel} keep-alive ping failed`,
this._logContext({ error: err.message })
);
socket.terminate();
}
}, KEEPALIVE_INTERVAL_MS);
}
stopKeepAlive() {
if (this.keepAliveInterval) {
clearInterval(this.keepAliveInterval);
this.keepAliveInterval = null;
}
}
// OpenAI rejects PCM session rates below 24kHz, so 16kHz capture can't be
// declared as-is; it is upsampled to the declared rate before sending.
_resampleToInputRate(pcmBuffer) {
if (this.captureRate === this.inputRate) return pcmBuffer;
const src = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);
const ratio = this.inputRate / this.captureRate;
const out = new Int16Array(Math.floor(src.length * ratio));
for (let i = 0; i < out.length; i++) {
const pos = i / ratio;
const low = Math.floor(pos);
const high = Math.min(low + 1, src.length - 1);
out[i] = Math.round(src[low] + (src[high] - src[low]) * (pos - low));
}
return Buffer.from(out.buffer);
}
sendAudio(pcmBuffer) {
const isOpen = this.ws?.readyState === WebSocket.OPEN;
if (!isOpen) {
if (this.bufferingAudio && this.coldStartBufferSize < COLD_START_BUFFER_MAX) {
const copy = Buffer.from(pcmBuffer);
this.coldStartBuffer.push(copy);
this.coldStartBufferSize += copy.length;
}
return false;
}
if (this.coldStartBuffer.length > 0) {
debugLogger.debug(
`${this.providerLabel} flushing cold-start buffer`,
this._logContext({
chunks: this.coldStartBuffer.length,
bytes: this.coldStartBufferSize,
})
);
for (const buf of this.coldStartBuffer) {
const audio = this._resampleToInputRate(buf);
this.ws.send(
JSON.stringify({ type: "input_audio_buffer.append", audio: audio.toString("base64") })
);
this.audioBytesSent += audio.length;
}
this.coldStartBuffer = [];
this.coldStartBufferSize = 0;
}
const audio = this._resampleToInputRate(Buffer.from(pcmBuffer));
this.ws.send(
JSON.stringify({ type: "input_audio_buffer.append", audio: audio.toString("base64") })
);
this.audioBytesSent += audio.length;
return true;
}
async disconnect({ commit = true } = {}) {
debugLogger.debug(
`${this.providerLabel} disconnect`,
this._logContext({
audioBytesSent: this.audioBytesSent,
segments: this.completedSegments.length,
textLength: this.getFullTranscript().length,
// Discriminates "VAD never fired" from "turns fired but empty/failed".
speechStartedCount: this.speechStartedCount,
readyState: this.ws?.readyState,
})
);
if (!this.ws) return { text: this.getFullTranscript() };
this.isDisconnecting = true;
if (this.ws.readyState === WebSocket.CONNECTING) {
this.ws.once("open", () => this.ws?.close());
const result = { text: this.getFullTranscript() };
this.isDisconnecting = false;
return result;
}
if (this.ws.readyState === WebSocket.OPEN) {
if (commit && this.audioBytesSent > 0) {
const prevOnError = this.onError;
const timeoutMs = isLiveTranscribe(this.model)
? LIVE_TRANSCRIBE_COMMIT_TIMEOUT_MS
: DISCONNECT_TIMEOUT_MS;
await new Promise((resolve) => {
const done = () => {
clearTimeout(tid);
this._onTurnSettled = null;
this.onError = prevOnError;
resolve();
};
const tid = setTimeout(() => {
debugLogger.debug(
`${this.providerLabel} commit timeout, using accumulated text`,
this._logContext()
);
done();
}, timeoutMs);
// VAD can have several committed turns still transcribing. A terminal
// event ends the wait only after all known items have settled and,
// since an earlier turn can finish before our commit lands, after
// the server acks that commit. Servers that never send `committed`
// (Tinfoil) settle on the terminal event alone.
let commitAcknowledged = false;
this._onTurnSettled = (reason) => {
if (reason === "committed") commitAcknowledged = true;
const acknowledged = commitAcknowledged || !this.serverAcksCommits;
if (reason === "closed" || (acknowledged && !this.pendingTranscriptions.size)) {
done();
}
};
this.onError = (err) => {
if (
err?.message?.includes("buffer too small") ||
err?.message?.includes("commit_empty")
) {
this._onTurnSettled?.("committed");
} else {
prevOnError?.(err);
}
};
try {
this.ws.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
} catch {
done();
}
});
}
this.ws?.close();
}
const result = { text: this.getFullTranscript() };
this.cleanup();
this.isDisconnecting = false;
return result;
}
cleanup() {
clearTimeout(this.connectionTimeout);
this.connectionTimeout = null;
clearTimeout(this._sessionTimer);
this._sessionTimer = null;
this.stopKeepAlive();
// A socket that dies mid-commit can never deliver the turn.
this._onTurnSettled?.("closed");
if (this.ws) {
try {
this.ws.close();
} catch {}
this.ws = null;
}
this.isConnected = false;
this.isConnecting = false;
this.bufferingAudio = false;
}
}
module.exports = OpenAIRealtimeStreaming;
module.exports.normalizeLanguage = normalizeLanguage;