Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
const WebSocket = require("ws");
const {
ORUKEET_MODEL,
ORUKEET_PCM_PROTOCOL,
ORUKEET_PIPELINE_PROTOCOL,
ORUKEET_CLEANUP_MODEL,
ORUKEET_CLEANUP_RUNTIME_MODELS,
isOrukeetClientToken,
} = require("./orukeetProtocol");
const MAX_PENDING_BYTES = 2 * 1024 * 1024;
// Mono 16 kHz PCM16.
const BYTES_PER_SECOND = 32000;
const MAX_FINAL_WAIT_MS = 30000;
const PIPELINE_CLEANUP_BUDGET_MS = 15000;
const PIPELINE_FINAL_MARGIN_MS = 2000;
const finiteMetric = (value) => (Number.isFinite(value) && value >= 0 ? value : undefined);
// Managed Cloud keeps the capture for a batch upload, so a stalled or
// unreachable GPU host should fail over within seconds rather than hold the
// user for the full 30 s budget. BYOK servers keep the defaults: they have no
// fallback, and a busy self-hosted CPU may legitimately answer slowly.
const MANAGED_STREAM_OPTIONS = {
// A refused commit must close this attempt instead of retrying for 30 s.
retryCapacity: false,
timeoutMs: 10000,
// Long enough to ride out a Wi-Fi roam or cellular hand-off.
livenessMs: 8000,
finalTimeoutMs: (audioSeconds) => 5000 + audioSeconds * 100,
};
function languageMetadata(message) {
const language = message.language;
const confidence = message.language_confidence;
const supportedScore = message.language_supported_score;
const hasSupportedScore =
Number.isFinite(supportedScore) && supportedScore >= 0 && supportedScore <= 1;
const extra = {
...(hasSupportedScore ? { languageSupportedScore: supportedScore } : {}),
...(Number.isFinite(message.language_audio_seconds) && message.language_audio_seconds > 0
? { languageAudioSeconds: message.language_audio_seconds }
: {}),
};
if (
typeof language === "string" &&
/^[a-z]{2}$/.test(language) &&
Number.isFinite(confidence) &&
confidence >= 0 &&
confidence <= 1
) {
return {
language,
languageConfidence: confidence,
...extra,
};
}
return { language: null, languageConfidence: null, ...(hasSupportedScore ? extra : {}) };
}
function capacityRetryDelay(requestedMs) {
if (!Number.isFinite(requestedMs)) return 100;
// Select a fixed local delay, rounding the server's hint up. Large hints
// wait for the final deadline instead of overflowing Node's timer to 1 ms.
for (const delay of [20, 50, 100, 250, 500, 1000, 2000, 5000, 10000, 30000]) {
if (requestedMs <= delay) return delay;
}
return 30000;
}
function streamingUrl(baseUrl) {
const url = new URL(baseUrl);
const local = ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname);
if (url.username || url.password || url.search || url.hash) {
throw new Error("Use an Orukeet server URL without credentials or query parameters");
}
if (
!["https:", "wss:"].includes(url.protocol) &&
!(local && ["http:", "ws:"].includes(url.protocol))
) {
throw new Error("Orukeet streaming requires HTTPS");
}
url.protocol = ["https:", "wss:"].includes(url.protocol) ? "wss:" : "ws:";
url.pathname =
url.pathname.replace(/\/$/, "").replace(/\/v1(?:\/audio\/transcriptions(?:\/stream)?)?$/, "") +
"/v1/audio/transcriptions/stream";
return url.toString();
}
// Upload PCM while recording; commit has an explicit final acknowledgement.
// Credentials stay in the main process and are sent only to the selected host.
class OrukeetStreaming {
constructor({
createSocket = (url, options, protocols) => new WebSocket(url, protocols, options),
timeoutMs = 30000,
retryCapacity = true,
livenessMs = null,
finalTimeoutMs = () => timeoutMs,
} = {}) {
this.createSocket = createSocket;
this.timeoutMs = timeoutMs;
this.retryCapacity = retryCapacity;
this.livenessMs = livenessMs;
this.finalTimeoutMs = finalTimeoutMs;
this.ws = null;
this.isConnected = false;
this.pendingAudio = [];
this.pendingBytes = 0;
this.audioBytesSent = 0;
this.currentModel = ORUKEET_MODEL;
this.result = null;
this.finalPromise = null;
this.intentionalClose = false;
}
beginConnecting() {
this.connecting = true;
}
async connect({
baseUrl,
apiKey,
clientToken,
protocol = ORUKEET_PCM_PROTOCOL,
requireAccountLimits = false,
}) {
if (this.intentionalClose || this.failure || this.ws) {
throw new Error("Create a new Orukeet adapter for each recording");
}
if (Boolean(apiKey) === Boolean(clientToken)) {
throw new Error("Supply exactly one Orukeet service key or client token");
}
const pipeline = protocol === ORUKEET_PIPELINE_PROTOCOL;
if (![ORUKEET_PCM_PROTOCOL, ORUKEET_PIPELINE_PROTOCOL].includes(protocol))
throw new Error("Invalid protocol");
if (clientToken && !isOrukeetClientToken(clientToken, pipeline)) {
throw new Error("Invalid Orukeet client token");
}
this.pipeline = pipeline;
this.requireAccountLimits = requireAccountLimits || pipeline;
const url = pipeline
? streamingUrl(baseUrl).replace("/v1/audio/transcriptions/stream", "/v1/pipeline/stream")
: streamingUrl(baseUrl);
this.beginConnecting();
this.ws = this.createSocket(
url,
{
headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : {},
perMessageDeflate: false,
handshakeTimeout: this.timeoutMs,
maxPayload: MAX_PENDING_BYTES,
},
clientToken ? [protocol, `auth.${clientToken}`] : undefined
);
return new Promise((resolve, reject) => {
const timer = setTimeout(
() => this.fail(new Error("Orukeet connection timed out")),
this.timeoutMs
);
this.connectResolve = () => {
clearTimeout(timer);
resolve();
};
this.connectReject = (error) => {
clearTimeout(timer);
reject(error);
};
this.ws.on("message", (data) => {
try {
this.handleMessage(JSON.parse(data.toString()));
} catch (error) {
this.fail(error);
}
});
this.ws.on("error", (error) => this.fail(error));
this.ws.on("close", () => {
if (!this.intentionalClose)
this.fail(new Error("Orukeet connection closed before completion"));
this.isConnected = false;
});
});
}
handleMessage(message) {
if (this.intentionalClose || this.failure) return;
if (message.type === "ready") {
if (this.isConnected) throw new Error("Duplicate Orukeet ready message");
if (
message.sample_rate !== 16000 ||
message.encoding !== "pcm_s16le" ||
message.channels !== 1 ||
!Number.isFinite(message.max_seconds) ||
message.max_seconds <= 0 ||
message.max_seconds > 600
) {
throw new Error("Orukeet server does not support mono 16 kHz PCM");
}
if (this.requireAccountLimits && message.account_limits !== true)
throw new Error("Orukeet account limits missing");
if (this.pipeline && (message.pipeline_protocol !== 2 || message.cleanup !== true))
throw new Error("Invalid pipeline handshake");
this.maxAudioBytes = message.max_seconds * BYTES_PER_SECOND;
this.isConnected = true;
this.connecting = false;
for (const data of this.pendingAudio) this.writeAudio(data);
this.pendingAudio = [];
this.pendingBytes = 0;
this.connectResolve?.();
this.connectResolve = this.connectReject = null;
this.keepAlive = setInterval(() => {
if (this.isConnected) this.sendControl({ type: "ping" });
}, 15000);
this.keepAlive.unref?.();
this.startLivenessCheck();
} else if (message.type === "language") {
// Advisory audio-language metadata never commits a transcript or triggers a paste.
// The final message remains authoritative for this recording.
if (!this.isConnected || this.result) return;
const metadata = languageMetadata(message);
if (metadata.language || Number.isFinite(metadata.languageSupportedScore))
this.onLanguage?.(metadata);
} else if (message.type === "transcript") {
if (!this.pipeline || !this.finalResolve || typeof message.text !== "string") return;
// This is ASR completion, before cleanup starts. Never paste this provisional
// event; retain its timing separately from the eventual combined final, and
// its text in case cleanup never finishes.
this.asrFinalMs ??= Math.max(0, performance.now() - this.finalStartedAt);
this.rawTranscript = message.text;
} else if (message.type === "final") {
// A duplicate or unsolicited final must never result in a second paste.
if (this.result) return;
if (!this.finalResolve || typeof message.text !== "string") {
throw new Error("Unexpected Orukeet final transcript");
}
if (
this.pipeline &&
(typeof message.raw_text !== "string" ||
!["complete", "fallback", "skipped"].includes(message.cleanup_status))
)
throw new Error("Invalid pipeline final");
if (
this.pipeline &&
message.cleanup_receipt !== undefined &&
(typeof message.cleanup_receipt !== "string" || message.cleanup_receipt.length > 2048)
)
throw new Error("Invalid cleanup receipt");
this.result = {
text: this.pipeline ? message.raw_text : message.text || "",
...(this.pipeline
? {
cleanupText: message.text,
cleanupStatus: message.cleanup_status,
cleanupOptions: this.cleanupOptions,
asrFinalMs: this.asrFinalMs ?? Math.max(0, performance.now() - this.finalStartedAt),
cleanup: {
status: message.cleanup_status,
...(message.cleanup_status === "complete" &&
message.cleanup_method === "model" &&
ORUKEET_CLEANUP_RUNTIME_MODELS.includes(message.cleanup_model)
? { model: ORUKEET_CLEANUP_MODEL }
: {}),
processingMs: finiteMetric(message.cleanup_ms),
queueMs: finiteMetric(message.cleanup_queue_ms),
firstTextMs: finiteMetric(message.cleanup_first_text_ms),
inputTokens: finiteMetric(message.cleanup_input_tokens),
outputTokens: finiteMetric(message.cleanup_output_tokens),
},
}
: this.cleanupUnavailable
? { cleanup: { status: "fallback" } }
: {}),
model: message.model || this.currentModel,
audioBytesSent: this.audioBytesSent,
inferenceMs: message.inference_ms,
serverMs: message.server_ms,
queueMs: message.queue_ms,
...("language" in message ? languageMetadata(message) : {}),
};
if (this.pipeline && message.cleanup_receipt) this.onUsageReceipt?.(message.cleanup_receipt);
this.settleFinal();
} else if (message.type === "error") {
if (message.code === "capacity" && this.finalResolve && this.retryCapacity) {
clearTimeout(this.retryTimer);
this.retryTimer = setTimeout(() => {
if (this.finalResolve) this.sendControl({ type: "commit" });
}, capacityRetryDelay(message.retry_after_ms));
} else {
this.fail(new Error(message.message || `Orukeet transcription failed: ${message.code}`));
}
}
}
// A wedged server can keep its TCP connection open, so a stall only shows up
// as missing pongs. The check arms on the first pong: a server that never
// answers pings keeps today's behaviour instead of failing every recording.
// A pending commit is bounded by the final deadline instead, since inference
// may hold the server's event loop.
startLivenessCheck() {
if (!this.livenessMs) return;
const intervalMs = this.livenessMs / 2;
let lastHeardAt = null;
let lastCheckAt = performance.now();
this.ws.on("pong", () => {
lastHeardAt = performance.now();
});
this.ws.on("message", () => {
if (lastHeardAt !== null) lastHeardAt = performance.now();
});
const check = () => {
if (this.intentionalClose || this.failure || this.finalPromise) return;
const now = performance.now();
const late = now - lastCheckAt >= intervalMs * 1.5;
lastCheckAt = now;
if (lastHeardAt !== null) {
// A late tick means this process was held up (a blocked main thread or
// App Nap), not the server, so the next ping gets a full window.
if (late) lastHeardAt = now;
else if (now - lastHeardAt > this.livenessMs) {
return this.fail(new Error("Orukeet server stopped responding"));
}
}
this.ws.ping();
};
check();
// After a blocked main thread, timers run before pending socket reads;
// setImmediate lets a pong that already arrived count first.
this.livenessTimer = setInterval(() => setImmediate(check), intervalMs);
this.livenessTimer.unref?.();
}
sendAudio(data) {
if (this.intentionalClose || this.failure || this.finalPromise) return;
const buffer = Buffer.from(data);
if (buffer.length % 2) return this.fail(new Error("Orukeet requires complete PCM16 samples"));
if (this.isConnected) {
try {
this.writeAudio(buffer);
} catch (error) {
this.fail(error);
}
} else if (this.connecting && this.pendingBytes + buffer.length <= MAX_PENDING_BYTES) {
this.pendingAudio.push(buffer);
this.pendingBytes += buffer.length;
} else this.fail(new Error("Orukeet audio upload could not keep up"));
}
writeAudio(buffer) {
if (
this.ws.readyState !== WebSocket.OPEN ||
this.ws.bufferedAmount + buffer.length > MAX_PENDING_BYTES ||
this.audioBytesSent + buffer.length > this.maxAudioBytes
) {
throw new Error("Orukeet recording exceeds the server or upload capacity");
}
this.ws.send(buffer);
this.audioBytesSent += buffer.length;
}
sendControl(message, onSent) {
try {
if (!this.isConnected || this.ws.readyState !== WebSocket.OPEN) {
throw new Error("Orukeet connection closed before completion");
}
this.ws.send(JSON.stringify(message), onSent);
} catch (error) {
this.fail(error);
}
}
finalize() {
if (this.failure) return Promise.reject(this.failure);
if (this.result) return Promise.resolve(this.result);
if (this.finalPromise) return this.finalPromise;
if (!this.isConnected) return Promise.reject(new Error("Orukeet connection is not ready"));
if (!this.audioBytesSent)
return Promise.resolve({ text: "", model: this.currentModel, audioBytesSent: 0 });
this.finalStartedAt = performance.now();
this.finalPromise = new Promise((resolve, reject) => {
this.finalResolve = resolve;
this.finalReject = reject;
const timedOut = () => this.fail(new Error("Orukeet final transcript timed out"));
const audioSeconds = this.audioBytesSent / BYTES_PER_SECOND;
// The commit queues behind audio still uploading, so the scaled deadline
// starts once it is written; the cap bounds a slow upload.
this.finalCapTimer = setTimeout(timedOut, MAX_FINAL_WAIT_MS);
this.sendControl({ type: "commit" }, (error) => {
if (error || !this.finalResolve) return;
this.finalTimer = setTimeout(
timedOut,
this.pipeline
? Math.min(
MAX_FINAL_WAIT_MS,
this.finalTimeoutMs(audioSeconds) +
PIPELINE_CLEANUP_BUDGET_MS +
PIPELINE_FINAL_MARGIN_MS
)
: this.finalTimeoutMs(audioSeconds)
);
});
});
return this.finalPromise;
}
settleFinal() {
this.onFinalTranscript?.(this.result.text);
this.finalResolve?.(this.result);
this.clearFinal();
}
clearFinal() {
clearTimeout(this.finalCapTimer);
clearTimeout(this.finalTimer);
clearTimeout(this.retryTimer);
this.finalResolve = this.finalReject = null;
}
fail(error) {
if (this.failure) return;
// The final already completed this recording. Orukeet closes a finished
// socket when the account holds another warm one, so a later close or
// error only ends the connection.
if (this.result) return this.close();
// ASR already finished when cleanup stalled or the socket dropped: keep its
// text, which is pasted without cleanup, instead of transcribing again.
if (this.pipeline && this.finalResolve && typeof this.rawTranscript === "string") {
this.result = {
text: this.rawTranscript,
cleanupError: error.message,
asrFinalMs: this.asrFinalMs,
cleanup: { status: "fallback" },
model: this.currentModel,
audioBytesSent: this.audioBytesSent,
};
this.settleFinal();
this.ws?.terminate();
return this.close();
}
this.failure = error;
// Before `ready` (including while main mints the session token, before
// connect() runs), the failed start is the report and falls back on it;
// onError is for an established stream, so a refused socket does not also
// surface as a streaming error.
const connecting = this.connecting || Boolean(this.connectReject);
this.connectReject?.(error);
this.connectResolve = this.connectReject = null;
this.finalReject?.(error);
this.clearFinal();
this.close();
if (!connecting) this.onError?.(error);
}
close() {
if (this.intentionalClose) return;
this.intentionalClose = true;
clearInterval(this.keepAlive);
clearInterval(this.livenessTimer);
this.isConnected = false;
this.connecting = false;
this.pendingAudio = [];
this.pendingBytes = 0;
// A failed socket has nothing left to say, and a graceful close would wait
// behind unsent audio for up to 30 s while the account's socket slot stays taken.
if (this.failure) this.ws?.terminate();
else this.ws?.close();
this.onClose?.();
}
async disconnect() {
// Stop is also used for cancellation and idle cleanup: only finalize()
// commits audio, so cancelled recordings never submit inference.
const result = this.result || {
text: "",
model: this.currentModel,
audioBytesSent: this.audioBytesSent,
};
this.connectReject?.(new Error("Orukeet connection cancelled"));
this.connectResolve = this.connectReject = null;
this.close();
if (this.finalReject) this.finalReject(new Error("Orukeet recording cancelled"));
this.clearFinal();
return result;
}
}
module.exports = {
OrukeetStreaming,
streamingUrl,
MANAGED_STREAM_OPTIONS,
MAX_FINAL_WAIT_MS,
PIPELINE_CLEANUP_BUDGET_MS,
PIPELINE_FINAL_MARGIN_MS,
};