Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

18.3 KB
Raw Blame
const WebSocket = require("ws");
const {
  ORUKEET_MODEL,
  ORUKEET_PCM_PROTOCOL,
  ORUKEET_PIPELINE_PROTOCOL,
  ORUKEET_CLEANUP_MODEL,
  ORUKEET_CLEANUP_RUNTIME_MODELS,
  isOrukeetClientToken,
} = require("./orukeetProtocol");

const MAX_PENDING_BYTES = 2 * 1024 * 1024;
// Mono 16 kHz PCM16.
const BYTES_PER_SECOND = 32000;
const MAX_FINAL_WAIT_MS = 30000;
const PIPELINE_CLEANUP_BUDGET_MS = 15000;
const PIPELINE_FINAL_MARGIN_MS = 2000;
const finiteMetric = (value) => (Number.isFinite(value) && value >= 0 ? value : undefined);

// Managed Cloud keeps the capture for a batch upload, so a stalled or
// unreachable GPU host should fail over within seconds rather than hold the
// user for the full 30 s budget. BYOK servers keep the defaults: they have no
// fallback, and a busy self-hosted CPU may legitimately answer slowly.
const MANAGED_STREAM_OPTIONS = {
  // A refused commit must close this attempt instead of retrying for 30 s.
  retryCapacity: false,
  timeoutMs: 10000,
  // Long enough to ride out a Wi-Fi roam or cellular hand-off.
  livenessMs: 8000,
  finalTimeoutMs: (audioSeconds) => 5000 + audioSeconds * 100,
};

function languageMetadata(message) {
  const language = message.language;
  const confidence = message.language_confidence;
  const supportedScore = message.language_supported_score;
  const hasSupportedScore =
    Number.isFinite(supportedScore) && supportedScore >= 0 && supportedScore <= 1;
  const extra = {
    ...(hasSupportedScore ? { languageSupportedScore: supportedScore } : {}),
    ...(Number.isFinite(message.language_audio_seconds) && message.language_audio_seconds > 0
      ? { languageAudioSeconds: message.language_audio_seconds }
      : {}),
  };
  if (
    typeof language === "string" &&
    /^[a-z]{2}$/.test(language) &&
    Number.isFinite(confidence) &&
    confidence >= 0 &&
    confidence <= 1
  ) {
    return {
      language,
      languageConfidence: confidence,
      ...extra,
    };
  }
  return { language: null, languageConfidence: null, ...(hasSupportedScore ? extra : {}) };
}

function capacityRetryDelay(requestedMs) {
  if (!Number.isFinite(requestedMs)) return 100;
  // Select a fixed local delay, rounding the server's hint up. Large hints
  // wait for the final deadline instead of overflowing Node's timer to 1 ms.
  for (const delay of [20, 50, 100, 250, 500, 1000, 2000, 5000, 10000, 30000]) {
    if (requestedMs <= delay) return delay;
  }
  return 30000;
}

function streamingUrl(baseUrl) {
  const url = new URL(baseUrl);
  const local = ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname);
  if (url.username || url.password || url.search || url.hash) {
    throw new Error("Use an Orukeet server URL without credentials or query parameters");
  }
  if (
    !["https:", "wss:"].includes(url.protocol) &&
    !(local && ["http:", "ws:"].includes(url.protocol))
  ) {
    throw new Error("Orukeet streaming requires HTTPS");
  }
  url.protocol = ["https:", "wss:"].includes(url.protocol) ? "wss:" : "ws:";
  url.pathname =
    url.pathname.replace(/\/$/, "").replace(/\/v1(?:\/audio\/transcriptions(?:\/stream)?)?$/, "") +
    "/v1/audio/transcriptions/stream";
  return url.toString();
}

// Upload PCM while recording; commit has an explicit final acknowledgement.
// Credentials stay in the main process and are sent only to the selected host.
class OrukeetStreaming {
  constructor({
    createSocket = (url, options, protocols) => new WebSocket(url, protocols, options),
    timeoutMs = 30000,
    retryCapacity = true,
    livenessMs = null,
    finalTimeoutMs = () => timeoutMs,
  } = {}) {
    this.createSocket = createSocket;
    this.timeoutMs = timeoutMs;
    this.retryCapacity = retryCapacity;
    this.livenessMs = livenessMs;
    this.finalTimeoutMs = finalTimeoutMs;
    this.ws = null;
    this.isConnected = false;
    this.pendingAudio = [];
    this.pendingBytes = 0;
    this.audioBytesSent = 0;
    this.currentModel = ORUKEET_MODEL;
    this.result = null;
    this.finalPromise = null;
    this.intentionalClose = false;
  }

  beginConnecting() {
    this.connecting = true;
  }

  async connect({
    baseUrl,
    apiKey,
    clientToken,
    protocol = ORUKEET_PCM_PROTOCOL,
    requireAccountLimits = false,
  }) {
    if (this.intentionalClose || this.failure || this.ws) {
      throw new Error("Create a new Orukeet adapter for each recording");
    }
    if (Boolean(apiKey) === Boolean(clientToken)) {
      throw new Error("Supply exactly one Orukeet service key or client token");
    }
    const pipeline = protocol === ORUKEET_PIPELINE_PROTOCOL;
    if (![ORUKEET_PCM_PROTOCOL, ORUKEET_PIPELINE_PROTOCOL].includes(protocol))
      throw new Error("Invalid protocol");
    if (clientToken && !isOrukeetClientToken(clientToken, pipeline)) {
      throw new Error("Invalid Orukeet client token");
    }
    this.pipeline = pipeline;
    this.requireAccountLimits = requireAccountLimits || pipeline;
    const url = pipeline
      ? streamingUrl(baseUrl).replace("/v1/audio/transcriptions/stream", "/v1/pipeline/stream")
      : streamingUrl(baseUrl);
    this.beginConnecting();
    this.ws = this.createSocket(
      url,
      {
        headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : {},
        perMessageDeflate: false,
        handshakeTimeout: this.timeoutMs,
        maxPayload: MAX_PENDING_BYTES,
      },
      clientToken ? [protocol, `auth.${clientToken}`] : undefined
    );
    return new Promise((resolve, reject) => {
      const timer = setTimeout(
        () => this.fail(new Error("Orukeet connection timed out")),
        this.timeoutMs
      );
      this.connectResolve = () => {
        clearTimeout(timer);
        resolve();
      };
      this.connectReject = (error) => {
        clearTimeout(timer);
        reject(error);
      };
      this.ws.on("message", (data) => {
        try {
          this.handleMessage(JSON.parse(data.toString()));
        } catch (error) {
          this.fail(error);
        }
      });
      this.ws.on("error", (error) => this.fail(error));
      this.ws.on("close", () => {
        if (!this.intentionalClose)
          this.fail(new Error("Orukeet connection closed before completion"));
        this.isConnected = false;
      });
    });
  }

  handleMessage(message) {
    if (this.intentionalClose || this.failure) return;
    if (message.type === "ready") {
      if (this.isConnected) throw new Error("Duplicate Orukeet ready message");
      if (
        message.sample_rate !== 16000 ||
        message.encoding !== "pcm_s16le" ||
        message.channels !== 1 ||
        !Number.isFinite(message.max_seconds) ||
        message.max_seconds <= 0 ||
        message.max_seconds > 600
      ) {
        throw new Error("Orukeet server does not support mono 16 kHz PCM");
      }
      if (this.requireAccountLimits && message.account_limits !== true)
        throw new Error("Orukeet account limits missing");
      if (this.pipeline && (message.pipeline_protocol !== 2 || message.cleanup !== true))
        throw new Error("Invalid pipeline handshake");
      this.maxAudioBytes = message.max_seconds * BYTES_PER_SECOND;
      this.isConnected = true;
      this.connecting = false;
      for (const data of this.pendingAudio) this.writeAudio(data);
      this.pendingAudio = [];
      this.pendingBytes = 0;
      this.connectResolve?.();
      this.connectResolve = this.connectReject = null;
      this.keepAlive = setInterval(() => {
        if (this.isConnected) this.sendControl({ type: "ping" });
      }, 15000);
      this.keepAlive.unref?.();
      this.startLivenessCheck();
    } else if (message.type === "language") {
      // Advisory audio-language metadata never commits a transcript or triggers a paste.
      // The final message remains authoritative for this recording.
      if (!this.isConnected || this.result) return;
      const metadata = languageMetadata(message);
      if (metadata.language || Number.isFinite(metadata.languageSupportedScore))
        this.onLanguage?.(metadata);
    } else if (message.type === "transcript") {
      if (!this.pipeline || !this.finalResolve || typeof message.text !== "string") return;
      // This is ASR completion, before cleanup starts. Never paste this provisional
      // event; retain its timing separately from the eventual combined final, and
      // its text in case cleanup never finishes.
      this.asrFinalMs ??= Math.max(0, performance.now() - this.finalStartedAt);
      this.rawTranscript = message.text;
    } else if (message.type === "final") {
      // A duplicate or unsolicited final must never result in a second paste.
      if (this.result) return;
      if (!this.finalResolve || typeof message.text !== "string") {
        throw new Error("Unexpected Orukeet final transcript");
      }
      if (
        this.pipeline &&
        (typeof message.raw_text !== "string" ||
          !["complete", "fallback", "skipped"].includes(message.cleanup_status))
      )
        throw new Error("Invalid pipeline final");
      if (
        this.pipeline &&
        message.cleanup_receipt !== undefined &&
        (typeof message.cleanup_receipt !== "string" || message.cleanup_receipt.length > 2048)
      )
        throw new Error("Invalid cleanup receipt");
      this.result = {
        text: this.pipeline ? message.raw_text : message.text || "",
        ...(this.pipeline
          ? {
              cleanupText: message.text,
              cleanupStatus: message.cleanup_status,
              cleanupOptions: this.cleanupOptions,
              asrFinalMs: this.asrFinalMs ?? Math.max(0, performance.now() - this.finalStartedAt),
              cleanup: {
                status: message.cleanup_status,
                ...(message.cleanup_status === "complete" &&
                message.cleanup_method === "model" &&
                ORUKEET_CLEANUP_RUNTIME_MODELS.includes(message.cleanup_model)
                  ? { model: ORUKEET_CLEANUP_MODEL }
                  : {}),
                processingMs: finiteMetric(message.cleanup_ms),
                queueMs: finiteMetric(message.cleanup_queue_ms),
                firstTextMs: finiteMetric(message.cleanup_first_text_ms),
                inputTokens: finiteMetric(message.cleanup_input_tokens),
                outputTokens: finiteMetric(message.cleanup_output_tokens),
              },
            }
          : this.cleanupUnavailable
            ? { cleanup: { status: "fallback" } }
            : {}),
        model: message.model || this.currentModel,
        audioBytesSent: this.audioBytesSent,
        inferenceMs: message.inference_ms,
        serverMs: message.server_ms,
        queueMs: message.queue_ms,
        ...("language" in message ? languageMetadata(message) : {}),
      };
      if (this.pipeline && message.cleanup_receipt) this.onUsageReceipt?.(message.cleanup_receipt);
      this.settleFinal();
    } else if (message.type === "error") {
      if (message.code === "capacity" && this.finalResolve && this.retryCapacity) {
        clearTimeout(this.retryTimer);
        this.retryTimer = setTimeout(() => {
          if (this.finalResolve) this.sendControl({ type: "commit" });
        }, capacityRetryDelay(message.retry_after_ms));
      } else {
        this.fail(new Error(message.message || `Orukeet transcription failed: ${message.code}`));
      }
    }
  }

  // A wedged server can keep its TCP connection open, so a stall only shows up
  // as missing pongs. The check arms on the first pong: a server that never
  // answers pings keeps today's behaviour instead of failing every recording.
  // A pending commit is bounded by the final deadline instead, since inference
  // may hold the server's event loop.
  startLivenessCheck() {
    if (!this.livenessMs) return;
    const intervalMs = this.livenessMs / 2;
    let lastHeardAt = null;
    let lastCheckAt = performance.now();
    this.ws.on("pong", () => {
      lastHeardAt = performance.now();
    });
    this.ws.on("message", () => {
      if (lastHeardAt !== null) lastHeardAt = performance.now();
    });
    const check = () => {
      if (this.intentionalClose || this.failure || this.finalPromise) return;
      const now = performance.now();
      const late = now - lastCheckAt >= intervalMs * 1.5;
      lastCheckAt = now;
      if (lastHeardAt !== null) {
        // A late tick means this process was held up (a blocked main thread or
        // App Nap), not the server, so the next ping gets a full window.
        if (late) lastHeardAt = now;
        else if (now - lastHeardAt > this.livenessMs) {
          return this.fail(new Error("Orukeet server stopped responding"));
        }
      }
      this.ws.ping();
    };
    check();
    // After a blocked main thread, timers run before pending socket reads;
    // setImmediate lets a pong that already arrived count first.
    this.livenessTimer = setInterval(() => setImmediate(check), intervalMs);
    this.livenessTimer.unref?.();
  }

  sendAudio(data) {
    if (this.intentionalClose || this.failure || this.finalPromise) return;
    const buffer = Buffer.from(data);
    if (buffer.length % 2) return this.fail(new Error("Orukeet requires complete PCM16 samples"));
    if (this.isConnected) {
      try {
        this.writeAudio(buffer);
      } catch (error) {
        this.fail(error);
      }
    } else if (this.connecting && this.pendingBytes + buffer.length <= MAX_PENDING_BYTES) {
      this.pendingAudio.push(buffer);
      this.pendingBytes += buffer.length;
    } else this.fail(new Error("Orukeet audio upload could not keep up"));
  }

  writeAudio(buffer) {
    if (
      this.ws.readyState !== WebSocket.OPEN ||
      this.ws.bufferedAmount + buffer.length > MAX_PENDING_BYTES ||
      this.audioBytesSent + buffer.length > this.maxAudioBytes
    ) {
      throw new Error("Orukeet recording exceeds the server or upload capacity");
    }
    this.ws.send(buffer);
    this.audioBytesSent += buffer.length;
  }

  sendControl(message, onSent) {
    try {
      if (!this.isConnected || this.ws.readyState !== WebSocket.OPEN) {
        throw new Error("Orukeet connection closed before completion");
      }
      this.ws.send(JSON.stringify(message), onSent);
    } catch (error) {
      this.fail(error);
    }
  }

  finalize() {
    if (this.failure) return Promise.reject(this.failure);
    if (this.result) return Promise.resolve(this.result);
    if (this.finalPromise) return this.finalPromise;
    if (!this.isConnected) return Promise.reject(new Error("Orukeet connection is not ready"));
    if (!this.audioBytesSent)
      return Promise.resolve({ text: "", model: this.currentModel, audioBytesSent: 0 });
    this.finalStartedAt = performance.now();
    this.finalPromise = new Promise((resolve, reject) => {
      this.finalResolve = resolve;
      this.finalReject = reject;
      const timedOut = () => this.fail(new Error("Orukeet final transcript timed out"));
      const audioSeconds = this.audioBytesSent / BYTES_PER_SECOND;
      // The commit queues behind audio still uploading, so the scaled deadline
      // starts once it is written; the cap bounds a slow upload.
      this.finalCapTimer = setTimeout(timedOut, MAX_FINAL_WAIT_MS);
      this.sendControl({ type: "commit" }, (error) => {
        if (error || !this.finalResolve) return;
        this.finalTimer = setTimeout(
          timedOut,
          this.pipeline
            ? Math.min(
                MAX_FINAL_WAIT_MS,
                this.finalTimeoutMs(audioSeconds) +
                  PIPELINE_CLEANUP_BUDGET_MS +
                  PIPELINE_FINAL_MARGIN_MS
              )
            : this.finalTimeoutMs(audioSeconds)
        );
      });
    });
    return this.finalPromise;
  }

  settleFinal() {
    this.onFinalTranscript?.(this.result.text);
    this.finalResolve?.(this.result);
    this.clearFinal();
  }

  clearFinal() {
    clearTimeout(this.finalCapTimer);
    clearTimeout(this.finalTimer);
    clearTimeout(this.retryTimer);
    this.finalResolve = this.finalReject = null;
  }

  fail(error) {
    if (this.failure) return;
    // The final already completed this recording. Orukeet closes a finished
    // socket when the account holds another warm one, so a later close or
    // error only ends the connection.
    if (this.result) return this.close();
    // ASR already finished when cleanup stalled or the socket dropped: keep its
    // text, which is pasted without cleanup, instead of transcribing again.
    if (this.pipeline && this.finalResolve && typeof this.rawTranscript === "string") {
      this.result = {
        text: this.rawTranscript,
        cleanupError: error.message,
        asrFinalMs: this.asrFinalMs,
        cleanup: { status: "fallback" },
        model: this.currentModel,
        audioBytesSent: this.audioBytesSent,
      };
      this.settleFinal();
      this.ws?.terminate();
      return this.close();
    }
    this.failure = error;
    // Before `ready` (including while main mints the session token, before
    // connect() runs), the failed start is the report and falls back on it;
    // onError is for an established stream, so a refused socket does not also
    // surface as a streaming error.
    const connecting = this.connecting || Boolean(this.connectReject);
    this.connectReject?.(error);
    this.connectResolve = this.connectReject = null;
    this.finalReject?.(error);
    this.clearFinal();
    this.close();
    if (!connecting) this.onError?.(error);
  }

  close() {
    if (this.intentionalClose) return;
    this.intentionalClose = true;
    clearInterval(this.keepAlive);
    clearInterval(this.livenessTimer);
    this.isConnected = false;
    this.connecting = false;
    this.pendingAudio = [];
    this.pendingBytes = 0;
    // A failed socket has nothing left to say, and a graceful close would wait
    // behind unsent audio for up to 30 s while the account's socket slot stays taken.
    if (this.failure) this.ws?.terminate();
    else this.ws?.close();
    this.onClose?.();
  }

  async disconnect() {
    // Stop is also used for cancellation and idle cleanup: only finalize()
    // commits audio, so cancelled recordings never submit inference.
    const result = this.result || {
      text: "",
      model: this.currentModel,
      audioBytesSent: this.audioBytesSent,
    };
    this.connectReject?.(new Error("Orukeet connection cancelled"));
    this.connectResolve = this.connectReject = null;
    this.close();
    if (this.finalReject) this.finalReject(new Error("Orukeet recording cancelled"));
    this.clearFinal();
    return result;
  }
}

module.exports = {
  OrukeetStreaming,
  streamingUrl,
  MANAGED_STREAM_OPTIONS,
  MAX_FINAL_WAIT_MS,
  PIPELINE_CLEANUP_BUDGET_MS,
  PIPELINE_FINAL_MARGIN_MS,
};