Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

16.9 KB
Raw Blame
const WebSocket = require("ws");
const debugLogger = require("./debugLogger");

const SAMPLE_RATE = 16000;
const WEBSOCKET_TIMEOUT_MS = 30000;
const TERMINATION_TIMEOUT_MS = 5000;
const KEEPALIVE_INTERVAL_MS = 15000;

// Corti's WSS transport: OAuth token in the query, a JSON config message after
// open, then raw PCM frames. Mirrors the AssemblyAI/Deepgram streaming classes.
class CortiStreaming {
  constructor() {
    this.ws = null;
    this.sessionId = null;
    this.isConnected = false;
    this.onPartialTranscript = null;
    this.onFinalTranscript = null;
    this.onError = null;
    this.onSessionEnd = null;
    this.onConnectionLost = null;
    this.connectionLossNotified = false;
    this.pendingResolve = null;
    this.pendingReject = null;
    this.connectionTimeout = null;
    this.accumulatedText = "";
    this.completedSegments = [];
    this.pendingAck = null;
    this.isDisconnecting = false;
    this.configAccepted = false;
    this.preConfigBuffer = [];
    this.preConfigBufferSize = 0;
    this.bufferingAudio = false;
    this.sessionStartedAt = null;
    this.audioBytesSent = 0;
    this.currentModel = "corti-transcribe";
    this.sampleRate = SAMPLE_RATE;
    this.warmConnection = null;
    this.warmConnectionReady = false;
    this.warmConnectionOptions = null;
    this.warmSessionId = null;
    this.warmSessionStartedAt = null;
    this.keepAliveInterval = null;
  }

  buildWebSocketUrl(options) {
    const params = new URLSearchParams({
      "tenant-name": options.tenant,
      token: `Bearer ${options.token}`,
    });
    return `wss://api.${options.environment}.corti.app/audio-bridge/v2/transcribe?${params}`;
  }

  buildConfiguration(options) {
    const configuration = {
      primaryLanguage: options.language && options.language !== "auto" ? options.language : "en",
      interimResults: true,
      automaticPunctuation: true,
      audioFormat: `audio/pcm; rate=${this.sampleRate}; channels=1; bits=16`,
    };
    if (options.keyterms && options.keyterms.length > 0) {
      configuration.keyterms = { terms: options.keyterms.map((term) => ({ term })) };
    }
    return configuration;
  }

  // Starts holding audio before the socket exists, covering the token mint, the
  // handshake and the wait for CONFIG_ACCEPTED so sendAudio() doesn't drop the
  // first words.
  beginConnecting() {
    this.bufferingAudio = true;
    this.preConfigBuffer = [];
    this.preConfigBufferSize = 0;
  }

  async connect(options = {}) {
    const { token, environment, tenant } = options;
    if (!token || !environment || !tenant) {
      throw new Error("Corti streaming requires token, environment, and tenant");
    }

    if (this.isConnected) {
      debugLogger.debug("Corti streaming already connected");
      return;
    }

    this.accumulatedText = "";
    this.completedSegments = [];
    this.configAccepted = false;
    this.connectionLossNotified = false;
    // The caller may already be holding audio from before its token mint.
    if (!this.bufferingAudio) this.beginConnecting();
    this.audioBytesSent = 0;
    this.sampleRate = options.sampleRate || SAMPLE_RATE;

    // A warm socket is bound to the region and tenant it opened against.
    if (
      this.hasWarmConnection() &&
      (this.warmConnectionOptions.environment !== environment ||
        this.warmConnectionOptions.tenant !== tenant)
    ) {
      debugLogger.debug("Corti warm connection differs, cold-starting", {
        warm: [this.warmConnectionOptions.environment, this.warmConnectionOptions.tenant],
        requested: [environment, tenant],
      });
      this.cleanupWarmConnection();
    }

    // Reuse the pre-warmed socket for an instant start; cold-connect otherwise.
    if (this.useWarmConnection()) {
      debugLogger.debug("Corti using warm connection - instant start");
      this.flushPreConfigBuffer();
      return;
    }

    const url = this.buildWebSocketUrl(options);
    const configuration = this.buildConfiguration(options);
    debugLogger.debug("Corti streaming connecting", { environment, tenant });

    return new Promise((resolve, reject) => {
      this.pendingResolve = resolve;
      this.pendingReject = reject;

      this.connectionTimeout = setTimeout(() => {
        this.cleanup();
        reject(new Error("Corti WebSocket connection timeout"));
      }, WEBSOCKET_TIMEOUT_MS);

      const ws = new WebSocket(url);
      this.ws = ws;
      ws.on("open", () => {
        debugLogger.debug("Corti WebSocket connected, sending config");
        ws.send(JSON.stringify({ type: "config", configuration }));
      });
      this.attachSocketHandlers(ws);
    });
  }

  // Live-socket wiring shared by the cold connect and a promoted warm connection.
  // A socket that cleanup() already replaced can still deliver an ack, error or
  // close; those events must not touch the start now using this.ws.
  attachSocketHandlers(ws) {
    ws.on("message", (data) => {
      if (ws !== this.ws) return;
      this.handleMessage(data);
    });

    ws.on("error", (error) => {
      if (ws !== this.ws) return;
      const wasActive = this.isConnected;
      debugLogger.error("Corti WebSocket error", { error: error.message });
      this.cleanup();
      if (this.pendingReject) {
        this.pendingReject(error);
        this.pendingReject = null;
        this.pendingResolve = null;
      }
      if (wasActive && !this.isDisconnecting) {
        this.notifyConnectionLost(error);
      } else if (!this.isDisconnecting) {
        this.onError?.(error);
      }
    });

    ws.on("close", (code, reason) => {
      if (ws !== this.ws) return;
      const wasActive = this.isConnected;
      debugLogger.debug("Corti WebSocket closed", {
        code,
        reason: reason?.toString(),
        wasActive,
      });
      if (this.pendingReject) {
        this.pendingReject(new Error(`Corti WebSocket closed before ready (code: ${code})`));
        this.pendingReject = null;
        this.pendingResolve = null;
      }
      this.resolvePendingAck();
      this.cleanup();
      if (wasActive && !this.isDisconnecting) {
        this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));
      }
    });
  }

  notifyConnectionLost(error) {
    if (this.connectionLossNotified) return;
    this.connectionLossNotified = true;
    if (this.onConnectionLost) {
      this.onConnectionLost(error);
    } else {
      this.onError?.(error);
    }
  }

  // Pre-open the socket and finish the config handshake before recording, so the
  // first words aren't lost waiting for CONFIG_ACCEPTED. Mirrors Deepgram/AssemblyAI.
  async warmup(options = {}) {
    const { token, environment, tenant } = options;
    if (!token || !environment || !tenant) {
      throw new Error("Corti warmup requires token, environment, and tenant");
    }
    if (this.warmConnection) {
      debugLogger.debug(
        this.warmConnectionReady
          ? "Corti connection already warm"
          : "Corti warmup already in progress, skipping"
      );
      return;
    }

    this.warmConnectionReady = false;
    this.warmConnectionOptions = options;
    this.warmSessionId = null;
    this.sampleRate = options.sampleRate || SAMPLE_RATE;

    const url = this.buildWebSocketUrl(options);
    const configuration = this.buildConfiguration(options);
    debugLogger.debug("Corti warming up connection", { environment, tenant });

    return new Promise((resolve, reject) => {
      let settled = false;
      const socket = new WebSocket(url);
      this.warmConnection = socket;

      const warmupTimeout = setTimeout(() => {
        if (settled) return;
        settled = true;
        if (this.warmConnection === socket) this.cleanupWarmConnection();
        reject(new Error("Corti warmup connection timeout"));
      }, WEBSOCKET_TIMEOUT_MS);

      socket.on("open", () => {
        debugLogger.debug("Corti warm connection opened, sending config");
        socket.send(JSON.stringify({ type: "config", configuration }));
      });

      socket.on("message", (data) => {
        let message;
        try {
          message = JSON.parse(data.toString());
        } catch (err) {
          return;
        }
        if (message.type === "CONFIG_ACCEPTED" && !settled) {
          settled = true;
          clearTimeout(warmupTimeout);
          this.warmConnectionReady = true;
          this.warmSessionId = message.sessionId || null;
          this.warmSessionStartedAt = Date.now();
          this.startKeepAlive();
          debugLogger.debug("Corti connection warmed up", { sessionId: this.warmSessionId });
          resolve();
        }
      });

      socket.on("error", (error) => {
        debugLogger.error("Corti warmup connection error", { error: error.message });
        if (this.warmConnection === socket) this.cleanupWarmConnection();
        if (!settled) {
          settled = true;
          clearTimeout(warmupTimeout);
          reject(error);
        }
      });

      socket.on("close", (code, reason) => {
        clearTimeout(warmupTimeout);
        // A dropped socket closes after its replacement may have opened; leave that one be.
        if (this.warmConnection !== socket) {
          if (!settled) {
            settled = true;
            reject(new Error(`Corti warmup connection closed (code: ${code})`));
          }
          return;
        }
        const wasReady = this.warmConnectionReady;
        debugLogger.debug("Corti warm connection closed", {
          code,
          reason: reason?.toString(),
          wasReady,
        });
        this.cleanupWarmConnection();
        if (!settled) {
          settled = true;
          reject(new Error(`Corti warmup connection closed (code: ${code})`));
        }
      });
    });
  }

  hasWarmConnection() {
    return (
      this.warmConnection !== null &&
      this.warmConnectionReady &&
      this.warmConnection.readyState === WebSocket.OPEN
    );
  }

  // Promote the warm socket to the active session — config is already accepted, so
  // audio flows immediately. Returns false (clearing any stale socket) if none usable.
  useWarmConnection() {
    if (!this.hasWarmConnection()) {
      this.cleanupWarmConnection();
      return false;
    }

    this.stopKeepAlive();
    this.ws = this.warmConnection;
    this.isConnected = true;
    this.configAccepted = true;
    this.sessionId = this.warmSessionId || null;
    this.sessionStartedAt = this.warmSessionStartedAt || Date.now();
    this.warmConnection = null;
    this.warmConnectionReady = false;
    this.warmSessionId = null;
    this.warmSessionStartedAt = null;

    this.ws.removeAllListeners("open");
    this.ws.removeAllListeners("message");
    this.ws.removeAllListeners("error");
    this.ws.removeAllListeners("close");
    this.attachSocketHandlers(this.ws);
    return true;
  }

  startKeepAlive() {
    this.stopKeepAlive();
    this.keepAliveInterval = setInterval(() => {
      if (this.warmConnection?.readyState === WebSocket.OPEN) {
        try {
          this.warmConnection.ping();
        } catch (err) {
          debugLogger.debug("Corti keep-alive ping failed", { error: err.message });
          this.cleanupWarmConnection();
        }
      } else {
        this.stopKeepAlive();
      }
    }, KEEPALIVE_INTERVAL_MS);
  }

  stopKeepAlive() {
    if (this.keepAliveInterval) {
      clearInterval(this.keepAliveInterval);
      this.keepAliveInterval = null;
    }
  }

  cleanupWarmConnection() {
    this.stopKeepAlive();
    if (this.warmConnection) {
      try {
        this.warmConnection.close(1000);
      } catch (err) {
        // Ignore close errors
      }
      this.warmConnection = null;
    }
    this.warmConnectionReady = false;
    this.warmConnectionOptions = null;
    this.warmSessionId = null;
    this.warmSessionStartedAt = null;
  }

  handleMessage(data) {
    let message;
    try {
      message = JSON.parse(data.toString());
    } catch (err) {
      debugLogger.error("Corti message parse error", { error: err.message });
      return;
    }

    switch (message.type) {
      case "CONFIG_ACCEPTED":
        this.isConnected = true;
        this.configAccepted = true;
        this.sessionId = message.sessionId || null;
        this.sessionStartedAt = Date.now();
        clearTimeout(this.connectionTimeout);
        this.flushPreConfigBuffer();
        debugLogger.debug("Corti session started", { sessionId: this.sessionId });
        if (this.pendingResolve) {
          this.pendingResolve();
          this.pendingResolve = null;
          this.pendingReject = null;
        }
        break;

      case "transcript": {
        const text = message.data?.text;
        if (!text) break;
        if (message.data.isFinal) {
          const trimmed = text.trim();
          if (!trimmed) break;
          this.completedSegments.push(trimmed);
          this.accumulatedText = this.completedSegments.join(" ");
          const startedAt =
            this.sessionStartedAt != null && typeof message.data.start === "number"
              ? this.sessionStartedAt + message.data.start * 1000
              : Date.now();
          this.onFinalTranscript?.(this.accumulatedText, startedAt);
        } else {
          this.onPartialTranscript?.(text);
        }
        break;
      }

      case "flushed":
      case "ended":
        if (this.pendingAck?.type === message.type) {
          this.resolvePendingAck();
        }
        break;

      case "delta_usage":
      case "usage":
        break;

      case "error":
        debugLogger.error("Corti streaming error", { error: message.error });
        this.onError?.(new Error(message.error?.title || message.error?.details || "Corti error"));
        break;

      default:
        debugLogger.debug("Corti unknown message type", { type: message.type });
    }
  }

  flushPreConfigBuffer() {
    // A stale socket's ack can arrive while this start has no open socket yet.
    if (this.preConfigBuffer.length === 0 || this.ws?.readyState !== WebSocket.OPEN) return;
    debugLogger.debug("Corti flushing pre-config buffer", {
      chunks: this.preConfigBuffer.length,
      bytes: this.preConfigBufferSize,
    });
    for (const frame of this.preConfigBuffer) {
      this.ws.send(frame);
      this.audioBytesSent += frame.length;
    }
    this.preConfigBuffer = [];
    this.preConfigBufferSize = 0;
  }

  sendAudio(pcmBuffer) {
    if (!this.configAccepted) {
      // Corti rejects audio sent before it acks config; cap the startup buffer at ~3s.
      if (!this.bufferingAudio || this.preConfigBufferSize >= 3 * this.sampleRate * 2) {
        return false;
      }
      const copy = Buffer.from(pcmBuffer);
      this.preConfigBuffer.push(copy);
      this.preConfigBufferSize += copy.length;
      return true;
    }
    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
      return false;
    }

    this.audioBytesSent += pcmBuffer.length;
    this.ws.send(pcmBuffer);
    return true;
  }

  finalize() {
    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
      return false;
    }
    this.ws.send(JSON.stringify({ type: "flush" }));
    debugLogger.debug("Corti flush sent");
    return true;
  }

  // Resolves a single in-flight waitForAck (server ack or socket close).
  resolvePendingAck() {
    if (!this.pendingAck) return;
    clearTimeout(this.pendingAck.timer);
    this.pendingAck.resolve();
    this.pendingAck = null;
  }

  waitForAck(type) {
    return new Promise((resolve) => {
      const timer = setTimeout(() => {
        this.pendingAck = null;
        resolve();
      }, TERMINATION_TIMEOUT_MS);
      this.pendingAck = { type, resolve, timer };
    });
  }

  async disconnect(closeStream = true) {
    if (!this.ws) {
      // A stop before the socket exists must not leave the start's held audio behind.
      this.cleanup();
      return { text: this.accumulatedText };
    }

    this.isDisconnecting = true;

    if (closeStream && this.ws.readyState === WebSocket.OPEN) {
      // Corti's close handshake: flush buffered audio, wait for the trailing
      // finals (`flushed`), then signal `end` and close cleanly.
      this.ws.send(JSON.stringify({ type: "flush" }));
      await this.waitForAck("flushed");

      if (this.ws?.readyState === WebSocket.OPEN) {
        this.ws.send(JSON.stringify({ type: "end" }));
      }

      const result = { text: this.accumulatedText };
      this.onSessionEnd?.(result);
      this.cleanup();
      this.isDisconnecting = false;
      return result;
    }

    const result = { text: this.accumulatedText };
    this.cleanup();
    this.isDisconnecting = false;
    return result;
  }

  cleanup() {
    clearTimeout(this.connectionTimeout);
    this.connectionTimeout = null;
    this.preConfigBuffer = [];
    this.preConfigBufferSize = 0;
    this.bufferingAudio = false;

    if (this.ws) {
      try {
        this.ws.close(1000);
      } catch (err) {
        // Ignore close errors
      }
      this.ws = null;
    }

    this.isConnected = false;
    this.configAccepted = false;
    this.sessionId = null;
    this.resolvePendingAck();
  }

  getStatus() {
    return {
      isConnected: this.isConnected,
      sessionId: this.sessionId,
    };
  }
}

module.exports = CortiStreaming;