Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

24.4 KB
Raw Blame
const WebSocket = require("ws");
const debugLogger = require("./debugLogger");

const SAMPLE_RATE = 16000;
const WEBSOCKET_TIMEOUT_MS = 30000;
const TERMINATION_TIMEOUT_MS = 5000;
const TOKEN_REFRESH_BUFFER_MS = 30000;
const TOKEN_EXPIRY_MS = 300000;
const REWARM_DELAY_MS = 2000;
const MAX_REWARM_ATTEMPTS = 10;
const KEEPALIVE_INTERVAL_MS = 15000;
const COLD_START_BUFFER_MAX = 3 * SAMPLE_RATE * 2; // 3 seconds of 16-bit PCM
const MIN_FRAME_MS = 50;
// AssemblyAI hard-closes the session outside 50-1000 ms but documents 50-250 ms
// as the supported shape, so the split targets the recommended ceiling: 1000 would
// put the first slice of a coalesced read exactly on the limit it exists to avoid.
const MAX_FRAME_MS = 250;
// Both bounds follow the rate the socket opened at: Note Recording streams at
// 24 kHz, where a frame sized against this module's 16 kHz default lasts only
// 33 ms (#2140).
const minFrameBytes = (sampleRate) => Math.ceil((sampleRate * 2 * MIN_FRAME_MS) / 1000);
const maxFrameBytes = (sampleRate) => Math.floor((sampleRate * 2 * MAX_FRAME_MS) / 1000);

class AssemblyAiStreaming {
  constructor() {
    this.ws = null;
    this.sessionId = null;
    this.isConnected = false;
    this.onPartialTranscript = null;
    this.onFinalTranscript = null;
    this.onError = null;
    this.onSessionEnd = null;
    this.onConnectionLost = null;
    this.connectionLossNotified = false;
    this.pendingResolve = null;
    this.pendingReject = null;
    this.connectionTimeout = null;
    this.accumulatedText = "";
    this.lastTurnText = "";
    this.turns = [];
    this.terminationResolve = null;
    this.cachedToken = null;
    this.tokenFetchedAt = null;
    this.mode = null;
    this.warmConnection = null;
    this.warmConnectionReady = false;
    this.warmConnectionOptions = null;
    this.warmSessionId = null;
    this.requestedModel = null;
    this.sessionSampleRate = SAMPLE_RATE;
    this.rewarmAttempts = 0;
    this.rewarmTimer = null;
    this.keepAliveInterval = null;
    this.isDisconnecting = false;
    this.pendingAudio = [];
    this.pendingAudioBytes = 0;
    this.bufferingAudio = false;
    this.completedSegments = [];
    this.speechStartedAt = null;
  }

  buildWebSocketUrl(options) {
    const sampleRate = options.sampleRate || SAMPLE_RATE;
    this.sessionSampleRate = sampleRate;
    const params = new URLSearchParams({
      sample_rate: String(sampleRate),
      encoding: "pcm_s16le",
      format_turns: "true",
      token: options.token,
    });
    this.requestedModel = options.model || null;
    if (options.model) {
      params.set("speech_model", options.model);
    }
    if (options.minTurnSilence != null) {
      params.set("min_turn_silence", String(options.minTurnSilence));
    }
    if (options.maxTurnSilence != null) {
      params.set("max_turn_silence", String(options.maxTurnSilence));
    }
    if (options.keyterms && options.keyterms.length > 0) {
      params.set("keyterms_prompt", JSON.stringify(options.keyterms.slice(0, 100)));
    }
    return `wss://streaming.assemblyai.com/v3/ws?${params.toString()}`;
  }

  cacheToken(token) {
    this.cachedToken = token;
    this.tokenFetchedAt = Date.now();
    debugLogger.debug("AssemblyAI token cached", { expiresIn: TOKEN_EXPIRY_MS });
  }

  // BYOK and managed dictation share one client instance, so a token or warm
  // socket minted under one credential kind must never serve the other. Callers
  // that read the cache before connecting must adopt the mode first.
  adoptMode(options) {
    const mode = options.mode === "byok" ? "byok" : "openwhispr";
    if (this.mode !== null && this.mode !== mode) {
      debugLogger.debug("AssemblyAI credential mode changed, dropping cached session state", {
        from: this.mode,
        to: mode,
      });
      this.cachedToken = null;
      this.tokenFetchedAt = null;
      this.cleanupWarmConnection();
    }
    this.mode = mode;
  }

  isTokenValid() {
    if (!this.cachedToken || !this.tokenFetchedAt) return false;
    const age = Date.now() - this.tokenFetchedAt;
    return age < TOKEN_EXPIRY_MS - TOKEN_REFRESH_BUFFER_MS;
  }

  getCachedToken() {
    return this.isTokenValid() ? this.cachedToken : null;
  }

  startKeepAlive() {
    this.stopKeepAlive();
    this.keepAliveInterval = setInterval(() => {
      if (this.warmConnection && this.warmConnection.readyState === WebSocket.OPEN) {
        try {
          this.warmConnection.ping();
        } catch (err) {
          debugLogger.debug("AssemblyAI keep-alive ping failed", { error: err.message });
          this.cleanupWarmConnection();
        }
      } else {
        this.stopKeepAlive();
      }
    }, KEEPALIVE_INTERVAL_MS);
  }

  stopKeepAlive() {
    if (this.keepAliveInterval) {
      clearInterval(this.keepAliveInterval);
      this.keepAliveInterval = null;
    }
  }

  async warmup(options = {}) {
    const { token } = options;
    if (!token) {
      throw new Error("Streaming token is required for warmup");
    }

    this.adoptMode(options);
    if (this.warmConnection) {
      debugLogger.debug(
        this.warmConnectionReady
          ? "AssemblyAI connection already warm"
          : "AssemblyAI warmup already in progress, skipping"
      );
      return;
    }

    this.warmConnectionReady = false;
    this.warmSessionId = null;
    this.cachedToken = token;
    this.tokenFetchedAt = Date.now();
    this.warmConnectionOptions = options;
    this.rewarmAttempts = 0;

    const url = this.buildWebSocketUrl(options);
    debugLogger.debug("AssemblyAI warming up connection");

    return new Promise((resolve, reject) => {
      let settled = false;
      const socket = new WebSocket(url);
      this.warmConnection = socket;

      const warmupTimeout = setTimeout(() => {
        if (settled) return;
        settled = true;
        if (this.warmConnection === socket) this.cleanupWarmConnection();
        reject(new Error("AssemblyAI warmup connection timeout"));
      }, WEBSOCKET_TIMEOUT_MS);

      socket.on("open", () => {
        debugLogger.debug("AssemblyAI warm connection socket opened");
      });

      socket.on("message", (data) => {
        try {
          const message = JSON.parse(data.toString());
          if (message.type === "Begin" && !settled) {
            settled = true;
            clearTimeout(warmupTimeout);
            this.warmConnectionReady = true;
            this.warmSessionId = message.id || null;
            this.startKeepAlive();
            debugLogger.debug("AssemblyAI connection warmed up", { sessionId: message.id });
            resolve();
          }
        } catch (err) {
          debugLogger.error("AssemblyAI warmup message parse error", { error: err.message });
        }
      });

      socket.on("error", (error) => {
        clearTimeout(warmupTimeout);
        debugLogger.error("AssemblyAI warmup connection error", { error: error.message });
        if (this.warmConnection === socket) this.cleanupWarmConnection();
        if (!settled) {
          settled = true;
          reject(error);
        }
      });

      socket.on("close", (code, reason) => {
        clearTimeout(warmupTimeout);
        // A dropped socket closes after its replacement may have opened; leave that one be.
        if (this.warmConnection !== socket) {
          if (!settled) {
            settled = true;
            reject(new Error(`AssemblyAI warmup connection closed before ready (code: ${code})`));
          }
          return;
        }
        this.stopKeepAlive();
        const wasReady = this.warmConnectionReady;
        const savedOptions = this.warmConnectionOptions ? { ...this.warmConnectionOptions } : null;
        debugLogger.debug("AssemblyAI warm connection closed", {
          wasReady,
          code,
          reason: reason?.toString(),
        });
        this.cleanupWarmConnection();
        if (!settled) {
          settled = true;
          reject(new Error(`AssemblyAI warmup connection closed before ready (code: ${code})`));
          return;
        }
        if (wasReady && savedOptions) {
          this.warmConnectionOptions = savedOptions;
          this.scheduleRewarm();
        }
      });
    });
  }

  scheduleRewarm() {
    if (this.rewarmAttempts >= MAX_REWARM_ATTEMPTS) {
      debugLogger.debug("AssemblyAI max re-warm attempts reached, will cold-start next recording");
      return;
    }
    if (this.isConnected) {
      // Active session in progress, don't re-warm
      return;
    }
    const token = this.getCachedToken();
    if (!token || !this.warmConnectionOptions) {
      debugLogger.debug("AssemblyAI cannot re-warm: no valid token or options");
      return;
    }

    this.rewarmAttempts++;
    const delay = Math.min(REWARM_DELAY_MS * Math.pow(2, this.rewarmAttempts - 1), 60000);
    debugLogger.debug("AssemblyAI scheduling re-warm", {
      attempt: this.rewarmAttempts,
      delayMs: delay,
    });
    clearTimeout(this.rewarmTimer);
    this.rewarmTimer = setTimeout(() => {
      this.rewarmTimer = null;
      if (this.hasWarmConnection() || this.isConnected) return;
      this.warmup({ ...this.warmConnectionOptions, token }).catch((err) => {
        debugLogger.debug("AssemblyAI auto re-warm failed", { error: err.message });
      });
    }, delay);
  }

  useWarmConnection() {
    if (!this.warmConnection || !this.warmConnectionReady) {
      return false;
    }

    if (this.warmConnection.readyState !== WebSocket.OPEN) {
      debugLogger.debug("AssemblyAI warm connection readyState not OPEN, discarding", {
        readyState: this.warmConnection.readyState,
      });
      this.cleanupWarmConnection();
      return false;
    }

    this.stopKeepAlive();

    this.ws = this.warmConnection;
    this.isConnected = true;
    this.connectionLossNotified = false;
    this.sessionId = this.warmSessionId || null;
    this.warmConnection = null;
    this.warmConnectionReady = false;
    this.warmSessionId = null;

    this.ws.removeAllListeners("message");
    this.ws.on("message", (data) => {
      this.handleMessage(data);
    });

    this.ws.removeAllListeners("error");
    this.ws.on("error", (error) => {
      const wasActive = this.isConnected;
      debugLogger.error("AssemblyAI WebSocket error", { error: error.message });
      this.cleanup();
      if (wasActive && !this.isDisconnecting) this.notifyConnectionLost(error);
    });

    this.ws.removeAllListeners("close");
    this.ws.on("close", (code, reason) => {
      const wasActive = this.isConnected;
      debugLogger.debug("AssemblyAI WebSocket closed", {
        code,
        reason: reason?.toString(),
        wasActive,
      });
      this.cleanup();
      if (wasActive && !this.isDisconnecting) {
        this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));
      }
    });

    debugLogger.debug("AssemblyAI using pre-warmed connection");
    return true;
  }

  cleanupWarmConnection() {
    this.stopKeepAlive();
    if (this.warmConnection) {
      try {
        this.warmConnection.close();
      } catch (err) {
        // Ignore
      }
      this.warmConnection = null;
    }
    this.warmConnectionReady = false;
    this.warmConnectionOptions = null;
    this.warmSessionId = null;
  }

  hasWarmConnection() {
    return (
      this.warmConnection !== null &&
      this.warmConnectionReady &&
      this.warmConnection.readyState === WebSocket.OPEN
    );
  }

  // Starts holding audio before the socket exists, covering the token fetch and
  // the handshake so sendAudio() doesn't drop the first words.
  beginConnecting() {
    this.bufferingAudio = true;
    this.pendingAudio = [];
    this.pendingAudioBytes = 0;
  }

  async connect(options = {}) {
    const { token } = options;
    if (!token) {
      throw new Error("Streaming token is required");
    }

    if (this.isConnected) {
      debugLogger.debug("AssemblyAI streaming already connected");
      return;
    }

    // Reset accumulated text for new session
    this.accumulatedText = "";
    this.lastTurnText = "";
    this.turns = [];
    this.connectionLossNotified = false;
    // The caller may already be holding audio from before its token fetch.
    if (!this.bufferingAudio) this.beginConnecting();

    this.adoptMode(options);
    // The server pins speech_model at Begin, so a warm socket opened for another
    // model would keep that model for the whole session — and the Begin mismatch
    // warning could never fire for the model actually requested.
    if (
      this.hasWarmConnection() &&
      ((this.warmConnectionOptions.model || null) !== (options.model || null) ||
        (this.warmConnectionOptions.sampleRate || SAMPLE_RATE) !==
          (options.sampleRate || SAMPLE_RATE))
    ) {
      debugLogger.debug("AssemblyAI warm connection differs, cold-starting", {
        warm: this.warmConnectionOptions.model || null,
        requested: options.model || null,
        warmSampleRate: this.warmConnectionOptions.sampleRate || SAMPLE_RATE,
        requestedSampleRate: options.sampleRate || SAMPLE_RATE,
      });
      this.cleanupWarmConnection();
    }

    // Try to use pre-warmed connection for instant start
    if (this.hasWarmConnection()) {
      if (this.useWarmConnection()) {
        debugLogger.debug("AssemblyAI using warm connection - instant start");
        this.sendPendingAudio();
        return;
      }
    }

    const url = this.buildWebSocketUrl(options);
    debugLogger.debug("AssemblyAI streaming connecting (cold start)");

    return new Promise((resolve, reject) => {
      this.pendingResolve = resolve;
      this.pendingReject = reject;

      this.connectionTimeout = setTimeout(() => {
        this.cleanup();
        reject(new Error("AssemblyAI WebSocket connection timeout"));
      }, WEBSOCKET_TIMEOUT_MS);

      this.ws = new WebSocket(url);

      this.ws.on("open", () => {
        debugLogger.debug("AssemblyAI WebSocket connected");
      });

      this.ws.on("message", (data) => {
        this.handleMessage(data);
      });

      this.ws.on("error", (error) => {
        const wasActive = this.isConnected;
        debugLogger.error("AssemblyAI WebSocket error", { error: error.message });
        this.cleanup();
        if (this.pendingReject) {
          this.pendingReject(error);
          this.pendingReject = null;
          this.pendingResolve = null;
        }
        if (wasActive && !this.isDisconnecting) {
          this.notifyConnectionLost(error);
        } else if (!this.isDisconnecting) {
          this.onError?.(error);
        }
      });

      this.ws.on("close", (code, reason) => {
        const wasActive = this.isConnected;
        debugLogger.debug("AssemblyAI WebSocket closed", {
          code,
          reason: reason?.toString(),
          wasActive,
        });
        if (this.pendingReject) {
          this.pendingReject(new Error(`AssemblyAI WebSocket closed before ready (code: ${code})`));
          this.pendingReject = null;
          this.pendingResolve = null;
        }
        this.cleanup();
        if (wasActive && !this.isDisconnecting) {
          this.notifyConnectionLost(new Error(`Connection lost (code: ${code})`));
        }
      });
    });
  }

  notifyConnectionLost(error) {
    if (this.connectionLossNotified) return;
    this.connectionLossNotified = true;
    if (this.onConnectionLost) {
      this.onConnectionLost(error);
    } else {
      this.onError?.(error);
    }
  }

  handleMessage(data) {
    try {
      const message = JSON.parse(data.toString());

      switch (message.type) {
        case "Begin":
          this.sessionId = message.id;
          this.isConnected = true;
          clearTimeout(this.connectionTimeout);
          debugLogger.debug("AssemblyAI session started", { sessionId: this.sessionId });
          // AssemblyAI ignores unrecognized query params instead of rejecting them,
          // so a bad speech_model silently downgrades the session.
          if (
            message.configuration?.model &&
            this.requestedModel &&
            message.configuration.model !== this.requestedModel
          ) {
            debugLogger.warn(
              "AssemblyAI applied a different speech model than requested",
              { requested: this.requestedModel, applied: message.configuration.model },
              "transcription"
            );
          }
          // Audio held through the handshake goes out now: a short dictation may
          // send no further frame to carry it.
          this.sendPendingAudio();
          if (this.pendingResolve) {
            this.pendingResolve();
            this.pendingResolve = null;
            this.pendingReject = null;
          }
          break;

        case "Turn":
          if (message.transcript) {
            if (message.end_of_turn) {
              // Turn has ended - append once, then replace with formatted variant if needed
              const trimmedTranscript = message.transcript.trim();
              const normalizedTranscript = this.normalizeTurnText(trimmedTranscript);
              const previousTurn = this.turns[this.turns.length - 1];

              if (!trimmedTranscript || !normalizedTranscript) {
                break;
              }

              if (previousTurn && previousTurn.normalized === normalizedTranscript) {
                // AssemblyAI can emit the same turn twice (raw then formatted). Replace previous
                // turn only when this variant is formatted, otherwise ignore duplicate.
                if (message.turn_is_formatted && previousTurn.text !== trimmedTranscript) {
                  previousTurn.text = trimmedTranscript;
                  this.completedSegments[this.completedSegments.length - 1] = trimmedTranscript;
                  this.lastTurnText = trimmedTranscript;
                  this.accumulatedText = this.turns.map((turn) => turn.text).join(" ");
                  this.onFinalTranscript?.(this.accumulatedText, previousTurn.startedAt);
                  debugLogger.debug("AssemblyAI formatted turn update applied", {
                    text: trimmedTranscript.slice(0, 100),
                    totalAccumulated: this.accumulatedText.length,
                  });
                } else {
                  debugLogger.debug("AssemblyAI duplicate turn ignored", {
                    text: trimmedTranscript.slice(0, 100),
                  });
                }
                break;
              }

              const speechTimestamp = this.speechStartedAt || Date.now();
              this.speechStartedAt = null;
              this.turns.push({
                text: trimmedTranscript,
                normalized: normalizedTranscript,
                startedAt: speechTimestamp,
              });
              this.completedSegments.push(trimmedTranscript);
              this.lastTurnText = trimmedTranscript;
              this.accumulatedText = this.turns.map((turn) => turn.text).join(" ");
              this.onFinalTranscript?.(this.accumulatedText, speechTimestamp);
              debugLogger.debug("AssemblyAI final transcript (end_of_turn)", {
                text: message.transcript.slice(0, 100),
                totalAccumulated: this.accumulatedText.length,
              });
            } else if (message.turn_is_formatted) {
              // Formatted but turn not ended yet - show as preview without accumulating
              this.onPartialTranscript?.(message.transcript);
            } else {
              // Partial transcript - show real-time updates (current turn only)
              this.onPartialTranscript?.(message.transcript);
            }
          }
          break;

        case "Termination":
          debugLogger.debug("AssemblyAI session terminated", {
            audioDuration: message.audio_duration_seconds,
          });
          // Resolve any pending termination wait
          if (this.terminationResolve) {
            this.terminationResolve({
              audioDuration: message.audio_duration_seconds,
              text: this.accumulatedText,
            });
            this.terminationResolve = null;
          }
          this.onSessionEnd?.({
            audioDuration: message.audio_duration_seconds,
            text: this.accumulatedText,
          });
          this.cleanup();
          break;

        case "Error":
          debugLogger.error("AssemblyAI streaming error", { error: message.error });
          this.onError?.(new Error(message.error));
          break;

        case "SpeechStarted":
          this.speechStartedAt = Date.now();
          break;

        default:
          debugLogger.debug("AssemblyAI unknown message type", { type: message.type });
      }
    } catch (err) {
      debugLogger.error("AssemblyAI message parse error", { error: err.message });
    }
  }

  normalizeTurnText(text) {
    return text
      .toLowerCase()
      .replace(/[^\p{L}\p{N}\s]/gu, " ")
      .replace(/\s+/g, " ")
      .trim();
  }

  sendAudio(pcmBuffer) {
    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
      // Held only while a start is in flight (up to three seconds), then framed like live audio.
      if (
        !this.bufferingAudio ||
        this.isConnected ||
        this.pendingAudioBytes >= COLD_START_BUFFER_MAX
      ) {
        return false;
      }
      this.pendingAudio.push(Buffer.from(pcmBuffer));
      this.pendingAudioBytes += pcmBuffer.length;
      return true;
    }

    this.pendingAudio.push(pcmBuffer);
    this.pendingAudioBytes += pcmBuffer.length;
    this.sendPendingAudio();
    return true;
  }

  sendPendingAudio() {
    const minBytes = minFrameBytes(this.sessionSampleRate);
    if (this.pendingAudioBytes < minBytes) {
      return;
    }

    const frame = Buffer.concat(this.pendingAudio, this.pendingAudioBytes);
    this.pendingAudio = [];
    this.pendingAudioBytes = 0;

    // The native system-audio helpers forward raw stdout chunks, so one read can
    // far exceed the nominal 100 ms when a stalled main process lets the pipe
    // coalesce. Carry a sub-floor remainder forward rather than sending it short.
    const maxBytes = maxFrameBytes(this.sessionSampleRate);
    let offset = 0;
    while (frame.length - offset >= minBytes) {
      const end = Math.min(offset + maxBytes, frame.length);
      this.ws.send(frame.subarray(offset, end));
      offset = end;
    }
    if (offset < frame.length) {
      const remainder = frame.subarray(offset);
      this.pendingAudio.push(remainder);
      this.pendingAudioBytes = remainder.length;
    }
  }

  forceEndpoint() {
    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
      return false;
    }

    this.ws.send(JSON.stringify({ type: "ForceEndpoint" }));
    debugLogger.debug("AssemblyAI ForceEndpoint sent");
    return true;
  }

  async disconnect(terminate = true) {
    if (!this.ws) return { text: this.accumulatedText };

    this.isDisconnecting = true;

    if (terminate && this.ws.readyState === WebSocket.OPEN) {
      try {
        this.ws.send(JSON.stringify({ type: "Terminate" }));

        let timeoutId;
        const result = await Promise.race([
          new Promise((resolve) => {
            this.terminationResolve = resolve;
          }),
          new Promise((resolve) => {
            timeoutId = setTimeout(() => {
              debugLogger.debug("AssemblyAI termination timeout, using accumulated text");
              resolve({ text: this.accumulatedText });
            }, TERMINATION_TIMEOUT_MS);
          }),
        ]);
        clearTimeout(timeoutId);

        this.terminationResolve = null;
        this.cleanup();
        this.isDisconnecting = false;
        return result;
      } catch (err) {
        debugLogger.debug("AssemblyAI terminate send failed", { error: err.message });
      }
    }

    const result = { text: this.accumulatedText };
    this.cleanup();
    this.isDisconnecting = false;
    return result;
  }

  cleanup() {
    clearTimeout(this.connectionTimeout);
    this.connectionTimeout = null;

    this.pendingAudio = [];
    this.pendingAudioBytes = 0;
    this.bufferingAudio = false;
    this.completedSegments = [];
    this.speechStartedAt = null;

    if (this.ws) {
      try {
        this.ws.close();
      } catch (err) {
        // Ignore close errors
      }
      this.ws = null;
    }

    this.isConnected = false;
    this.sessionId = null;
    this.terminationResolve = null;
  }

  cleanupAll() {
    this.cleanup();
    this.cleanupWarmConnection();
    clearTimeout(this.rewarmTimer);
    this.rewarmTimer = null;
    this.cachedToken = null;
    this.tokenFetchedAt = null;
    this.warmConnectionOptions = null;
    this.turns = [];
  }

  getStatus() {
    return {
      isConnected: this.isConnected,
      sessionId: this.sessionId,
      hasWarmConnection: this.hasWarmConnection(),
      hasValidToken: this.isTokenValid(),
    };
  }
}

module.exports = AssemblyAiStreaming;