Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

17.3 KB
Raw Blame
const WebSocket = require("ws");
const debugLogger = require("./debugLogger");
const { computePcm16Rms } = require("../utils/audioUtils");

const SAMPLE_RATE = 16000;
const WEBSOCKET_TIMEOUT_MS = 15000;
// Budget for the final transcript, measured from audioStreamEnd (it lands
// ~500ms after, ~2s at the p95 tail). The renderer's gemini settle ceiling in
// audioManager.js is the same 3s; disconnect only spends what is left of it.
const DISCONNECT_TIMEOUT_MS = 3000;
const KEEPALIVE_INTERVAL_MS = 15000;
const COLD_START_BUFFER_MAX = 3 * SAMPLE_RATE * 2; // 3 seconds of 16-bit PCM
// Google documents a 1000-phrase ceiling but only promises quality up to 100;
// the batch dictionary paths truncate the same array at 100.
const MAX_CUSTOM_VOCABULARY = 100;
// Trailing room noise in dictation recordings measures 0.003-0.010 RMS while
// speech frames sit above ~0.0105, so only a frame at this level is a new
// utterance rather than the tail of a turn the server already closed.
const TURN_REOPEN_RMS = 0.012;
const GEMINI_LIVE_MODEL = "gemini-3.5-transcribe-live";

// In managed mode the model comes from the BYOK-shaped settings, which still
// hold another provider's id, so anything but a Gemini id is ignored.
const resolveLiveModel = (model) =>
  model && model.startsWith("gemini-") ? model : GEMINI_LIVE_MODEL;

const WS_BASE = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage";

// The two Live methods are auth-disjoint: BidiGenerateContent accepts only
// `key`, BidiGenerateContentConstrained only `access_token` (an auth_tokens
// resource name). Crossing them closes the socket with 1007 right after a
// successful HTTP 101, so the credential kind decides the method, not the URL.
function buildGeminiLiveUrl({ mode, token }) {
  if (mode === "byok") {
    return `${WS_BASE}.v1beta.GenerativeService.BidiGenerateContent?key=${token}`;
  }
  return `${WS_BASE}.v1alpha.GenerativeService.BidiGenerateContentConstrained?access_token=${token}`;
}

// Every Live failure arrives as a close frame after the upgrade succeeded (the
// socket's `error` event never carries them), and the server truncates close
// reasons to the 123-byte WebSocket limit — classify on the code alone; the
// reason is only logged by the close handler.
function closeFrameError(code) {
  if (code === 1011) {
    // "Token has been used too many times" / "Token has expired". The managed
    // tokens are single-use, so this is the one retryable auth failure; byok
    // re-handshakes with the same raw key and never sees it.
    return Object.assign(new Error("Gemini Live session token expired"), {
      code: "AUTH_EXPIRED",
    });
  }
  if (code === 1007 || code === 1008) {
    // 1007 is a malformed credential, 1008 one Google no longer knows (revoked
    // or deleted); neither is retryable, both are fixed in Settings.
    return Object.assign(new Error("Invalid Gemini API key. Check your key in Settings."), {
      code: "INVALID_KEY",
    });
  }
  return new Error(`Gemini Live closed before ready (code: ${code})`);
}

class GeminiLiveStreaming {
  constructor() {
    this.ws = null;
    this.isConnected = false;
    this.isConnecting = false;
    this.isDisconnecting = false;
    this.completedSegments = [];
    this.openTurnPartial = "";
    this.onPartialTranscript = null;
    this.onFinalTranscript = null;
    this.onError = null;
    this.onSessionEnd = null;
    this.onConnectionLost = null;
    this.pendingResolve = null;
    this.pendingReject = null;
    this.connectionTimeout = null;
    this.keepAliveInterval = null;
    this.coldStartBuffer = [];
    this.coldStartBufferSize = 0;
    this.bufferingAudio = false;
    this.audioBytesSent = 0;
    this.currentModel = GEMINI_LIVE_MODEL;
    this._connectionLossNotified = false;
    this._audioStreamEndSentAt = null;
    this._turnEnded = false;
    this._turnEndResolve = null;
  }

  // Starts buffering audio before the socket exists, covering the token fetch
  // and the 350ms-1.4s handshake so sendAudio() doesn't drop the first words.
  beginConnecting() {
    this.bufferingAudio = true;
    this.coldStartBuffer = [];
    this.coldStartBufferSize = 0;
  }

  // A turn the server never finalized (it can go silent after audioStreamEnd)
  // exists only as its partial, so the transcript keeps it rather than losing
  // everything since the last pause.
  getFullTranscript() {
    return [...this.completedSegments, this._openTurnText()].filter(Boolean).join(" ");
  }

  // The partial sometimes restates the finalized turns first, run together
  // without spaces ("...October.Before then"); only what follows is new.
  _openTurnText() {
    let text = this.openTurnPartial;
    for (const segment of this.completedSegments) {
      const rest = text.trimStart();
      if (!rest.startsWith(segment)) break;
      text = rest.slice(segment.length);
    }
    return text.trim();
  }

  // Test seam: overridden to point at a loopback server.
  buildWebSocketUrl(options) {
    return buildGeminiLiveUrl(options);
  }

  buildSetupMessage({ model, language, keyterms } = {}) {
    const customVocabulary = (keyterms || [])
      .map((term) => String(term).trim())
      .filter(Boolean)
      .slice(0, MAX_CUSTOM_VOCABULARY);
    const inputAudioTranscription = {};
    // `languageCodes` on its own costs punctuation and sentence casing, and an
    // invalid locale is accepted silently — only ever send a language the user
    // picked explicitly ("auto" is stripped upstream).
    if (language) inputAudioTranscription.languageCodes = [language];
    if (customVocabulary.length) inputAudioTranscription.customVocabulary = customVocabulary;

    return {
      setup: {
        model: `models/${resolveLiveModel(model)}`,
        generationConfig: { responseModalities: ["TEXT"] },
        inputAudioTranscription,
      },
    };
  }

  async connect(options = {}) {
    const { token, mode, refreshToken } = options;
    if (!token) {
      throw Object.assign(new Error("Gemini API key not configured. Add your key in Settings."), {
        code: "API_KEY_MISSING",
      });
    }

    if (this.isConnected || this.isConnecting) {
      debugLogger.debug("Gemini Live already connected/connecting");
      return;
    }

    if (!this.bufferingAudio) this.beginConnecting();
    this.completedSegments = [];
    this.openTurnPartial = "";
    this.audioBytesSent = 0;
    this.currentModel = resolveLiveModel(options.model);
    this._connectionLossNotified = false;
    this._audioStreamEndSentAt = null;
    this._turnEnded = false;

    try {
      await this._openSocket(options);
    } catch (err) {
      if (err.code !== "AUTH_EXPIRED" || mode === "byok" || !refreshToken) throw err;
      debugLogger.debug("Gemini Live re-minting single-use token after 1011 close");
      this.bufferingAudio = true;
      await this._openSocket({ ...options, token: await refreshToken() });
    }
  }

  _openSocket(options) {
    const url = this.buildWebSocketUrl(options);
    debugLogger.debug("Gemini Live connecting", {
      mode: options.mode === "byok" ? "byok" : "openwhispr",
      model: this.currentModel,
    });

    return new Promise((resolve, reject) => {
      this.pendingResolve = resolve;
      this.pendingReject = reject;

      this.isConnecting = true;
      this.connectionTimeout = setTimeout(() => {
        this.cleanup();
        reject(new Error("Gemini Live connection timeout"));
      }, WEBSOCKET_TIMEOUT_MS);

      this.ws = new WebSocket(url);

      this.ws.on("open", () => {
        // Audio sent before setupComplete is discarded by the server, so the
        // setup message is the only thing that goes out on open.
        this.ws.send(JSON.stringify(this.buildSetupMessage(options)));
      });

      this.ws.on("message", (data) => {
        this.handleMessage(data);
      });

      this.ws.on("error", (error) => {
        const wasActive = this.isConnected;
        debugLogger.error("Gemini Live WebSocket error", { error: error.message });
        this.cleanup();
        this._rejectPending(error);
        if (wasActive && !this.isDisconnecting) {
          this._notifyConnectionLost(error);
        } else if (!this.isDisconnecting) {
          this.onError?.(error);
        }
      });

      this.ws.on("close", (code, reason) => {
        const wasActive = this.isConnected;
        debugLogger.debug("Gemini Live WebSocket closed", {
          code,
          reason: reason?.toString() || "",
          wasActive,
        });
        this._resolveTurnEnd();
        this.cleanup();
        if (!wasActive) {
          this._rejectPending(closeFrameError(code));
        } else if (!this.isDisconnecting) {
          // A session killed mid-dictation (the ~10 minute cap, a dead network)
          // still hands over what it transcribed; there is no resumption handle
          // to reconnect with, so the caller finalizes with this text.
          this.onSessionEnd?.({ text: this.getFullTranscript() });
          this._notifyConnectionLost(new Error(`Connection lost (code: ${code})`));
        }
      });
    });
  }

  // The complete server vocabulary is `setupComplete` plus
  // `serverContent.{speechState, interimInputTranscription, inputTranscription,
  // generationComplete}`. There is no `turnComplete`, no `usageMetadata` and no
  // `sessionResumptionUpdate` on this model, so end-of-turn is the final
  // transcript arriving with `generationComplete` in the same millisecond.
  handleMessage(data) {
    try {
      const message = JSON.parse(data.toString());

      if (message.setupComplete) this._markConnected();

      const serverContent = message.serverContent;
      if (!serverContent) return;

      // Partials are revised, not appended to ("The quick brown" becomes
      // "the quick brown fox"), so consumers must replace the whole string.
      const partial = serverContent.interimInputTranscription?.text;
      if (partial) {
        this._turnEnded = false;
        this.openTurnPartial = partial;
        this.onPartialTranscript?.(partial);
      }

      const final = serverContent.inputTranscription?.text?.trim();
      if (final) {
        this.completedSegments.push(final);
        this.openTurnPartial = "";
        const fullText = this.getFullTranscript();
        this.onFinalTranscript?.(fullText, Date.now());
        debugLogger.debug("Gemini Live turn completed", {
          turnText: final.slice(0, 100),
          totalLength: fullText.length,
          segments: this.completedSegments.length,
        });
      }

      // The server closes a turn on trailing silence, often before the hotkey is
      // released; that generationComplete already answers the final. Speech
      // (a loud frame or a new partial) reopens a turn, so a later stop waits again.
      if (serverContent.generationComplete) this._resolveTurnEnd();
    } catch (err) {
      debugLogger.error("Gemini Live message parse error", { error: err.message });
    }
  }

  _markConnected() {
    if (this.isConnected) return;
    this.isConnected = true;
    this.isConnecting = false;
    clearTimeout(this.connectionTimeout);
    this.connectionTimeout = null;
    this.startKeepAlive();
    debugLogger.debug("Gemini Live setup complete", { model: this.currentModel });
    // A dictation shorter than the handshake leaves every frame in the buffer,
    // and sendAudio never runs again to drain it.
    this._flushColdStartBuffer();
    if (this.pendingResolve) {
      this.pendingResolve();
      this.pendingResolve = null;
      this.pendingReject = null;
    }
  }

  _rejectPending(error) {
    if (!this.pendingReject) return;
    this.pendingReject(error);
    this.pendingReject = null;
    this.pendingResolve = null;
  }

  _notifyConnectionLost(error) {
    if (this._connectionLossNotified) return;
    this._connectionLossNotified = true;
    if (this.onConnectionLost) {
      this.onConnectionLost(error);
    } else {
      this.onError?.(error);
    }
  }

  _resolveTurnEnd() {
    this._turnEnded = true;
    this._turnEndResolve?.();
    this._turnEndResolve = null;
  }

  // A warm socket sits idle between dictations with no other liveness check; a
  // network path that dies silently leaves isConnected stuck true and the next
  // recording gets sent into a dead socket.
  startKeepAlive() {
    this.stopKeepAlive();
    const socket = this.ws;
    if (!socket) return;

    socket.isAlive = true;
    socket.on("pong", () => {
      socket.isAlive = true;
    });

    this.keepAliveInterval = setInterval(() => {
      if (socket !== this.ws || socket.readyState !== WebSocket.OPEN) {
        this.stopKeepAlive();
        return;
      }
      if (socket.isAlive === false) {
        debugLogger.debug("Gemini Live keep-alive missed pong, terminating stale connection");
        socket.terminate();
        return;
      }
      socket.isAlive = false;
      try {
        socket.ping();
      } catch (err) {
        debugLogger.debug("Gemini Live keep-alive ping failed", { error: err.message });
        socket.terminate();
      }
    }, KEEPALIVE_INTERVAL_MS);
  }

  stopKeepAlive() {
    if (this.keepAliveInterval) {
      clearInterval(this.keepAliveInterval);
      this.keepAliveInterval = null;
    }
  }

  _sendAudioFrame(pcmBuffer) {
    this.ws.send(
      JSON.stringify({
        realtimeInput: {
          audio: { data: pcmBuffer.toString("base64"), mimeType: `audio/pcm;rate=${SAMPLE_RATE}` },
        },
      })
    );
    this.audioBytesSent += pcmBuffer.length;
    if (this._turnEnded && computePcm16Rms(pcmBuffer) >= TURN_REOPEN_RMS) this._turnEnded = false;
  }

  sendAudio(pcmBuffer) {
    // The capture worklet ends its stream with a "flushed" string sentinel;
    // only s16 PCM (even byte length) is audio, and only audioStreamEnd closes
    // a Gemini turn.
    if (!pcmBuffer || pcmBuffer.length % 2 !== 0) return false;

    if (this.ws?.readyState !== WebSocket.OPEN || !this.isConnected) {
      if (this.bufferingAudio && this.coldStartBufferSize < COLD_START_BUFFER_MAX) {
        const copy = Buffer.from(pcmBuffer);
        this.coldStartBuffer.push(copy);
        this.coldStartBufferSize += copy.length;
      }
      return false;
    }

    this._flushColdStartBuffer();
    this._sendAudioFrame(pcmBuffer);
    return true;
  }

  _flushColdStartBuffer() {
    if (this.coldStartBuffer.length === 0) return;
    debugLogger.debug("Gemini Live flushing cold-start buffer", {
      chunks: this.coldStartBuffer.length,
      bytes: this.coldStartBufferSize,
    });
    for (const buffered of this.coldStartBuffer) {
      this._sendAudioFrame(buffered);
    }
    this.coldStartBuffer = [];
    this.coldStartBufferSize = 0;
  }

  // Gemini has no mid-stream finalize: audioStreamEnd ends the turn, and the
  // server answers with the final transcript. Sending it from here (the stop
  // sequence calls finalize() before it waits for text) buys back the wait.
  finalize() {
    if (this.ws?.readyState !== WebSocket.OPEN || this._audioStreamEndSentAt) return false;
    this._audioStreamEndSentAt = Date.now();
    this.ws.send(JSON.stringify({ realtimeInput: { audioStreamEnd: true } }));
    debugLogger.debug("Gemini Live audioStreamEnd sent", { audioBytesSent: this.audioBytesSent });
    return true;
  }

  async disconnect(closeStream = true) {
    debugLogger.debug("Gemini Live disconnect", {
      audioBytesSent: this.audioBytesSent,
      segments: this.completedSegments.length,
      textLength: this.getFullTranscript().length,
      readyState: this.ws?.readyState,
    });

    if (!this.ws) return this._takeTranscript();

    this.isDisconnecting = true;

    if (closeStream && this.ws.readyState === WebSocket.OPEN && this.isConnected) {
      this._flushColdStartBuffer();
    }

    if (closeStream && this.ws.readyState === WebSocket.OPEN && this.audioBytesSent > 0) {
      this.finalize();
      if (!this._turnEnded) await this._awaitTurnEnd();
    }

    const result = this._takeTranscript();
    this.cleanup();
    this.isDisconnecting = false;
    return result;
  }

  // The stop sequence finalizes first and waits on the renderer side before it
  // gets here, so the budget runs from audioStreamEnd, not from this call.
  async _awaitTurnEnd() {
    const remaining = DISCONNECT_TIMEOUT_MS - (Date.now() - this._audioStreamEndSentAt);
    if (remaining <= 0) return;
    let timeoutId;
    await Promise.race([
      new Promise((resolve) => {
        this._turnEndResolve = resolve;
      }),
      new Promise((resolve) => {
        timeoutId = setTimeout(() => {
          debugLogger.debug("Gemini Live final transcript timeout, using accumulated text");
          resolve();
        }, remaining);
      }),
    ]);
    clearTimeout(timeoutId);
    this._turnEndResolve = null;
  }

  _takeTranscript() {
    const openTurnLength = this._openTurnText().length;
    if (openTurnLength) {
      debugLogger.debug("Gemini Live last turn was never finalized", { openTurnLength });
    }
    const result = { text: this.getFullTranscript() };
    this.completedSegments = [];
    this.openTurnPartial = "";
    return result;
  }

  cleanup() {
    clearTimeout(this.connectionTimeout);
    this.connectionTimeout = null;
    this.stopKeepAlive();

    if (this.ws) {
      try {
        this.ws.close();
      } catch {}
      this.ws = null;
    }

    this.isConnected = false;
    this.isConnecting = false;
    this.bufferingAudio = false;
  }

  getStatus() {
    return { isConnected: this.isConnected, isConnecting: this.isConnecting };
  }
}

module.exports = { GeminiLiveStreaming, buildGeminiLiveUrl, GEMINI_LIVE_MODEL };