Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

10.0 KB
Raw Blame
import { decodePcm, encodePcm } from "./pcm";
import type { ConnectVoice, VoiceEvent } from "./types";

const RATE = 24000;
type Playing = {
  source: AudioBufferSourceNode;
  itemId: string;
  index: number;
  start: number;
  end: number;
  offset: number;
};

/** Grok's WebSocket transport owns PCM capture, audio scheduling and interruption truncation. */
export const connectWebSocket: ConnectVoice = async ({
  channelId,
  signal,
  onEvent,
  onError,
  onOutput,
}) => {
  signal.throwIfAborted();
  let stream: MediaStream | undefined;
  let context: AudioContext | undefined;
  let capture: AudioWorkletNode | undefined;
  let source: MediaStreamAudioSourceNode | undefined;
  let silent: GainNode | undefined;
  let output: MediaStreamAudioDestinationNode | undefined;
  let socket: WebSocket | undefined;
  let closed = false;
  let configured = false;
  let nextPlayback = 0;
  let responseId: string | undefined;
  let interruptedResponse: string | undefined;
  const playing: Playing[] = [];
  const offsets = new Map<string, number>();
  const ready = Promise.withResolvers<void>();
  void ready.promise.catch(() => undefined);

  const stopPlayback = () => {
    for (const item of playing.splice(0)) {
      item.source.onended = null;
      item.source.stop();
      item.source.disconnect();
    }
    nextPlayback = context?.currentTime ?? 0;
    onEvent({ type: "output_audio_buffer.cleared" });
  };
  const close = () => {
    if (closed) return;
    closed = true;
    signal.removeEventListener("abort", close);
    ready.reject(new DOMException("Call ended", "AbortError"));
    if (socket) {
      socket.onclose = socket.onerror = socket.onmessage = socket.onopen = null;
      socket.close();
    }
    stopPlayback();
    capture?.disconnect();
    if (capture) capture.port.onmessage = null;
    source?.disconnect();
    silent?.disconnect();
    output?.disconnect();
    for (const track of stream?.getTracks() ?? []) track.stop();
    if (context && context.state !== "closed")
      void context
        .close()
        .catch(() => console.warn("Could not close the voice audio context."));
  };
  const fail = (message: string) => {
    if (closed) return;
    ready.reject(new Error(message));
    close();
    onError(new Error(message));
  };
  const send = (event: VoiceEvent) => {
    if (closed || socket?.readyState !== WebSocket.OPEN)
      throw new Error("The voice call is no longer connected.");
    socket.send(JSON.stringify(event));
  };
  signal.addEventListener("abort", close, { once: true });

  try {
    stream = await navigator.mediaDevices.getUserMedia({
      audio: {
        echoCancellation: true,
        noiseSuppression: true,
        autoGainControl: true,
        channelCount: 1,
      },
    });
    if (closed || signal.aborted) {
      for (const track of stream.getTracks()) track.stop();
      throw new DOMException("Call ended", "AbortError");
    }
    context = new AudioContext({ sampleRate: RATE });
    await context.resume();
    if (
      context.sampleRate !== RATE ||
      !context.audioWorklet ||
      context.state !== "running"
    ) {
      throw new Error(
        "This browser could not start call audio. Allow sound and try again.",
      );
    }
    await context.audioWorklet.addModule(
      new URL("./pcm-worklet.js", import.meta.url),
    );
    signal.throwIfAborted();
    source = context.createMediaStreamSource(stream);
    capture = new AudioWorkletNode(context, "voice-capture");
    silent = context.createGain();
    silent.gain.value = 0;
    source.connect(capture);
    capture.connect(silent).connect(context.destination);
    output = context.createMediaStreamDestination();
    onOutput(output.stream);

    const response = await fetch("/api/voice/calls", {
      method: "POST",
      credentials: "include",
      signal,
      headers: { "Content-Type": "application/json", "x-openbot-voice": "1" },
      body: JSON.stringify({ channelId }),
    });
    const body: unknown = await response.json();
    if (!response.ok)
      throw new Error(
        body &&
          typeof body === "object" &&
          "error" in body &&
          typeof body.error === "string"
          ? body.error
          : "Could not start the voice call.",
      );
    if (
      !body ||
      typeof body !== "object" ||
      !("transport" in body) ||
      body.transport !== "websocket" ||
      !("url" in body) ||
      typeof body.url !== "string" ||
      !("clientSecret" in body) ||
      typeof body.clientSecret !== "string" ||
      !("session" in body) ||
      !body.session ||
      typeof body.session !== "object"
    )
      throw new Error("Invalid voice connection configuration.");
    const address = new URL(body.url);
    if (
      address.origin !== "wss://api.x.ai" ||
      address.pathname !== "/v1/realtime"
    )
      throw new Error("Unsupported voice service address.");
    signal.throwIfAborted();
    socket = new WebSocket(body.url, [
      `xai-client-secret.${body.clientSecret}`,
    ]);
    const ws = socket;
    const audio = context;
    const destination = output;
    ws.onopen = () => send({ type: "session.update", session: body.session });
    ws.onerror = () =>
      fail(
        "Could not connect to Grok Voice. Check your connection and try again.",
      );
    ws.onclose = () =>
      fail("The voice connection closed. Your chat is still available.");
    ws.onmessage = ({ data }) => {
      if (closed) return;
      try {
        const raw: unknown = JSON.parse(data);
        if (
          !raw ||
          typeof raw !== "object" ||
          !("type" in raw) ||
          typeof raw.type !== "string"
        )
          throw new Error("Invalid voice event.");
        const event = raw as VoiceEvent;
        if (event.type === "session.updated") {
          configured = true;
          ready.resolve();
        }
        if (event.type === "response.created") {
          const response = event.response;
          if (
            response &&
            typeof response === "object" &&
            "id" in response &&
            typeof response.id === "string"
          )
            responseId = response.id;
        }
        if (event.type === "input_audio_buffer.speech_started") {
          interruptedResponse = responseId;
          const now = audio.currentTime;
          const active = playing.find((item) => item.end > now);
          if (active)
            send({
              type: "conversation.item.truncate",
              item_id: active.itemId,
              content_index: active.index,
              audio_end_ms: Math.floor(
                active.offset + Math.max(0, now - active.start) * 1000,
              ),
            });
          stopPlayback();
        }
        if (
          event.type === "response.output_audio.delta" ||
          event.type === "response.audio.delta"
        ) {
          if (
            interruptedResponse &&
            (event.response_id ?? responseId) === interruptedResponse
          )
            return;
          if (
            typeof event.delta !== "string" ||
            typeof event.item_id !== "string"
          )
            throw new Error("Invalid audio event.");
          const samples = decodePcm(event.delta);
          if (!samples.length) return;
          if (nextPlayback - audio.currentTime > 30)
            throw new Error(
              "Voice playback could not keep up. Please call again.",
            );
          const buffer = audio.createBuffer(1, samples.length, RATE);
          buffer.copyToChannel(samples, 0);
          const node = audio.createBufferSource();
          node.buffer = buffer;
          node.connect(audio.destination);
          node.connect(destination);
          const start = Math.max(audio.currentTime, nextPlayback);
          const offset = offsets.get(event.item_id) ?? 0;
          const item: Playing = {
            source: node,
            itemId: event.item_id,
            index:
              typeof event.content_index === "number" ? event.content_index : 0,
            start,
            end: start + buffer.duration,
            offset,
          };
          if (offsets.size > 128) offsets.clear();
          offsets.set(event.item_id, offset + buffer.duration * 1000);
          if (!playing.length) onEvent({ type: "output_audio_buffer.started" });
          playing.push(item);
          nextPlayback = item.end;
          node.onended = () => {
            const index = playing.indexOf(item);
            if (index >= 0) playing.splice(index, 1);
            node.disconnect();
            if (!playing.length && !closed)
              onEvent({ type: "output_audio_buffer.stopped" });
          };
          node.start(start);
          return;
        }
        // Older Grok sessions use the shorter event name; consumers only see the normalized form.
        if (event.type === "response.audio_transcript.delta")
          event.type = "response.output_audio_transcript.delta";
        if (event.type === "response.audio_transcript.done")
          event.type = "response.output_audio_transcript.done";
        onEvent(event);
      } catch {
        fail(
          "The voice service sent invalid audio or controls. Please call again.",
        );
      }
    };
    capture.port.onmessage = ({ data }: MessageEvent<ArrayBuffer>) => {
      if (closed || !configured || ws.readyState !== WebSocket.OPEN) return;
      if (ws.bufferedAmount > 1024 * 1024) {
        fail("Your connection is too slow for live audio. Please call again.");
        return;
      }
      send({ type: "input_audio_buffer.append", audio: encodePcm(data) });
    };
    for (const track of stream.getAudioTracks())
      track.onended = () =>
        fail("The microphone disconnected. Check it and call again.");
    await ready.promise;
    signal.throwIfAborted();
    const microphone = stream;
    return {
      stream,
      send,
      close,
      mute(muted) {
        for (const track of microphone.getAudioTracks()) track.enabled = !muted;
        if (muted) send({ type: "input_audio_buffer.clear" });
      },
    };
  } catch (error) {
    close();
    throw error;
  }
};