Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

5.2 KB
Raw Blame
import type { ConnectVoice, VoiceEvent } from "./types";

/** WebRTC owns audio buffering and interruption truncation; the data channel carries controls. */
export const connectWebRtc: ConnectVoice = async ({
  channelId,
  signal,
  onEvent,
  onError,
  onOutput,
}) => {
  if (typeof RTCPeerConnection === "undefined")
    throw new Error("This browser does not support WebRTC voice calls.");
  signal.throwIfAborted();
  let stream: MediaStream | undefined;
  let peer: RTCPeerConnection | undefined;
  let channel: RTCDataChannel | undefined;
  let audio: HTMLAudioElement | undefined;
  let closed = false;
  const ready = Promise.withResolvers<void>();
  // Permission and SDP negotiation can fail before the ready promise is awaited.
  void ready.promise.catch(() => undefined);
  const close = () => {
    if (closed) return;
    closed = true;
    signal.removeEventListener("abort", abort);
    ready.reject(new DOMException("Call ended", "AbortError"));
    channel?.close();
    peer?.close();
    for (const track of stream?.getTracks() ?? []) track.stop();
    if (audio) {
      audio.pause();
      audio.srcObject = null;
    }
  };
  const fail = (message: string) => {
    if (closed) return;
    ready.reject(new Error(message));
    close();
    onError(new Error(message));
  };
  const abort = () => close();
  signal.addEventListener("abort", abort, { once: true });
  try {
    stream = await navigator.mediaDevices.getUserMedia({
      audio: {
        echoCancellation: true,
        noiseSuppression: true,
        autoGainControl: true,
      },
    });
    // A permission dialog can resolve after hangup; its newly granted tracks still need releasing.
    if (closed || signal.aborted) {
      for (const track of stream.getTracks()) track.stop();
      throw new DOMException("Call ended", "AbortError");
    }
    peer = new RTCPeerConnection();
    audio = new Audio();
    audio.autoplay = true;
    const player = audio;
    peer.ontrack = ({ track, streams }) => {
      if (closed) return;
      const output = streams[0] ?? new MediaStream([track]);
      player.srcObject = output;
      onOutput(output);
      void player
        .play()
        .catch(() =>
          fail(
            "Audio playback was blocked. Allow sound in your browser, then call again.",
          ),
        );
    };
    peer.onconnectionstatechange = () => {
      if (
        peer?.connectionState === "failed" ||
        peer?.connectionState === "disconnected"
      ) {
        fail(
          "The voice connection was lost. Your chat is still available. Call again to reconnect.",
        );
      }
    };
    for (const track of stream.getAudioTracks()) {
      peer.addTrack(track, stream);
      track.onended = () =>
        fail("The microphone disconnected. Check it and call again.");
    }
    channel = peer.createDataChannel("oai-events");
    channel.onopen = () => ready.resolve();
    channel.onerror = () =>
      fail("The voice connection failed. Please call again.");
    channel.onclose = () => fail("The voice call ended. You can call again.");
    channel.onmessage = ({ data }) => {
      if (closed) return;
      try {
        const event: unknown = JSON.parse(data);
        if (
          event &&
          typeof event === "object" &&
          "type" in event &&
          typeof event.type === "string"
        ) {
          onEvent(event as VoiceEvent);
        }
      } catch {
        fail(
          "The voice service returned an unreadable event. Please call again.",
        );
      }
    };
    const offer = await peer.createOffer();
    await peer.setLocalDescription(offer);
    const response = await fetch("/api/voice/calls", {
      method: "POST",
      credentials: "include",
      headers: { "Content-Type": "application/json", "x-openbot-voice": "1" },
      body: JSON.stringify({ channelId, sdp: offer.sdp }),
      signal,
    });
    if (!response.ok) {
      const body: unknown = await response.json().catch(() => null);
      throw new Error(
        body &&
          typeof body === "object" &&
          "error" in body &&
          typeof body.error === "string"
          ? body.error
          : "Could not connect the voice call. Please try again.",
      );
    }
    if (closed) throw new DOMException("Call ended", "AbortError");
    const answer: unknown = await response.json();
    if (
      !answer ||
      typeof answer !== "object" ||
      !("transport" in answer) ||
      answer.transport !== "webrtc" ||
      !("sdp" in answer) ||
      typeof answer.sdp !== "string"
    ) {
      throw new Error(
        "The voice service returned an invalid connection. Please try again.",
      );
    }
    await peer.setRemoteDescription({ type: "answer", sdp: answer.sdp });
    await ready.promise;
    signal.throwIfAborted();
    const events = channel;
    const microphone = stream;
    return {
      stream,
      send(event) {
        if (closed || events.readyState !== "open")
          throw new Error("The voice call is no longer connected.");
        events.send(JSON.stringify(event));
      },
      mute(muted) {
        for (const track of microphone.getAudioTracks()) track.enabled = !muted;
      },
      close,
    };
  } catch (error) {
    close();
    throw error;
  }
};