Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

25.0 KB
Raw Blame
import { Predicate } from "effect";
import { useAtomValue } from "@effect/atom-react";
import {
  createRealtimeVoiceSession,
  createVoiceCallScope,
  runComposedVoiceCall,
  type VoiceCallChatHandlers,
  type VoiceCallScope,
} from "@akeru/client-runtime/voice";
import { BotId } from "@akeru/contracts";
import { useNavigate } from "@tanstack/react-router";
import * as Cause from "effect/Cause";
import {
  useCallback,
  useEffect,
  useMemo,
  useReducer,
  useRef,
  useState,
  type ReactNode,
} from "react";

import type { Bot } from "../roster/types";
import { resolveStickyBotEngine } from "../roster/botEngineSelection";
import { useBotThreadRuntime } from "../roster/useBotThreadRuntime";
import { useRosterStore } from "../roster/rosterStore";
import { toastManager } from "../ui/toast";
import { usePrimarySettings } from "../../hooks/useSettings";
import { resolveAppModelSelectionState } from "../../modelSelection";
import {
  applyProviderInstanceSettings,
  deriveProviderInstanceEntries,
  sortProviderInstanceEntries,
} from "../../providerInstances";
import { primaryServerProvidersAtom, serverEnvironment } from "../../state/server";
import { useProjects } from "../../state/entities";
import { useEnvironmentConnectionState, usePrimaryEnvironmentId } from "../../state/environments";
import { useAtomCommand } from "../../state/use-atom-command";
import { captureVoiceUtterance, playVoiceAudio } from "./browserVoiceAudio";
import { composedVoiceAdapters } from "./composedVoiceCall";
import {
  listenForMicrophoneLoss,
  reduceVoiceCallUiState,
  resolveVoiceCallOfferSdp,
  scheduleVoiceDisconnectTimeout,
  VOICE_MODE_CHANGED_MESSAGE,
  voiceConnectionStateAction,
  voiceEnvironmentConnectionLost,
  voiceStartErrorDescription,
  waitForIceGathering,
} from "./voiceCall.logic";
import { VoiceCallBar } from "./VoiceCallControls";
import {
  type ActiveVoiceCall,
  VoiceCallContext,
  type VoiceCallContextValue,
} from "./voiceCallContext";

export { handleVoiceChannelMessage, type VoiceCallChatHandlers } from "@akeru/client-runtime/voice";

export {
  listenForMicrophoneLoss,
  reduceVoiceCallUiState,
  resolveVoiceCallOfferSdp,
  scheduleVoiceDisconnectTimeout,
  VOICE_MODE_CHANGED_MESSAGE,
  voiceConnectionStateAction,
  voiceEnvironmentConnectionLost,
  voiceStartErrorDescription,
  type VoiceCallUiAction,
  type VoiceCallUiState,
  waitForIceGathering,
} from "./voiceCall.logic";

export {
  BotVoiceCallButton,
  BotVoiceCallButtonView,
  SelectedBotVoiceCallButton,
  VoiceCallBarView,
  VoiceCallStartingBarView,
} from "./VoiceCallControls";

export { useOptionalVoiceCall, useVoiceCall } from "./voiceCallContext";

interface ActiveBrowserCall {
  readonly call: ActiveVoiceCall;
  readonly environmentId: NonNullable<ReturnType<typeof usePrimaryEnvironmentId>>;
  /** Cancels every capture, provider request, reply wait, and playback owned by this call. */
  readonly scope: VoiceCallScope;
  /** Null for composed calls, which take turns over RPC instead of WebRTC. */
  readonly peer: RTCPeerConnection | null;
  readonly microphone: MediaStream;
  readonly speaker: HTMLAudioElement;
  /** Removes this call's data channel listeners. Null for composed calls, which have no channel. */
  readonly eventListeners: AbortController | null;
  stopListeningForDeviceLoss: () => void;
  disconnectTimer: ReturnType<typeof setTimeout> | null;
  environmentDisconnectTimer: ReturnType<typeof setTimeout> | null;
}

interface PendingBrowserCall {
  cancelled: boolean;
  readonly abortController: AbortController;
  readonly environmentId: NonNullable<ReturnType<typeof usePrimaryEnvironmentId>>;
  failure: Error | null;
  scope: VoiceCallScope | null;
  microphone: MediaStream | null;
  peer: RTCPeerConnection | null;
  speaker: HTMLAudioElement | null;
  /** Removes the data channel listeners registered while the call starts. */
  readonly eventListeners: AbortController;
  stopListeningForDeviceLoss: () => void;
}

function stopBrowserCall(active: ActiveBrowserCall): void {
  active.scope.cancel();

  if (active.disconnectTimer !== null) clearTimeout(active.disconnectTimer);

  if (active.environmentDisconnectTimer !== null) {
    clearTimeout(active.environmentDisconnectTimer);
  }

  active.stopListeningForDeviceLoss();

  active.eventListeners?.abort();

  active.microphone.getTracks().forEach((track) => track.stop());
  active.peer?.close();
  active.speaker.pause();
  active.speaker.srcObject = null;
}

function cleanPendingBrowserCall(pending: PendingBrowserCall): void {
  pending.abortController.abort();
  pending.scope?.cancel();
  pending.stopListeningForDeviceLoss();

  pending.eventListeners.abort();

  pending.microphone?.getTracks().forEach((track) => track.stop());
  pending.peer?.close();
  pending.speaker?.pause();

  if (pending.speaker) pending.speaker.srcObject = null;
}

export function VoiceCallProvider({ children }: { readonly children: ReactNode }) {
  const navigate = useNavigate();
  const environmentId = usePrimaryEnvironmentId();
  const environmentConnection = useEnvironmentConnectionState(environmentId);
  const projects = useProjects();
  const startVoiceCall = useAtomCommand(serverEnvironment.startVoiceCall, { reportFailure: false });

  const hangupVoiceCall = useAtomCommand(serverEnvironment.hangupVoiceCall, {
    reportFailure: false,
  });

  const transcribeVoice = useAtomCommand(serverEnvironment.transcribeVoice, {
    reportFailure: false,
  });

  const synthesizeVoice = useAtomCommand(serverEnvironment.synthesizeVoice, {
    reportFailure: false,
  });

  const cancelVoice = useAtomCommand(serverEnvironment.cancelVoice, { reportFailure: false });
  const [activeCall, dispatchCall] = useReducer(reduceVoiceCallUiState, null);
  const [reconnecting, setReconnecting] = useState(false);
  const [startingBotId, setStartingBotId] = useState<string | null>(null);
  const sessionBotId = startingBotId ?? activeCall?.botId ?? null;

  const sessionBot = useRosterStore((state) =>
    sessionBotId === null
      ? null
      : (state.bots.find((candidate) => candidate.id === sessionBotId) ?? null),
  );

  const settings = usePrimarySettings();
  const providers = useAtomValue(primaryServerProvidersAtom);

  const instanceEntries = useMemo(
    () =>
      sortProviderInstanceEntries(
        applyProviderInstanceSettings(deriveProviderInstanceEntries(providers), settings),
      ),
    [providers, settings],
  );

  const defaultSelection = useMemo(
    () => resolveAppModelSelectionState(settings, providers),
    [providers, settings],
  );

  const sessionModelSelection = useMemo(
    () =>
      sessionBot
        ? resolveStickyBotEngine({
            engine: sessionBot.engine,
            instanceEntries,
            settings,
            providers,
            defaultSelection,
          })
        : null,
    [defaultSelection, instanceEntries, providers, sessionBot, settings],
  );

  const runtime = useBotThreadRuntime(sessionBotId ?? "", sessionModelSelection);
  const runtimeRef = useRef(runtime);
  runtimeRef.current = runtime;
  const settingsRef = useRef(settings);
  settingsRef.current = settings;
  const voiceCommandsRef = useRef({ transcribeVoice, synthesizeVoice, cancelVoice });
  voiceCommandsRef.current = { transcribeVoice, synthesizeVoice, cancelVoice };
  // Composed calls wait for the bot's reply; these listeners re-read the turn after each render.
  const turnListenersRef = useRef(new Set<() => void>());
  useEffect(() => {
    for (const changed of turnListenersRef.current) changed();
  }, [runtime.latestTurn, runtime.messages]);
  const activeRef = useRef<ActiveBrowserCall | null>(null);
  const mountedRef = useRef(true);
  const pendingStartRef = useRef<PendingBrowserCall | null>(null);
  const startingBotRef = useRef<string | null>(null);
  const hangupCommandRef = useRef(hangupVoiceCall);
  hangupCommandRef.current = hangupVoiceCall;

  const endBrowserCall = useCallback(
    (
      current: ActiveBrowserCall,
      notice?: {
        readonly type: "warning" | "error";
        readonly title: string;
        readonly description?: string;
      },
    ) => {
      if (activeRef.current !== current) return;
      activeRef.current = null;
      setReconnecting(false);
      stopBrowserCall(current);
      dispatchCall({ type: "hung-up" });
      void hangupCommandRef.current({
        environmentId: current.environmentId,
        input: { callId: current.call.callId },
      });

      if (notice) toastManager.add(notice);
    },
    [],
  );

  useEffect(() => {
    const connection = environmentConnection.data;
    const pending = pendingStartRef.current;

    if (
      pending &&
      (environmentId !== pending.environmentId || voiceEnvironmentConnectionLost(connection))
    ) {
      pending.failure ??= new Error(
        environmentId !== pending.environmentId
          ? "The active environment changed."
          : "The environment connection was lost.",
      );
      cleanPendingBrowserCall(pending);
    }

    const current = activeRef.current;

    if (!current) return;

    if (environmentId !== current.environmentId) {
      endBrowserCall(current, {
        type: "warning",
        title: "Call ended",
        description: "The active environment changed.",
      });

      return;
    }

    if (!voiceEnvironmentConnectionLost(connection)) {
      if (current.environmentDisconnectTimer !== null) {
        clearTimeout(current.environmentDisconnectTimer);
        current.environmentDisconnectTimer = null;
      }

      setReconnecting(false);

      return;
    }

    // Composed calls run every turn over RPC, so they cannot ride out a disconnect.
    if (current.peer === null) {
      endBrowserCall(current, {
        type: "warning",
        title: "Call connection lost",
        description: "Start a new call after the environment reconnects.",
      });

      return;
    }

    setReconnecting(true);
    current.environmentDisconnectTimer ??= setTimeout(() => {
      current.environmentDisconnectTimer = null;
      endBrowserCall(current, {
        type: "warning",
        title: "Call connection lost",
        description: "Start a new call after the environment reconnects.",
      });
    }, 5_000);
  }, [endBrowserCall, environmentConnection.data, environmentId]);

  const returnToCall = useCallback(() => {
    const call = activeRef.current?.call;

    if (!call) return;
    useRosterStore.getState().selectBot(call.botId);
    void navigate({ to: "/bots/$botId", params: { botId: call.botId } });
  }, [navigate]);

  const startOrReturn = useCallback(
    (bot: Bot) => {
      const current = activeRef.current;

      if (current !== null) {
        if (current.call.botId === bot.id) returnToCall();
        else toastManager.add({ type: "warning", title: "A call is already active" });

        return;
      }

      if (startingBotRef.current !== null) return;

      if (environmentId === null) {
        toastManager.add({ type: "error", title: "Voice is unavailable" });

        return;
      }

      if (!projects.some((project) => project.environmentId === environmentId)) {
        toastManager.add({
          type: "error",
          title: "Voice is unavailable",
          description: "Your workspace is still loading. Try again in a moment.",
        });

        return;
      }

      startingBotRef.current = bot.id;
      setStartingBotId(bot.id);
      void (async () => {
        const pending: PendingBrowserCall = {
          cancelled: false,
          abortController: new AbortController(),
          environmentId,
          failure: null,
          scope: null,
          microphone: null,
          peer: null,
          speaker: null,
          eventListeners: new AbortController(),
          stopListeningForDeviceLoss: () => {},
        };

        pendingStartRef.current = pending;
        let serverCallId: string | null = null;

        const hangupCancelledStart = (callId: string) =>
          void hangupCommandRef.current({ environmentId, input: { callId } });

        const activate = (browserCall: ActiveBrowserCall) => {
          activeRef.current = browserCall;
          dispatchCall({ type: "connected", call: browserCall.call });
          setReconnecting(false);
          pending.stopListeningForDeviceLoss = () => {};

          browserCall.stopListeningForDeviceLoss();
          browserCall.stopListeningForDeviceLoss = listenForMicrophoneLoss(
            browserCall.microphone,
            () =>
              endBrowserCall(browserCall, {
                type: "warning",
                title: "Microphone disconnected",
                description: "Connect a microphone, then start a new call.",
              }),
          );
        };

        try {
          if (!navigator.mediaDevices?.getUserMedia) {
            throw new DOMException("No microphone API", "NotFoundError");
          }

          const microphone = await navigator.mediaDevices.getUserMedia({
            audio: {
              echoCancellation: true,
              noiseSuppression: true,
              autoGainControl: true,
            },
            video: false,
          });

          pending.microphone = microphone;

          if (pending.cancelled) {
            microphone.getTracks().forEach((track) => track.stop());

            return;
          }

          const scope = createVoiceCallScope({ botId: bot.id, environmentId });
          pending.scope = scope;
          pending.stopListeningForDeviceLoss = listenForMicrophoneLoss(microphone, () => {
            pending.failure ??= new Error("The microphone disconnected.");
            pending.peer?.close();
          });
          const speaker = new Audio();
          pending.speaker = speaker;

          if (settingsRef.current.voice.provider === "composed") {
            // Capture only while listening; the loop enables tracks for each utterance.
            microphone.getAudioTracks().forEach((track) => {
              track.enabled = false;
            });

            const result = await startVoiceCall({
              environmentId,
              input: { botId: BotId.make(bot.id) },
            });

            if (Predicate.isTagged(result, "Failure")) {
              const cause = Cause.squash(result.cause);
              throw new Error(
                cause instanceof Error ? cause.message : "Could not start the voice call.",
              );
            }

            const callId = result.value.call.callId;
            serverCallId = callId;

            if (pending.cancelled) return hangupCancelledStart(callId);

            if (pending.failure) throw pending.failure;

            if (result.value.transport !== "composed") throw new Error(VOICE_MODE_CHANGED_MESSAGE);

            const browserCall: ActiveBrowserCall = {
              call: result.value.call,
              environmentId,
              scope,
              peer: null,
              microphone,
              speaker,
              eventListeners: null,
              stopListeningForDeviceLoss: pending.stopListeningForDeviceLoss,
              disconnectTimer: null,
              environmentDisconnectTimer: null,
            };

            activate(browserCall);
            const commands = () => voiceCommandsRef.current;
            void runComposedVoiceCall(
              scope,
              composedVoiceAdapters({
                callId,
                capture: (signal) => captureVoiceUtterance(microphone, signal),
                play: (audio, signal) => playVoiceAudio(speaker, audio, signal),
                transcribe: (input) => commands().transcribeVoice({ environmentId, input }),
                synthesize: (input) => commands().synthesizeVoice({ environmentId, input }),
                cancel: (operationId) =>
                  commands().cancelVoice({ environmentId, input: { operationId } }),
                sendMessage: (text) => runtimeRef.current.sendVoiceMessage(text),
                readTurn: () => ({
                  latestTurn: runtimeRef.current.latestTurn,
                  messages: runtimeRef.current.messages,
                }),
                subscribeTurn: (changed) => {
                  turnListenersRef.current.add(changed);

                  return () => turnListenersRef.current.delete(changed);
                },
              }),
            ).catch((cause: unknown) => {
              if (scope.signal.aborted) return;
              endBrowserCall(browserCall, {
                type: "error",
                title: "Voice call ended",
                description: voiceStartErrorDescription(cause),
              });
            });

            return;
          }

          const peer = new RTCPeerConnection();
          pending.peer = peer;
          speaker.autoplay = true;
          peer.ontrack = (event) => {
            if (!event.streams[0]) return;
            speaker.srcObject = event.streams[0];
            void speaker.play().catch(() => undefined);
          };

          microphone.getTracks().forEach((track) => peer.addTrack(track, microphone));
          const events = peer.createDataChannel("oai-events");
          const startListeners = { signal: pending.eventListeners.signal };
          events.addEventListener(
            "error",
            () => {
              pending.failure ??= new Error("The call connection failed.");
            },
            startListeners,
          );

          events.addEventListener(
            "close",
            () => {
              pending.failure ??= new Error("The voice session closed.");
            },
            startListeners,
          );

          const chatHandlers: VoiceCallChatHandlers = {
            appendTranscript: (role, text) => runtimeRef.current.appendTranscript(role, text),
            sendGoalMessage: (text) => runtimeRef.current.send(text, []),
            speechStarted: () => {
              microphone.getAudioTracks().forEach((track) => {
                track.enabled = false;
              });
            },
            speechFinished: () => {
              microphone.getAudioTracks().forEach((track) => {
                track.enabled = true;
              });
            },
            sessionFailed: (message) => {
              const active = activeRef.current;

              if (active?.scope === scope) {
                endBrowserCall(active, {
                  type: "error",
                  title: "Voice session failed",
                  description: message,
                });

                return;
              }

              pending.failure ??= new Error(message);
              pending.peer?.close();
            },
          };

          // One session per call drops replayed event ids and duplicate send_to_chat calls.
          const session = createRealtimeVoiceSession(scope, chatHandlers, (payload) => {
            if (events.readyState === "open") events.send(payload);
          });

          const receive = (channelMessage: MessageEvent) =>
            session.receive(String(channelMessage.data));

          events.addEventListener("message", receive, startListeners);

          const offer = await peer.createOffer();

          if (pending.failure) throw pending.failure;
          await peer.setLocalDescription(offer);

          if (pending.failure) throw pending.failure;
          await waitForIceGathering(peer, 5_000, pending.abortController.signal);

          if (pending.cancelled) return;

          if (pending.failure) throw pending.failure;
          const sdp = resolveVoiceCallOfferSdp(peer, offer);

          if (!sdp) throw new Error("The microphone did not produce a call offer.");

          const result = await startVoiceCall({
            environmentId,
            input: { botId: BotId.make(bot.id), sdp },
          });

          if (Predicate.isTagged(result, "Failure")) {
            const cause = Cause.squash(result.cause);
            throw new Error(
              cause instanceof Error ? cause.message : "Could not start the voice call.",
            );
          }

          serverCallId = result.value.call.callId;

          if (pending.cancelled) return hangupCancelledStart(serverCallId);

          if (pending.failure) throw pending.failure;

          if (result.value.transport !== "webrtc") throw new Error(VOICE_MODE_CHANGED_MESSAGE);

          if (!result.value.answerSdp) throw new Error("The voice call did not return an answer.");
          await peer.setRemoteDescription({ type: "answer", sdp: result.value.answerSdp });

          if (pending.cancelled) return hangupCancelledStart(serverCallId);

          if (pending.failure) throw pending.failure;

          const eventListeners = new AbortController();

          const browserCall: ActiveBrowserCall = {
            call: result.value.call,
            environmentId,
            scope,
            peer,
            microphone,
            speaker,
            eventListeners,
            stopListeningForDeviceLoss: pending.stopListeningForDeviceLoss,
            disconnectTimer: null,
            environmentDisconnectTimer: null,
          };

          activate(browserCall);
          // The call's own listeners replace the start listeners, so a channel failure ends it.
          pending.eventListeners.abort();
          const callListeners = { signal: eventListeners.signal };
          events.addEventListener("message", receive, callListeners);
          events.addEventListener(
            "error",
            () =>
              endBrowserCall(browserCall, {
                type: "error",
                title: "Call connection failed",
                description: "Start a new call to continue.",
              }),
            callListeners,
          );
          events.addEventListener(
            "close",
            () =>
              endBrowserCall(browserCall, {
                type: "warning",
                title: "Call ended",
                description: "The voice session closed.",
              }),
            callListeners,
          );
          peer.onconnectionstatechange = () => {
            const action = voiceConnectionStateAction(peer.connectionState);

            if (action === "recovered") {
              if (browserCall.disconnectTimer !== null) {
                clearTimeout(browserCall.disconnectTimer);
                browserCall.disconnectTimer = null;
              }

              setReconnecting(false);

              return;
            }

            if (action === "wait-for-recovery") {
              setReconnecting(true);
              browserCall.disconnectTimer ??= scheduleVoiceDisconnectTimeout(
                () => peer.connectionState,
                (recovered) => {
                  browserCall.disconnectTimer = null;

                  if (!recovered) {
                    endBrowserCall(browserCall, {
                      type: "warning",
                      title: "Call connection lost",
                      description: "Start a new call to continue.",
                    });
                  }
                },
              );

              return;
            }

            if (action === "end") {
              endBrowserCall(browserCall, {
                type: "error",
                title: "Call connection failed",
                description: "Start a new call to continue.",
              });
            }
          };
        } catch (error) {
          cleanPendingBrowserCall(pending);

          if (serverCallId !== null) {
            void hangupVoiceCall({ environmentId, input: { callId: serverCallId } });
          }

          if (pending.cancelled) return;
          toastManager.add({
            type: "error",
            title: "Could not start call",
            description: voiceStartErrorDescription(error),
          });
        } finally {
          if (pendingStartRef.current === pending) pendingStartRef.current = null;

          if (startingBotRef.current === bot.id) startingBotRef.current = null;

          if (mountedRef.current) setStartingBotId(null);
        }
      })();
    },
    [endBrowserCall, environmentId, hangupVoiceCall, projects, returnToCall, startVoiceCall],
  );

  const hangup = useCallback(() => {
    const current = activeRef.current;

    if (current) {
      endBrowserCall(current);

      return;
    }

    const pending = pendingStartRef.current;

    if (!pending) return;
    pending.cancelled = true;
    cleanPendingBrowserCall(pending);
    pendingStartRef.current = null;
    startingBotRef.current = null;
    setStartingBotId(null);
  }, [endBrowserCall]);

  useEffect(() => {
    mountedRef.current = true;

    return () => {
      mountedRef.current = false;
      const pending = pendingStartRef.current;

      if (pending) {
        pending.cancelled = true;
        cleanPendingBrowserCall(pending);
        pendingStartRef.current = null;
      }

      const current = activeRef.current;

      if (!current) return;
      activeRef.current = null;
      stopBrowserCall(current);
      void hangupCommandRef.current({
        environmentId: current.environmentId,
        input: { callId: current.call.callId },
      });
    };
  }, []);

  const value = useMemo<VoiceCallContextValue>(
    () => ({ activeCall, reconnecting, startingBotId, startOrReturn, hangup, returnToCall }),
    [activeCall, hangup, reconnecting, returnToCall, startOrReturn, startingBotId],
  );

  return (
    <VoiceCallContext.Provider value={value}>
      {children}
      <VoiceCallBar />
    </VoiceCallContext.Provider>
  );
}