Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

5.1 KB
Raw Blame
import * as Predicate from "effect/Predicate";
import { createReplyPlaybackSession } from "@akeru/client-runtime/reply-playback";
import { storedReplySynthesisCapability } from "@akeru/client-runtime/reply-playback";
import { useAtomValue } from "@effect/atom-react";
import { useEffect, useMemo, useRef } from "react";
import { useAtomCommand } from "~/state/use-atom-command";
import { primaryServerSettingsAtom, serverEnvironment } from "~/state/server";
import { usePrimaryEnvironmentId } from "~/state/environments";
import { createBrowserReplyAudio } from "./replyPlaybackAudio";
import { synthesizeVoiceChunks } from "@akeru/client-runtime/voice";

type ReplySynthesisResult = {
  readonly _tag: string;
  readonly value?: { readonly audioBase64: string; readonly mimeType: "audio/mpeg" };
};

function isSynthesisSuccess(
  value: ReplySynthesisResult,
): value is ReplySynthesisResult & { readonly _tag: "Success" } {
  return Predicate.isTagged(value, "Success");
}

const OTHER_ENVIRONMENT_SPEECH_UNAVAILABLE =
  "Reading replies aloud is only available for this device's primary environment.";

let operationSequence = 0;

const operationId = () => `voice-${Date.now()}-${operationSequence++}`;

export function useWebReplyPlaybackSession() {
  const environmentId = usePrimaryEnvironmentId();
  const settings = useAtomValue(primaryServerSettingsAtom);
  const synthesize = useAtomCommand(serverEnvironment.synthesizeVoice, { reportFailure: false });
  const cancel = useAtomCommand(serverEnvironment.cancelVoice, { reportFailure: false });
  const voice = settings.voice;
  const voiceRef = useRef(voice);

  const session = useMemo(
    () =>
      createWebReplyPlaybackSession({
        ...(environmentId ? { environmentId } : {}),
        voice: () => voiceRef.current,
        ...(environmentId
          ? {
              synthesize: ({ input }) => synthesize({ environmentId, input }),
              cancel: async ({ input }) => {
                await cancel({ environmentId, input });
              },
            }
          : {}),
      }),
    [cancel, environmentId, synthesize],
  );

  // Voice setting changes update the live session, so playback and automatic readout survive them.
  // The per-environment check still applies, so other environments' replies stay unavailable.
  useEffect(() => {
    voiceRef.current = voice;
    session.refreshSynthesis();
  }, [session, voice]);

  return session;
}

export function createWebReplyPlaybackSession(
  options: {
    readonly environmentId?: string | null;
    readonly voice?:
      | Parameters<typeof storedReplySynthesisCapability>[0]
      | (() => Parameters<typeof storedReplySynthesisCapability>[0]);
    readonly synthesize?: (target: {
      environmentId: string;
      input: { operationId: string; text: string };
    }) => Promise<ReplySynthesisResult>;
    readonly cancel?: (target: {
      environmentId: string;
      input: { operationId: string };
    }) => Promise<void>;
  } = {},
) {
  const environmentId = options.environmentId ?? null;
  const readVoice = () => (Predicate.isFunction(options.voice) ? options.voice() : options.voice);

  return createReplyPlaybackSession({
    storage: {
      getItem: async (key) =>
        typeof localStorage === "undefined" ? null : localStorage.getItem(key),
      setItem: async (key, value) => {
        if (typeof localStorage === "undefined") return;
        localStorage.setItem(key, value);
      },
    },
    // Speech runs on the primary environment with its voice settings, so replies from another
    // environment stay unavailable instead of being read by the wrong server.
    synthesis: (replyEnvironmentId) =>
      replyEnvironmentId === environmentId
        ? storedReplySynthesisCapability(readVoice())
        : {
            available: false,
            provider: "unavailable",
            voice: "unavailable",
            reason: OTHER_ENVIRONMENT_SPEECH_UNAVAILABLE,
          },
    prepare: async (request, signal, events) => {
      if (!environmentId || !options.synthesize || !options.cancel)
        throw new Error("Voice synthesis is unavailable.");

      if (request.identity.environmentId !== environmentId)
        throw new Error(OTHER_ENVIRONMENT_SPEECH_UNAVAILABLE);
      const id = operationId();

      const abort = () => {
        void options.cancel?.({ environmentId, input: { operationId: id } });
      };

      signal.addEventListener("abort", abort, { once: true });

      try {
        const parts: BlobPart[] = [];
        let mimeType = "audio/mpeg";

        for (const result of await synthesizeVoiceChunks(request.text, signal, (text) =>
          options.synthesize!({ environmentId, input: { operationId: id, text } }),
        )) {
          if (!isSynthesisSuccess(result) || !result.value)
            throw new Error("Voice synthesis failed.");
          signal.throwIfAborted();
          const binary = atob(result.value.audioBase64);
          parts.push(Uint8Array.from(binary, (value) => value.charCodeAt(0)));
          mimeType = result.value.mimeType;
        }

        return createBrowserReplyAudio(new Blob(parts, { type: mimeType }), events);
      } finally {
        signal.removeEventListener("abort", abort);
      }
    },
  });
}