Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

2.9 KB
Raw Blame
import { Predicate } from "effect";
import { createReplyPlaybackSession } from "@akeru/client-runtime/reply-playback";
import { EnvironmentId } from "@akeru/contracts";
import { storedReplySynthesisCapability } from "@akeru/client-runtime/reply-playback";
import { createExpoReplyAudio } from "./expoReplyAudio";
import { synthesizeVoiceChunks } from "@akeru/client-runtime/voice";
import { decodeReplyAudioBase64 } from "./base64";
import * as SecureStore from "expo-secure-store";

type VoiceSettings = Parameters<typeof storedReplySynthesisCapability>[0];

/** The session lives above the navigator, so per-environment state comes from
    the playback request identity instead of a navigation route. */
export function createMobileReplyPlaybackSession(options: {
  readonly synthesize: (target: {
    environmentId: EnvironmentId;
    input: { operationId: string; text: string };
  }) => Promise<{
    _tag: string;
    value?: { audioBase64: string; mimeType: string };
  }>;
  readonly cancel: (target: {
    environmentId: EnvironmentId;
    input: { operationId: string };
  }) => Promise<void | { readonly _tag: string }>;
  readonly voiceSettings: (environmentId: string) => VoiceSettings;
}) {
  return createReplyPlaybackSession({
    storage: {
      getItem: async (key) => SecureStore.getItemAsync(key),
      setItem: async (key, value) => {
        await SecureStore.setItemAsync(key, value);
      },
    },
    synthesis: (environmentId) =>
      storedReplySynthesisCapability(options.voiceSettings(environmentId)),
    prepare: async (request, signal, events) => {
      const environmentId = EnvironmentId.make(request.identity.environmentId);
      const operationId = `voice-${Date.now()}-${Math.random()}`;

      const abort = () => {
        void options.cancel({ environmentId, input: { operationId } });
      };

      signal.addEventListener("abort", abort, { once: true });

      try {
        const segments: Uint8Array[] = [];
        let mimeType = "audio/mpeg";

        const results = await synthesizeVoiceChunks(request.text, signal, (text) =>
          options.synthesize({ environmentId, input: { operationId, text } }),
        );

        for (const result of results) {
          const value = result.value;

          if (!Predicate.isTagged(result, "Success") || !value)
            throw new Error("Voice synthesis failed.");
          segments.push(decodeReplyAudioBase64(value.audioBase64));
          mimeType = value.mimeType;
        }

        const bytes = new Uint8Array(
          segments.reduce((total, segment) => total + segment.length, 0),
        );

        let offset = 0;

        for (const segment of segments) {
          bytes.set(segment, offset);
          offset += segment.length;
        }

        return createExpoReplyAudio(bytes, mimeType, events);
      } finally {
        signal.removeEventListener("abort", abort);
      }
    },
  });
}