Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

12.9 KB
Raw Blame
import { asRecord } from "./work-log-command.ts";
import * as Predicate from "effect/Predicate";
import type { OrchestrationLatestTurn, OrchestrationMessage } from "@akeru/contracts";

export interface VoiceAudio {
  readonly audioBase64: string;
  readonly mimeType: "audio/webm" | "audio/wav" | "audio/mpeg" | "audio/mp4" | "audio/ogg";
}

export const VOICE_SYNTHESIS_MAX_CHARS = 4_000;

/** Splits stored speech at the server's synthesis input limit. MP3 segments are concatenated by clients. */
export function splitVoiceSynthesisText(text: string): ReadonlyArray<string> {
  const chunks: string[] = [];
  let offset = 0;

  while (offset < text.length) {
    const limit = Math.min(offset + VOICE_SYNTHESIS_MAX_CHARS, text.length);

    if (limit === text.length) {
      chunks.push(text.slice(offset));
      break;
    }

    const candidate = text.slice(offset, limit);
    const sentence = [...candidate.matchAll(/[.!?。!?](?=\s|$)/g)].at(-1)?.index;
    const whitespace = [...candidate.matchAll(/\s/g)].at(-1)?.index;

    let end =
      sentence !== undefined
        ? offset + sentence + 1
        : whitespace !== undefined
          ? offset + whitespace + 1
          : limit;

    if (end <= offset) end = limit;

    if (end > offset && /[\uD800-\uDBFF]$/.test(text.slice(offset, end))) end -= 1;
    chunks.push(text.slice(offset, end));
    offset = end;
  }

  return chunks;
}

export async function synthesizeVoiceChunks<T>(
  text: string,
  signal: AbortSignal,
  synthesize: (chunk: string) => Promise<T>,
): Promise<ReadonlyArray<T>> {
  const results: T[] = [];

  for (const chunk of splitVoiceSynthesisText(text)) {
    signal.throwIfAborted();
    results.push(await synthesize(chunk));
    signal.throwIfAborted();
  }

  return results;
}

/** Shared by call-bound audio and standalone dictation/read-aloud adapters. */
export async function runVoiceOperation<T, CancelResult>(
  signal: AbortSignal,
  execute: (operationId: string) => Promise<T>,
  cancel: (operationId: string) => Promise<CancelResult>,
  operationId: string,
): Promise<T> {
  signal.throwIfAborted();

  const onAbort = () => {
    void cancel(operationId).catch(() => undefined);
  };

  signal.addEventListener("abort", onAbort, { once: true });

  try {
    const result = await execute(operationId);
    signal.throwIfAborted();

    return result;
  } finally {
    signal.removeEventListener("abort", onAbort);
  }
}

/**
 * Finds the reply to speak for the voice turn started by `requestMessageId`.
 * If a newer chat turn has already replaced it as the latest turn, the voice
 * turn's final reply is still spoken, once none of its messages are streaming.
 * `observedTurnId` is the voice turn's id seen while it was the latest turn;
 * request messages carry no turn id, so it is how a late reply is found.
 */
export function correlatedVoiceReply(
  requestMessageId: string,
  latestTurn: OrchestrationLatestTurn | null,
  messages: readonly OrchestrationMessage[],
  observedTurnId: OrchestrationLatestTurn["turnId"] | null = null,
): string | null {
  if (latestTurn?.requestMessageId === requestMessageId) {
    if (latestTurn.state === "error" || latestTurn.state === "interrupted") {
      throw new Error("The bot turn did not complete. Continue in chat.");
    }

    if (latestTurn.state !== "completed") return null;

    // A completed turn without an assistant message has nothing to speak.
    if (latestTurn.assistantMessageId === null) return "";
    const message = messages.find((item) => item.id === latestTurn.assistantMessageId);

    return message?.role === "assistant" && !message.streaming ? message.text : null;
  }

  const request = messages.find((item) => item.id === requestMessageId && item.role === "user");
  const turnId = observedTurnId ?? request?.turnId;

  if (!turnId) return null;
  const replies = messages.filter((item) => item.turnId === turnId && item.role === "assistant");

  if (replies.some((item) => item.streaming)) return null;
  const reply = replies.at(-1);

  if (reply) return reply.text;

  // The voice turn was last seen still running, since it ends only while it is
  // the latest turn, so its reply may still arrive.
  if (observedTurnId !== null) return null;

  if (latestTurn && request && latestTurn.requestedAt > request.createdAt) {
    throw new Error("The bot turn did not complete. Continue in chat.");
  }

  return null;
}

export function waitForVoiceReply(
  signal: AbortSignal,
  read: () => string | null,
  subscribe: (changed: () => void) => () => void,
): Promise<string> {
  signal.throwIfAborted();

  return new Promise((resolve, reject) => {
    let unsubscribe = () => {};

    let settled = false;

    const finish = (text: string | null, cause?: unknown) => {
      if (settled) return;
      settled = true;
      unsubscribe();
      signal.removeEventListener("abort", aborted);

      if (cause !== undefined) reject(cause);
      else resolve(text ?? "");
    };

    const aborted = () => finish(null, signal.reason);

    const changed = () => {
      try {
        const text = read();

        if (text !== null) finish(text);
      } catch (error) {
        finish(null, error);
      }
    };

    signal.addEventListener("abort", aborted, { once: true });
    unsubscribe = subscribe(changed);

    if (settled) unsubscribe();
    else changed();
  });
}

export interface ComposedVoiceAdapters {
  readonly capture: (signal: AbortSignal) => Promise<VoiceAudio>;
  readonly transcribe: (audio: VoiceAudio, signal: AbortSignal) => Promise<string>;
  readonly sendAndWait: (text: string, signal: AbortSignal) => Promise<string>;
  readonly synthesize: (text: string, signal: AbortSignal) => Promise<VoiceAudio>;
  readonly play: (audio: VoiceAudio, signal: AbortSignal) => Promise<void>;
}

/** Capture resumes only after the accepted bot turn and all playback have finished. */
export async function runComposedVoiceCall(scope: VoiceCallScope, adapters: ComposedVoiceAdapters) {
  const pinned = { ...adapters };
  const { signal } = scope;

  while (!signal.aborted) {
    const audio = await pinned.capture(signal);
    signal.throwIfAborted();
    const text = (await pinned.transcribe(audio, signal)).trim();
    signal.throwIfAborted();

    if (!text) continue;
    const response = await pinned.sendAndWait(text, signal);
    signal.throwIfAborted();
    // Provider requests are bounded; play long responses sequentially, never concurrently.
    // The next chunk is synthesized while the current one plays so speech starts early.
    const chunks = splitVoiceSynthesisText(response);
    let next = chunks[0] === undefined ? undefined : pinned.synthesize(chunks[0], signal);

    for (let index = 0; next; index += 1) {
      const speech = await next;
      signal.throwIfAborted();
      const following = chunks[index + 1];
      next = following === undefined ? undefined : pinned.synthesize(following, signal);
      // Playback failure must not leave the prefetched request unhandled.
      next?.catch(() => undefined);
      await pinned.play(speech, signal);
      signal.throwIfAborted();
    }
  }
}

export interface VoiceCallIdentity {
  readonly botId: string;
  readonly environmentId: string;
}

export function createVoiceCallScope(identity: VoiceCallIdentity) {
  const controller = new AbortController();

  return {
    identity: Object.freeze({ ...identity }),
    signal: controller.signal,
    cancel: () => controller.abort(),
  };
}

export type VoiceCallScope = ReturnType<typeof createVoiceCallScope>;

export function createRealtimeVoiceSession(
  scope: VoiceCallScope,
  handlers: VoiceCallChatHandlers,
  reply: (payload: string) => void,
) {
  const state = { eventIds: new Set<string>(), functionCallIds: new Set<string>() };
  const snapshot = { ...handlers };

  const pinnedHandlers: VoiceCallChatHandlers = {
    appendTranscript: (role, text) => {
      if (!scope.signal.aborted) return snapshot.appendTranscript(role, text);
    },
    sendGoalMessage: (text) => (scope.signal.aborted ? false : snapshot.sendGoalMessage(text)),
    speechStarted: () => {
      if (!scope.signal.aborted) snapshot.speechStarted();
    },
    speechFinished: () => {
      if (!scope.signal.aborted) snapshot.speechFinished();
    },
    sessionFailed: (message) => {
      if (!scope.signal.aborted) snapshot.sessionFailed(message);
    },
  };

  return {
    receive(raw: string) {
      if (scope.signal.aborted) return;
      handleVoiceChannelMessage(
        raw,
        pinnedHandlers,
        (payload) => {
          if (!scope.signal.aborted) reply(payload);
        },
        state,
      );
    },
  };
}

export interface VoiceCallChatHandlers {
  readonly appendTranscript: (role: "user" | "assistant", text: string) => void | Promise<void>;
  readonly sendGoalMessage: (text: string) => boolean | Promise<boolean>;
  readonly speechStarted: () => void;
  readonly speechFinished: () => void;
  readonly sessionFailed: (message: string) => void;
}

function invokeVoiceHandler<T>(execute: () => T | Promise<T>): Promise<T> {
  try {
    return Promise.resolve(execute());
  } catch (error) {
    return Promise.reject(error);
  }
}

function stringField(value: Parameters<typeof asRecord>[0], field: string): string | null {
  if (!Predicate.isObjectOrArray(value)) return null;
  const candidate = Predicate.hasProperty(value, field) ? value[field] : undefined;

  return Predicate.isString(candidate) ? candidate : null;
}

const VOICE_EVENT_ID_MEMORY = 4_096;

export function handleVoiceChannelMessage(
  raw: string,
  handlers: VoiceCallChatHandlers,
  reply: (payload: string) => void = () => {},
  state = { eventIds: new Set<string>(), functionCallIds: new Set<string>() },
): void {
  let event: unknown;

  try {
    event = JSON.parse(raw);
  } catch {
    return;
  }

  const eventId = stringField(event, "event_id");

  if (eventId !== null) {
    if (state.eventIds.has(eventId)) return;
    state.eventIds.add(eventId);

    // Replays arrive close to the original, so only recent ids need remembering.
    if (state.eventIds.size > VOICE_EVENT_ID_MEMORY) {
      state.eventIds.delete(state.eventIds.values().next().value!);
    }
  }

  const type = stringField(event, "type");
  const transcript = stringField(event, "transcript")?.trim() ?? "";

  if (type === "conversation.item.input_audio_transcription.completed" && transcript.length > 0) {
    void invokeVoiceHandler(() => handlers.appendTranscript("user", transcript)).catch(() =>
      handlers.sessionFailed("Could not save the voice transcript."),
    );

    return;
  }

  if (
    (type === "response.output_audio_transcript.done" ||
      type === "response.audio_transcript.done") &&
    transcript.length > 0
  ) {
    void invokeVoiceHandler(() => handlers.appendTranscript("assistant", transcript)).catch(() =>
      handlers.sessionFailed("Could not save the voice transcript."),
    );

    return;
  }

  if (type === "output_audio_buffer.started") {
    handlers.speechStarted();

    return;
  }

  if (type === "output_audio_buffer.stopped") {
    handlers.speechFinished();

    return;
  }

  if (type === "response.done") {
    const response = Predicate.isObjectOrArray(event)
      ? Predicate.hasProperty(event, "response")
        ? event.response
        : null
      : null;

    const status = stringField(response, "status");

    if (status !== "completed" && status !== "cancelled" && status !== "incomplete") {
      handlers.sessionFailed("The voice response failed.");
    }

    return;
  }

  if (
    type === "response.function_call_arguments.done" &&
    stringField(event, "name") === "send_to_chat"
  ) {
    const callId = stringField(event, "call_id");

    if (callId === null || state.functionCallIds.has(callId)) return;
    state.functionCallIds.add(callId);

    const finish = (
      output:
        | { readonly ok: true; readonly delivered: "chat" }
        | { readonly ok: false; readonly error: string },
    ) => {
      if (callId === null) return;
      reply(
        JSON.stringify({
          type: "conversation.item.create",
          item: {
            type: "function_call_output",
            call_id: callId,
            output: JSON.stringify(output),
          },
        }),
      );
      reply(JSON.stringify({ type: "response.create" }));
    };

    let parsedArguments: unknown;

    try {
      parsedArguments = JSON.parse(stringField(event, "arguments") ?? "");
    } catch {
      finish({ ok: false, error: "Could not parse the tool arguments." });

      return;
    }

    const message = stringField(parsedArguments, "message")?.trim() ?? "";

    if (message.length === 0) {
      finish({ ok: false, error: "The message was empty." });

      return;
    }

    void invokeVoiceHandler(() => handlers.sendGoalMessage(message))
      .catch(() => false)
      .then((delivered) =>
        finish(
          delivered
            ? { ok: true, delivered: "chat" }
            : { ok: false, error: "The chat did not accept the message." },
        ),
      );

    return;
  }

  if (type === "error") {
    handlers.sessionFailed("The voice session failed. Start a new call to continue.");
  }
}