Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

9.4 KB
Raw Blame
import * as Predicate from "effect/Predicate";
import { createAutomaticReadoutTracker, type CompletedReply } from "./automaticReadout.ts";
import {
  storedReplySynthesisCapability,
  type StoredReplySynthesisCapability,
} from "./capability.ts";
import {
  createReplyPlaybackController,
  type ReplyAudioEvents,
  type ReplyAudioHandle,
  type ReplyPlaybackContext,
  type ReplyPlaybackRequest,
} from "./controller.ts";
import { spokenTextDisclosure } from "./disclosure.ts";
import { replyReadoutMessageAction } from "./messageAction.ts";
import { createReplyReadoutPreference, type ReplyReadoutStorage } from "./preference.ts";

export interface ReplyPlaybackMessage {
  readonly id: string;
  readonly role: "user" | "assistant" | "system";
  readonly streaming: boolean;
  readonly text: string;
  readonly updatedAt: string;
}

type SpokenReply = NonNullable<ReturnType<typeof replyReadoutMessageAction>>;

// Large enough that a full pass over a long chat fits, so revisits reuse cached speech.
export const SPOKEN_CACHE_LIMIT = 2_000;

export interface ReplyPlaybackAction {
  readonly request: ReplyPlaybackRequest;
  readonly disclosure?: string;
  readonly unavailableReason?: string;
}

export function createReplyPlaybackSession(options: {
  readonly storage: ReplyReadoutStorage;
  readonly prepare: (
    request: ReplyPlaybackRequest,
    signal: AbortSignal,
    events: ReplyAudioEvents,
  ) => Promise<ReplyAudioHandle>;
  readonly synthesis?:
    | StoredReplySynthesisCapability
    | ((environmentId: string) => StoredReplySynthesisCapability);
}) {
  const resolveSynthesis = (environmentId: string | null) =>
    Predicate.isFunction(options.synthesis)
      ? environmentId
        ? options.synthesis(environmentId)
        : storedReplySynthesisCapability()
      : (options.synthesis ?? storedReplySynthesisCapability());

  // Environment-scoped overrides keep one chat's voice settings out of another environment.
  let synthesisOverride: StoredReplySynthesisCapability | null = null;
  const environmentSynthesisOverrides = new Map<string, StoredReplySynthesisCapability>();

  const currentSynthesis = (environmentId: string | null) =>
    (environmentId ? environmentSynthesisOverrides.get(environmentId) : undefined) ??
    synthesisOverride ??
    resolveSynthesis(environmentId);

  const synthesisListeners = new Set<() => void>();
  // `useSyncExternalStore` needs the same object until the capability changes, while resolvers
  // build a fresh one per call, so equal capabilities reuse the last snapshot.
  let synthesisSnapshot: StoredReplySynthesisCapability | null = null;

  const readSynthesisSnapshot = () => {
    const next = currentSynthesis(context?.environmentId ?? null);

    if (!synthesisSnapshot || !sameSynthesis(synthesisSnapshot, next)) synthesisSnapshot = next;

    return synthesisSnapshot;
  };

  const notifySynthesis = () => {
    for (const listener of synthesisListeners) listener();
  };

  const notifyIfSynthesisChanged = () => {
    const previous = synthesisSnapshot;

    if (readSynthesisSnapshot() !== previous) notifySynthesis();
  };

  const tracker = createAutomaticReadoutTracker();
  const controller = createReplyPlaybackController(options.prepare);

  const preference = createReplyReadoutPreference(options.storage, () => {
    tracker.setEnabled(false);
    controller.disableAutomaticReadout();
  });

  preference.subscribe(() => tracker.setEnabled(preference.getSnapshot().enabled));
  let context: ReplyPlaybackContext | null = null;
  let scope: string | null = null;
  let seen = new Set<string>();
  let sequence = 0;
  let baseline: string | null = null;
  // Rows and the observer ask for the same reply on every render, so spoken text is cached per
  // message and reused while its stored text is unchanged.
  const spokenCache = new Map<string, { text: string; spoken: SpokenReply }>();

  const spokenFor = (message: ReplyPlaybackMessage): SpokenReply | null => {
    if (message.role !== "assistant" || message.streaming) return null;
    const cached = spokenCache.get(message.id);

    if (cached && cached.text === message.text) return cached.spoken;
    const spoken = replyReadoutMessageAction(message);

    if (!spoken) return null;
    // Re-insert so the map stays in write order, then evict the oldest entry past the limit.
    spokenCache.delete(message.id);
    spokenCache.set(message.id, { text: message.text, spoken });

    if (spokenCache.size > SPOKEN_CACHE_LIMIT) {
      const oldest = spokenCache.keys().next();

      if (!oldest.done) spokenCache.delete(oldest.value);
    }

    return spoken;
  };

  const identityBase = () =>
    context
      ? {
          environmentId: context.environmentId,
          threadId: context.threadId,
          provider: context.provider,
          voice: context.voice,
        }
      : null;

  const actionFor = (message: ReplyPlaybackMessage): ReplyPlaybackAction | null => {
    const spoken = spokenFor(message);
    const base = identityBase();

    if (!spoken || !base) return null;
    const synthesis = currentSynthesis(base.environmentId);

    const request: ReplyPlaybackRequest = {
      identity: {
        ...base,
        messageId: message.id,
        contentVersion: message.updatedAt,
      },
      text: spoken.text,
      automatic: false,
    };

    const disclosure = spokenTextDisclosure(spoken);

    if (!spoken.speakable) {
      return {
        request,
        ...(disclosure
          ? { disclosure, unavailableReason: disclosure }
          : { unavailableReason: "This reply has no readable text." }),
      };
    }

    if (!synthesis.available) {
      return {
        request,
        ...(disclosure ? { disclosure } : {}),
        unavailableReason: synthesis.reason,
      };
    }

    return disclosure ? { request, disclosure } : { request };
  };

  return {
    controller,
    preference,
    get synthesis() {
      return readSynthesisSnapshot();
    },
    /** The capability replies from `environmentId` use, including applied settings. */
    synthesisFor: currentSynthesis,
    getSynthesisSnapshot: readSynthesisSnapshot,
    subscribeSynthesis: (listener: () => void) => {
      synthesisListeners.add(listener);

      return () => synthesisListeners.delete(listener);
    },
    // Tells listeners that a synthesis function now returns different values.
    refreshSynthesis: notifySynthesis,
    /** Pass `environmentId` when the capability belongs to one environment's settings. */
    setSynthesis: (next: StoredReplySynthesisCapability, environmentId?: string) => {
      if (environmentId) environmentSynthesisOverrides.set(environmentId, next);
      else synthesisOverride = next;
      notifySynthesis();
    },
    actionFor,
    setContext: (next: ReplyPlaybackContext | null) => {
      context = next;
      controller.setContext(next);
      const nextScope = next ? `${next.environmentId}/${next.threadId}` : null;

      if (nextScope !== scope) {
        scope = nextScope;
        seen = new Set();
        sequence = 0;
        baseline = null;
        tracker.reset(scope, 0);
      }

      notifyIfSynthesisChanged();
    },
    clearContextIf: (environmentId: string, threadId: string) => {
      if (context?.environmentId === environmentId && context.threadId === threadId) {
        context = null;
        controller.setContext(null);
        scope = null;
        seen = new Set();
        sequence = 0;
        baseline = null;
        tracker.reset(null, 0);
        notifyIfSynthesisChanged();
      }
    },
    observe: (messages: ReadonlyArray<ReplyPlaybackMessage>) => {
      const versions = new Map<string, string>();
      const live: CompletedReply[] = [];

      for (const message of messages) {
        if (message.role !== "assistant") continue;
        versions.set(message.id, message.updatedAt);

        if (seen.has(message.id)) continue;
        const spoken = spokenFor(message);

        if (!spoken?.speakable) continue;
        live.push({
          messageId: message.id,
          contentVersion: message.updatedAt,
          text: spoken.text,
          successful: true,
        });
      }

      controller.reconcileMessages(versions);

      if (scope === null) return;

      if (baseline === null) {
        for (const message of messages) {
          if (message.role !== "assistant") continue;
          seen.add(message.id);

          if (baseline === null || message.updatedAt > baseline) baseline = message.updatedAt;
        }

        baseline ??= "";
        sequence = seen.size;
        tracker.hydrate(sequence);

        return;
      }

      const base = identityBase();

      for (const reply of live) {
        seen.add(reply.messageId);
        sequence += 1;

        if (reply.contentVersion <= baseline) continue;
        const next = tracker.completed(scope, sequence, reply);

        if (next && base && currentSynthesis(base.environmentId).available) {
          void controller.start({
            identity: { ...base, messageId: next.messageId, contentVersion: next.contentVersion },
            text: next.text,
            automatic: true,
          });
        }
      }
    },
    dispose: () => {
      controller.dispose();
      synthesisListeners.clear();
    },
  };
}

function sameSynthesis(a: StoredReplySynthesisCapability, b: StoredReplySynthesisCapability) {
  return (
    a.available === b.available &&
    a.provider === b.provider &&
    a.voice === b.voice &&
    (a.available || b.available || a.reason === b.reason)
  );
}

export type ReplyPlaybackSession = ReturnType<typeof createReplyPlaybackSession>;