Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

8.3 KB
Raw Blame
/**
 * Text to speech, with the speech service the owner already set up (the
 * same one voice typing uses, see pickSpeechEngine): OpenAI
 * (gpt-4o-mini-tts, falling back to tts-1), Gemini (Flash TTS), Deepgram
 * (Aura 2) or ElevenLabs (Flash v2.5). MP3, or WAV from Gemini (speechMime
 * says which); every client plays both. Used to read Mercury's replies aloud
 * in the apps.
 */
import type { SpeechEngine } from './transcribe.js';
import { geminiStyle } from '../media/spoken-text.js';

/** Longest text spoken in one request; clients send a long reply in parts. */
export const SPEAK_MAX_CHARS = 4000;
/** Deepgram's own limit per request. */
const DEEPGRAM_MAX_CHARS = 2000;

const OPENAI_VOICE = 'coral';
const DEEPGRAM_MODEL = 'aura-2-thalia-en';
const ELEVENLABS_VOICE = 'JBFqnCBsd6RMkjVDRZzb';
const ELEVENLABS_MODEL = 'eleven_flash_v2_5';
const GEMINI_VOICE = 'Kore';
/** Speaking models, newest first; the next is tried when one isn't offered or is busy. */
const GEMINI_SPEAK_MODELS = ['gemini-3.8-flash-tts', 'gemini-2.5-flash-preview-tts'];

/** Voices each service offers by name (ElevenLabs uses voice ids; Deepgram, Aura model names). */
export const SPEECH_VOICES: Partial<Record<SpeechEngine['kind'], string[]>> = {
  openai: ['alloy', 'ash', 'ballad', 'coral', 'echo', 'fable', 'nova', 'onyx', 'sage', 'shimmer', 'verse', 'marin', 'cedar'],
  gemini: ['Kore', 'Puck', 'Charon', 'Fenrir', 'Aoede', 'Leda', 'Orus', 'Zephyr'],
  deepgram: ['aura-2-thalia-en', 'aura-2-andromeda-en', 'aura-2-apollo-en', 'aura-2-arcas-en', 'aura-2-helena-en', 'aura-2-orion-en'],
};

/**
 * mp3 plays everywhere; "ogg" is Ogg/Opus, what Telegram and WhatsApp play
 * as a voice note. OpenAI and Deepgram make it themselves; Gemini and
 * ElevenLabs output is converted with ffmpeg when it is installed (else it
 * stays as it was, and goes as an audio file).
 */
export type SpeechFormat = 'mp3' | 'ogg';

/** The audio type synthesizeSpeech returns for an engine (and format). */
export function speechMime(engine: SpeechEngine, format: SpeechFormat = 'mp3'): string {
  if (format === 'ogg' && (engine.kind === 'openai' || engine.kind === 'deepgram')) return 'audio/ogg';
  return engine.kind === 'gemini' ? 'audio/wav' : 'audio/mpeg';
}

type Fetch = typeof fetch;

export interface SpeakOptions {
  fetch?: Fetch;
  signal?: AbortSignal;
  /** A voice the service knows (SPEECH_VOICES; an ElevenLabs voice id). */
  voice?: string;
  /** How to say it ("calm and warm", "excited"): OpenAI gpt-4o-mini-tts and Gemini follow it. */
  instructions?: string;
  format?: SpeechFormat;
}

/** Speak `text` with `engine`; resolves with audio bytes of type speechMime(engine). */
export async function synthesizeSpeech(text: string, engine: SpeechEngine, opts: SpeakOptions = {}): Promise<Uint8Array> {
  const input = text.trim();
  if (!input) throw new Error('nothing to say');
  const f = opts.fetch ?? fetch;
  const signal = opts.signal;
  const ogg = opts.format === 'ogg';

  if (engine.kind === 'deepgram') {
    const model = opts.voice && /^aura-/i.test(opts.voice) ? opts.voice : DEEPGRAM_MODEL;
    const encoding = ogg ? 'encoding=opus&container=ogg' : 'encoding=mp3';
    const res = await f(`https://api.deepgram.com/v1/speak?model=${encodeURIComponent(model)}&${encoding}`, {
      method: 'POST',
      headers: { Authorization: `Token ${engine.apiKey}`, 'Content-Type': 'application/json' },
      body: JSON.stringify({ text: input.slice(0, DEEPGRAM_MAX_CHARS) }),
      signal,
    });
    return audioOf(res, 'Deepgram');
  }

  if (engine.kind === 'gemini') {
    let last: Response | null = null;
    for (const model of GEMINI_SPEAK_MODELS) {
      const res = await f(`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`, {
        method: 'POST',
        headers: { 'x-goog-api-key': engine.apiKey, 'Content-Type': 'application/json' },
        body: JSON.stringify({
          // Gemini takes the style as words in front of the text, and reads out anything
          // that isn't a short description of the voice (geminiStyle).
          contents: [{ parts: [{ text: (geminiStyle(opts.instructions) ? `Say ${geminiStyle(opts.instructions)}: ` : '') + input.slice(0, SPEAK_MAX_CHARS) }] }],
          generationConfig: { responseModalities: ['AUDIO'], speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: opts.voice || GEMINI_VOICE } } } },
        }),
        signal,
      });
      if (res.ok) {
        const body = (await res.json().catch(() => ({}))) as { candidates?: Array<{ content?: { parts?: Array<{ inlineData?: { mimeType?: string; data?: string } }> } }> };
        const part = body.candidates?.[0]?.content?.parts?.find((p) => p.inlineData?.data)?.inlineData;
        if (!part?.data) throw new Error('Gemini sent no audio');
        return asWav(Buffer.from(part.data, 'base64'), part.mimeType ?? '');
      }
      last = res;
      if (res.status !== 404 && res.status < 500) break;
    }
    return audioOf(last!, 'Gemini');
  }

  if (engine.kind === 'elevenlabs') {
    const res = await f(`https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(opts.voice || ELEVENLABS_VOICE)}?output_format=mp3_44100_128`, {
      method: 'POST',
      headers: { 'xi-api-key': engine.apiKey, 'Content-Type': 'application/json' },
      body: JSON.stringify({ text: input.slice(0, SPEAK_MAX_CHARS), model_id: ELEVENLABS_MODEL }),
      signal,
    });
    return audioOf(res, 'ElevenLabs');
  }

  const base = (engine.baseUrl || 'https://api.openai.com/v1').replace(/\/+$/, '');
  const say = (model: string) => f(`${base}/audio/speech`, {
    method: 'POST',
    headers: { ...(engine.headers ?? { Authorization: `Bearer ${engine.apiKey}` }), 'Content-Type': 'application/json' },
    body: JSON.stringify({
      model,
      voice: opts.voice || OPENAI_VOICE,
      input: input.slice(0, SPEAK_MAX_CHARS),
      // "opus" is Ogg/Opus: a voice note as it is.
      response_format: ogg ? 'opus' : 'mp3',
      // Only the instructable models take a style; tts-1 rejects the field.
      ...(opts.instructions && !/^tts-1/i.test(model) ? { instructions: opts.instructions } : {}),
    }),
    signal,
  });
  // A chosen model is used as is; otherwise the newer one, then the classic one
  // on an OpenAI-compatible endpoint that doesn't have it.
  if (engine.model) return audioOf(await say(engine.model), engine.label);
  let res = await say('gpt-4o-mini-tts');
  if (res.status === 400 || res.status === 404) res = await say('tts-1');
  return audioOf(res, engine.label || 'OpenAI');
}

/**
 * Gemini sends a WAV file, or raw 16-bit PCM ("audio/L16;rate=24000"):
 * the latter gets a WAV header so every player takes it.
 */
export function asWav(bytes: Uint8Array, mime: string): Uint8Array {
  if (bytes.byteLength >= 12 && Buffer.from(bytes.subarray(0, 4)).toString('latin1') === 'RIFF') return bytes;
  const rate = Number(/rate=(\d+)/i.exec(mime)?.[1] ?? 24000);
  const header = Buffer.alloc(44);
  header.write('RIFF', 0, 'latin1');
  header.writeUInt32LE(36 + bytes.byteLength, 4);
  header.write('WAVE', 8, 'latin1');
  header.write('fmt ', 12, 'latin1');
  header.writeUInt32LE(16, 16);
  header.writeUInt16LE(1, 20); // PCM
  header.writeUInt16LE(1, 22); // mono
  header.writeUInt32LE(rate, 24);
  header.writeUInt32LE(rate * 2, 28);
  header.writeUInt16LE(2, 32);
  header.writeUInt16LE(16, 34);
  header.write('data', 36, 'latin1');
  header.writeUInt32LE(bytes.byteLength, 40);
  return new Uint8Array(Buffer.concat([header, Buffer.from(bytes)]));
}

async function audioOf(res: Response, vendor: string): Promise<Uint8Array> {
  if (!res.ok) {
    let detail = '';
    try {
      const body = (await res.json()) as { error?: { message?: string } | string; detail?: { message?: string } | string; err_msg?: string };
      const e = body.error ?? body.detail ?? body.err_msg;
      detail = typeof e === 'string' ? e : e?.message ?? '';
    } catch { /* not JSON */ }
    if (res.status === 401 || res.status === 403) throw new Error(`${vendor} rejected the key. Check it in Settings.`);
    if (res.status === 429) throw new Error(`${vendor} is rate-limiting right now. Try again in a moment.`);
    throw new Error(detail ? `${vendor}: ${detail.slice(0, 200)}` : `${vendor} answered ${res.status}`);
  }
  const bytes = new Uint8Array(await res.arrayBuffer());
  if (bytes.byteLength === 0) throw new Error(`${vendor} sent no audio`);
  return bytes;
}