Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

11.8 KB
Raw Blame
// @ts-check
// Gemini native TTS lane: 30 prebuilt voices, controllable by prompt.
//
// Two rungs, in this order:
//   1. Vertex AI (`GOOGLE_CLOUD_PROJECT`) authenticated with the GCP service
//      account and billed to the platform's Google credits. This is the lane
//      that actually runs in production: same anchor doctrine as
//      api/_lib/vertex-gemini.js (no third-party quota, no API key to rot,
//      standing owner-approved spend (docs/ops/gcp-credits-plan.md).
//   2. The Generative Language API with `GEMINI_API_KEY`, for local dev and as
//      a backstop when the Vertex token mint fails.
//
// Gemini TTS returns raw little-endian s16 PCM at 24 kHz, so every clip is
// wrapped in a RIFF/WAVE header before it leaves this module: browsers sniff
// the container, and the content-type we report stays truthful.
//
// Unlike every other lane here, the voice is only half the character: Gemini
// takes a natural-language style instruction in the prompt itself ("Say this
// in a warm, conspiratorial whisper:"), which the Voice Lab exposes as the
// "direction" field.

import { pcmToWav } from './tts-nvidia.js';
import { getGcpAccessToken } from './gcp-auth.js';
import { env } from './env.js';

// Google's own SDKs accept either name for a Generative Language API key, and
// this workspace has historically used GOOGLE_API_KEY. Read both so a working
// key never sits in .env unused.
function geminiApiKey() {
	return env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || null;
}

// Preview TTS models. Flash is the real-time lane; Pro is the quality lane.
export const GEMINI_TTS_MODELS = [
	{
		id: 'gemini-2.5-flash-preview-tts',
		label: 'Gemini 2.5 Flash TTS',
		note: 'Fast · style-directable · 24 languages',
	},
	{
		id: 'gemini-2.5-pro-preview-tts',
		label: 'Gemini 2.5 Pro TTS',
		note: 'Highest quality · follows long directions',
	},
];
export const GEMINI_TTS_DEFAULT_MODEL = 'gemini-2.5-flash-preview-tts';
const GEMINI_MODEL_IDS = new Set(GEMINI_TTS_MODELS.map((m) => m.id));

export function isGeminiTtsModel(id) {
	return typeof id === 'string' && GEMINI_MODEL_IDS.has(id);
}

// Gemini's 30 prebuilt voices with the character Google documents for each.
// Ids are case-sensitive on the wire.
export const GEMINI_VOICES = [
	{ id: 'Zephyr', character: 'Bright' },
	{ id: 'Puck', character: 'Upbeat' },
	{ id: 'Charon', character: 'Informative' },
	{ id: 'Kore', character: 'Firm' },
	{ id: 'Fenrir', character: 'Excitable' },
	{ id: 'Leda', character: 'Youthful' },
	{ id: 'Orus', character: 'Firm' },
	{ id: 'Aoede', character: 'Breezy' },
	{ id: 'Callirrhoe', character: 'Easy-going' },
	{ id: 'Autonoe', character: 'Bright' },
	{ id: 'Enceladus', character: 'Breathy' },
	{ id: 'Iapetus', character: 'Clear' },
	{ id: 'Umbriel', character: 'Easy-going' },
	{ id: 'Algieba', character: 'Smooth' },
	{ id: 'Despina', character: 'Smooth' },
	{ id: 'Erinome', character: 'Clear' },
	{ id: 'Algenib', character: 'Gravelly' },
	{ id: 'Rasalgethi', character: 'Informative' },
	{ id: 'Laomedeia', character: 'Upbeat' },
	{ id: 'Achernar', character: 'Soft' },
	{ id: 'Alnilam', character: 'Firm' },
	{ id: 'Schedar', character: 'Even' },
	{ id: 'Gacrux', character: 'Mature' },
	{ id: 'Pulcherrima', character: 'Forward' },
	{ id: 'Achird', character: 'Friendly' },
	{ id: 'Zubenelgenubi', character: 'Casual' },
	{ id: 'Vindemiatrix', character: 'Gentle' },
	{ id: 'Sadachbia', character: 'Lively' },
	{ id: 'Sadaltager', character: 'Knowledgeable' },
	{ id: 'Sulafat', character: 'Warm' },
];
export const GEMINI_DEFAULT_VOICE = 'Kore';
const GEMINI_VOICE_IDS = new Set(GEMINI_VOICES.map((v) => v.id));

export function isGeminiVoice(id) {
	return typeof id === 'string' && GEMINI_VOICE_IDS.has(id);
}

// Gemini TTS emits 24 kHz mono s16.
const SAMPLE_RATE_HZ = 24_000;
const DEFAULT_TIMEOUT_MS = 45_000;

// The preview TTS models are not served from every Vertex region, and Google
// moves which ones carry a preview model without notice. Rather than bet the
// lane on one region, try each in turn: an explicit env pin first, then the
// two that have carried these models since launch. A region that does not host
// the model answers 404, which costs one round trip and falls through.
function vertexLocations() {
	const pinned = process.env.GOOGLE_CLOUD_LOCATION_TTS;
	const ordered = pinned ? [pinned, 'us-central1', 'global'] : ['us-central1', 'global'];
	return [...new Set(ordered)];
}

function vertexUrl(location, project, modelId) {
	const host =
		location === 'global' ? 'aiplatform.googleapis.com' : `${location}-aiplatform.googleapis.com`;
	return (
		`https://${host}/v1/projects/${project}/locations/${location}` +
		`/publishers/google/models/${modelId}:generateContent`
	);
}

/** True when either rung can serve. Read per call so late-injected env works. */
export function geminiTtsConfigured() {
	return Boolean(process.env.GOOGLE_CLOUD_PROJECT || geminiApiKey());
}

/** Which rungs are live, for the catalog endpoint's provider report. */
export function geminiTtsLanes() {
	return {
		vertex: Boolean(process.env.GOOGLE_CLOUD_PROJECT),
		apiKey: Boolean(geminiApiKey()),
	};
}

function tagged(message, code, extra = {}) {
	return Object.assign(new Error(message), { code, ...extra });
}

/**
 * Compose the request payload. `direction` is a natural-language style
 * instruction Gemini honors as part of the prompt; it is prefixed rather than
 * sent as a separate field because that is the documented control surface.
 *
 * The framing matters. A bare "<direction>: <text>" prompt occasionally reads
 * to the model as a request to ANSWER rather than to speak, and the API then
 * rejects the turn with "Model tried to generate text, but it should only be
 * used for TTS". Naming the task first ("Say the following...") makes the
 * transcript unambiguous; `synthesizeGeminiTts` still retries without the
 * direction if it happens anyway, so a clip is never lost to phrasing.
 */
function buildBody({ text, voice, direction }) {
	const style = String(direction || '').trim().replace(/[:.\s]+$/, '');
	const prompt = style ? `Say the following, ${style}: ${text}` : text;
	return {
		contents: [{ role: 'user', parts: [{ text: prompt }] }],
		generationConfig: {
			responseModalities: ['AUDIO'],
			speechConfig: {
				voiceConfig: { prebuiltVoiceConfig: { voiceName: voice } },
			},
		},
	};
}

/** Pull the base64 PCM out of a generateContent response. */
function extractAudio(data) {
	const parts = data?.candidates?.[0]?.content?.parts || [];
	for (const part of parts) {
		const inline = part?.inlineData || part?.inline_data;
		const b64 = inline?.data;
		if (typeof b64 === 'string' && b64.length) return Buffer.from(b64, 'base64');
	}
	const blocked = data?.promptFeedback?.blockReason || data?.candidates?.[0]?.finishReason;
	throw tagged(
		blocked ? `Gemini TTS returned no audio (${blocked})` : 'Gemini TTS returned no audio',
		blocked === 'SAFETY' || blocked === 'PROHIBITED_CONTENT' ? 'content_blocked' : 'provider_error',
	);
}

async function callGemini({ url, headers, body, timeoutMs }) {
	let resp;
	try {
		resp = await fetch(url, {
			method: 'POST',
			headers: { 'content-type': 'application/json', ...headers },
			body: JSON.stringify(body),
			signal: AbortSignal.timeout(timeoutMs),
		});
	} catch (e) {
		throw tagged(`Could not reach Gemini TTS: ${e?.message || 'fetch failed'}`, 'provider_unreachable');
	}
	if (!resp.ok) {
		const detail = await resp.text().catch(() => '');
		const code =
			resp.status === 429 ? 'rate_limited'
			: resp.status === 401 || resp.status === 403 ? 'invalid_key'
			: /should only be used for TTS/i.test(detail) ? 'answered_instead_of_spoke'
			: 'provider_error';
		throw tagged(`Gemini TTS returned ${resp.status}: ${detail.slice(0, 300)}`, code, {
			status: resp.status,
		});
	}
	return extractAudio(await resp.json());
}

/**
 * Synthesize one clip. Resolves with the COMPLETE audio buffer so a caller can
 * fail over to another lane before a byte reaches the client.
 *
 * @param {{ text:string, voice?:string, model?:string, direction?:string, timeoutMs?:number }} opts
 * @returns {Promise<{ audio:Buffer, contentType:string, format:string, voiceName:string, model:string, lane:string, sampleRateHz:number }>}
 * @throws {Error & { code:string }} not_configured | invalid_key | rate_limited |
 *         content_blocked | provider_unreachable | provider_error
 */
export async function synthesizeGeminiTts({
	text,
	voice = GEMINI_DEFAULT_VOICE,
	model = GEMINI_TTS_DEFAULT_MODEL,
	direction = '',
	timeoutMs = DEFAULT_TIMEOUT_MS,
} = {}) {
	if (!text || !String(text).trim()) throw tagged('text is required', 'invalid_argument');
	const voiceName = isGeminiVoice(voice) ? voice : GEMINI_DEFAULT_VOICE;
	const modelId = isGeminiTtsModel(model) ? model : GEMINI_TTS_DEFAULT_MODEL;
	const styled = buildBody({ text: String(text), voice: voiceName, direction: String(direction || '') });
	// The fallback prompt drops the style direction entirely. It is only used
	// when the styled prompt made the model answer instead of speak, so a
	// user's clip degrades to plain narration rather than to an error.
	const plain = direction ? buildBody({ text: String(text), voice: voiceName }) : null;

	/** One rung's request, retried once without the direction if Gemini answers. */
	async function speak({ url, headers, timeoutMs: ms }) {
		try {
			return await callGemini({ url, headers, body: styled, timeoutMs: ms });
		} catch (e) {
			if (e?.code !== 'answered_instead_of_spoke' || !plain) throw e;
			return callGemini({ url, headers, body: plain, timeoutMs: ms });
		}
	}

	const laneErrors = [];
	let pcm = null;
	let lane = '';

	// ── Rung 1: Vertex AI on platform credits ────────────────────────────────
	if (process.env.GOOGLE_CLOUD_PROJECT) {
		const project = process.env.GOOGLE_CLOUD_PROJECT;
		let token = null;
		try {
			token = await getGcpAccessToken();
		} catch (e) {
			laneErrors.push({ code: 'no_credentials', text: `vertex: no_credentials ${e?.message || ''}`.trim() });
		}
		for (const location of token ? vertexLocations() : []) {
			try {
				pcm = await speak({
					url: vertexUrl(location, project, modelId),
					headers: { authorization: `Bearer ${token}` },
					timeoutMs,
				});
				lane = `vertex:${location}`;
				break;
			} catch (e) {
				laneErrors.push({
					code: e?.code || 'error',
					text: `vertex/${location}: ${e?.code || 'error'} ${e?.message || ''}`.trim(),
				});
				// An auth or quota failure will repeat in every region; only a
				// missing-model 404 is worth retrying elsewhere.
				if (e?.code === 'invalid_key' || e?.code === 'rate_limited') break;
			}
		}
	}

	// ── Rung 2: Generative Language API key ──────────────────────────────────
	const apiKey = geminiApiKey();
	if (!pcm && apiKey) {
		try {
			pcm = await speak({
				url: `https://generativelanguage.googleapis.com/v1beta/models/${modelId}:generateContent`,
				headers: { 'x-goog-api-key': apiKey },
				timeoutMs,
			});
			lane = 'api-key';
		} catch (e) {
			laneErrors.push({
				code: e?.code || 'error',
				text: `api-key: ${e?.code || 'error'} ${e?.message || ''}`.trim(),
			});
		}
	}

	if (!pcm) {
		if (!laneErrors.length) throw tagged('Gemini TTS is not configured', 'not_configured');
		// When every rung failed the same way the cause is the request, not the
		// infrastructure: keep that code so the caller can answer 422 (blocked
		// prompt) or 429 (quota) instead of a blanket 502.
		const codes = new Set(laneErrors.map((e) => e.code));
		const code = codes.size === 1 ? [...codes][0] : 'provider_error';
		throw tagged(
			`Gemini TTS failed: ${laneErrors.map((e) => e.text).join('; ')}`,
			code === 'no_credentials' || code === 'error' ? 'provider_error' : code,
		);
	}

	return {
		audio: pcmToWav(pcm, { sampleRateHz: SAMPLE_RATE_HZ }),
		contentType: 'audio/wav',
		format: 'wav',
		voiceName,
		model: modelId,
		lane,
		sampleRateHz: SAMPLE_RATE_HZ,
	};
}