Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

13.8 KB
Raw Blame
/**
 * POST /api/tts/synthesize
 *
 * One endpoint that renders text on any voice lane the platform exposes:
 * Microsoft Edge, Gemini, NVIDIA Magpie, OpenAI and ElevenLabs. Callers pass
 * the `provider` + `voiceId` pair they got from /api/tts/catalog; everything
 * else (protocol, container, billing) is handled here.
 *
 * Body: {
 *   provider: 'edge'|'gemini'|'nvidia'|'openai'|'elevenlabs',
 *   voiceId: string,
 *   text: string,                 // <= 1000 chars
 *   model?: string,               // lane-specific; validated per lane
 *   direction?: string,           // style instruction (gemini, gpt-4o-mini-tts)
 *   speed?: number,               // 0.5 .. 2.0
 *   format?: string,              // openai/nvidia only; other lanes fix theirs
 *   language?: string,            // nvidia
 *   voice_settings?: object,      // elevenlabs
 * }
 *
 * Billing (`x-tts-billing` reports which rung served):
 *   free    : Edge / NVIDIA: no vendor invoice, nothing charged.
 *   gcp     : Gemini on Vertex: platform Google credits, nothing charged.
 *   byok    : the request carried `x-eleven-key`: the user's ElevenLabs account pays.
 *   credits : a vendor-billed lane on the platform key: metered to the caller's
 *             prepaid credit wallet before the upstream call, refunded if the
 *             clip never renders. 402 + top_up_url when short.
 *   cached  : R2 cache hit: no upstream call, nothing charged.
 *
 * Response: audio in the lane's container (content-type is always truthful).
 */

import { randomUUID } from 'node:crypto';
import { getSessionUser, authenticateBearer, extractBearer } from '../_lib/auth.js';
import { cors, method, wrap, error, json, readJson, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { sha256 } from '../_lib/crypto.js';
import { headObject, getObjectBuffer, putObject } from '../_lib/r2.js';
import { chargeCreditsForAction, refundCredits } from '../_lib/credits.js';
import { resolveElevenKey } from '../_lib/elevenlabs.js';
import {
	getProvider,
	providerAvailability,
	isKnownVoice,
	synthesizeVoice,
	usdForSynthesis,
	creditActionFor,
} from '../_lib/voice-providers.js';
import {
	laneOutages,
	laneOutageCopy,
	noteLaneFailure,
	noteLaneHealthy,
} from '../_lib/tts-lane-health.js';

export const maxDuration = 60;

const MAX_TEXT = 1000;
const MAX_DIRECTION = 500;

// Upstream error code → HTTP status, so a client can tell "your fault" from
// "our fault" without parsing prose.
const STATUS_BY_CODE = {
	invalid_argument: 400,
	content_blocked: 422,
	not_configured: 503,
	invalid_key: 502,
	rate_limited: 429,
	provider_unreachable: 502,
	provider_error: 502,
	// Gemini answered the prompt instead of speaking it, and there was no style
	// direction left to drop on the retry. That is an upstream misread, not a
	// caller error.
	answered_instead_of_spoke: 502,
};

/**
 * One sentence a picker can render verbatim. Upstream failures arrive as raw
 * vendor prose (a 403 body quoting project numbers, a stack of per-rung
 * attempts); pasting that into the UI tells a visitor nothing they can act on,
 * so each code gets a plain answer and, where one exists, a way forward.
 */
function userFacingLaneError(provider, err) {
	switch (err?.code) {
		// Our own validation prose ("text is required"), already written for a
		// caller and specific in a way a generic sentence would throw away.
		case 'invalid_argument':
			return err.message;
		case 'invalid_key':
		case 'not_configured':
			return (
				`${provider.label} is temporarily unavailable: the provider rejected this ` +
				"deployment's credentials. Microsoft Edge and NVIDIA Magpie are free and unaffected."
			);
		case 'rate_limited':
			return `${provider.label} is rate limiting right now. Wait a moment or pick another lane.`;
		case 'provider_unreachable':
			return `${provider.label} could not be reached. Try again, or pick another lane.`;
		case 'content_blocked':
			return `${provider.label} refused this text. Rephrase it and try again.`;
		case 'answered_instead_of_spoke':
			return `${provider.label} answered the text instead of speaking it. Simplify the direction and try again.`;
		default:
			return `${provider.label} synthesis failed. Try again, or pick another lane.`;
	}
}

export default wrap(async (req, res) => {
	if (cors(req, res, { methods: 'POST,OPTIONS', credentials: true })) return;
	if (!method(req, res, ['POST'])) return;

	let body;
	try {
		body = await readJson(req, 50_000);
	} catch (e) {
		return error(res, e.status || 400, 'bad_request', e.message);
	}

	const providerId = String(body.provider || 'edge').trim();
	const provider = getProvider(providerId);
	if (!provider) return error(res, 400, 'validation_error', `unknown provider "${providerId}"`);

	const text = String(body.text || '').trim();
	if (!text) return error(res, 400, 'validation_error', 'text is required');
	if (text.length > MAX_TEXT)
		return error(res, 400, 'validation_error', `text exceeds ${MAX_TEXT} characters`);

	const direction = String(body.direction || '').trim().slice(0, MAX_DIRECTION);

	// ── Auth + rate limit ─────────────────────────────────────────────────────
	// The keyless lanes serve anonymous visitors (that is the point of a free
	// lane); the vendor-billed ones need an identity to meter against.
	const session = await getSessionUser(req);
	const bearer = session ? null : await authenticateBearer(extractBearer(req));
	const userId = session?.id ?? bearer?.userId ?? null;

	if (!provider.anonymous && !userId)
		return error(res, 401, 'unauthorized', `sign in to use the ${provider.label} voices`);

	if (userId) {
		const rl = await limits.ttsSpeakUser(String(userId));
		if (!rl.success) return rateLimited(res, rl, 'TTS rate limit exceeded, try again later');
	} else {
		const rl = await limits.ttsSpeakIp(clientIp(req));
		if (!rl.success)
			return rateLimited(res, rl, 'TTS rate limit exceeded, sign in for a higher limit');
	}

	const { apiKey: elevenKey, byok } = resolveElevenKey(req);
	const availability = providerAvailability({ elevenUserKey: byok });
	if (!availability[providerId]) {
		return error(
			res,
			503,
			'not_configured',
			providerId === 'elevenlabs'
				? 'ElevenLabs is not configured on this server. Send your own key in the x-eleven-key header.'
				: `${provider.label} is not configured on this server`,
		);
	}

	// A lane can be configured and still be refusing every call (an expired key,
	// a billing hold upstream). The breaker remembers what the last real call
	// learned, so a lane that just refused credentials answers here in
	// milliseconds with the reason instead of burning a round trip per visitor.
	// A user's own ElevenLabs key is a separate credential and is never gated by
	// the platform key's outage. A 'health' cooldown (throttle, blip) still lets
	// an explicit request through, so a recovered lane proves itself immediately.
	const laneScopedToPlatform = !(byok && providerId === 'elevenlabs');
	let laneOutage = null;
	if (laneScopedToPlatform) {
		laneOutage = (await laneOutages([providerId])).get(providerId) || null;
		if (laneOutage === 'auth') {
			return error(
				res,
				503,
				'lane_unavailable',
				`${provider.label} is temporarily unavailable: ${laneOutageCopy(laneOutage)}. ` +
					'Microsoft Edge and NVIDIA Magpie are free and unaffected.',
				{ provider: providerId, retry_with: ['edge', 'nvidia'] },
			);
		}
	}

	const voiceId = String(body.voiceId || '').trim();
	const model = String(body.model || '').trim();
	const speed = Number(body.speed);
	const format = typeof body.format === 'string' ? body.format : undefined;
	const language = typeof body.language === 'string' ? body.language : undefined;
	const voiceSettings =
		body.voice_settings && typeof body.voice_settings === 'object' ? body.voice_settings : null;

	// A voiceId the lane does not have is a caller error, and it has to be caught
	// here: the router would otherwise swap in the lane default and return 200,
	// so a typo came back as a different voice, charged, with nothing in the
	// response saying so. Checked before the cache key is minted and before any
	// metering, so a bad id costs nothing. An omitted voiceId still means "lane
	// default" and passes.
	if (!(await isKnownVoice(providerId, voiceId))) {
		return error(
			res,
			400,
			'validation_error',
			`"${voiceId}" is not a ${provider.label} voice. Pick one from /api/tts/catalog?provider=${providerId}`,
		);
	}

	// ── R2 cache ──────────────────────────────────────────────────────────────
	// Checked BEFORE metering: a cached clip costs nothing upstream, so it is
	// never charged. BYOK clips are keyed separately: they render on a
	// different account and must not be served from (or into) the shared cache.
	const cacheHash = await sha256(
		[
			providerId,
			voiceId,
			text,
			model,
			direction,
			Number.isFinite(speed) ? speed : '',
			format || '',
			language || '',
			voiceSettings ? JSON.stringify(voiceSettings) : '',
			byok ? 'byok' : '',
		].join('\x00'),
	);
	const cacheKey = `tts/synth/${providerId}/${cacheHash}`;

	const cached = await headObject(cacheKey).catch(() => null);
	if (cached) {
		try {
			const buf = await getObjectBuffer(cacheKey);
			// HeadObjectCommand output is the raw S3 shape (PascalCase).
			res.setHeader('content-type', cached.ContentType || 'audio/mpeg');
			res.setHeader('content-length', String(buf.length));
			res.setHeader('x-tts-provider', providerId);
			res.setHeader('x-tts-cache', 'hit');
			res.setHeader('x-tts-billing', 'cached');
			res.setHeader('cache-control', 'private, max-age=86400');
			return res.end(buf);
		} catch {
			// Cache read failed, synthesize fresh.
		}
	}

	// ── Metering ──────────────────────────────────────────────────────────────
	const action = creditActionFor(providerId);
	const usd = usdForSynthesis({ provider: providerId, chars: text.length, byok });
	const billing =
		byok && providerId === 'elevenlabs' ? 'byok'
		: provider.billing === 'gcp' ? 'gcp'
		: usd > 0 ? 'credits'
		: 'free';

	let creditCharge = null;
	if (billing === 'credits' && action) {
		const idempotencyKey = `tts:${providerId}:${randomUUID()}`;
		try {
			const charged = await chargeCreditsForAction({
				user: session || { id: userId },
				action,
				usd,
				refType: 'tts',
				refId: cacheHash,
				idempotencyKey,
				meta: { chars: text.length, provider: providerId, voiceId, model },
			});
			creditCharge = { idempotencyKey, chargedUsd: charged.chargedUsd };
		} catch (err) {
			if (err?.code === 'insufficient_credits') {
				return json(res, 402, {
					error: 'insufficient_credits',
					feature: action,
					available_usd: err.available_usd,
					required_usd: err.required_usd,
					top_up_url: '/credits',
					message:
						`${provider.label} synthesis is metered to your credit balance. ` +
						'Top up with $THREE at /credits, switch to a free voice lane, ' +
						(provider.byok ? 'or bring your own key.' : 'or pick a free provider.'),
				});
			}
			throw err;
		}
	}

	const refundMetering = async () => {
		if (creditCharge?.chargedUsd > 0) {
			await refundCredits({
				userId,
				amountUsd: creditCharge.chargedUsd,
				action,
				refType: 'tts',
				refId: cacheHash,
				idempotencyKey: `${creditCharge.idempotencyKey}:refund`,
				meta: { reason: 'synthesis_failed' },
			}).catch((e) => console.warn('[tts/synthesize] credit refund failed:', e?.message || e));
		}
	};

	// ── Synthesize ────────────────────────────────────────────────────────────
	let out;
	try {
		out = await synthesizeVoice({
			provider: providerId,
			text,
			voiceId,
			model,
			direction,
			speed,
			format,
			language,
			voiceSettings,
			elevenKey,
		});
	} catch (err) {
		console.error('[tts/synthesize] lane failed', providerId, err?.code, err?.message);
		await refundMetering();
		if (laneScopedToPlatform) await noteLaneFailure(providerId, err?.code);
		return error(
			res,
			STATUS_BY_CODE[err?.code] || 502,
			err?.code || 'upstream_error',
			userFacingLaneError(provider, err),
			// The upstream body is often a multi-line JSON blob naming projects and
			// quota rules. It belongs in a debugger, not in a status line, so it
			// rides in its own field and the prose above stays readable.
			{ provider: providerId, detail: String(err?.message || '').slice(0, 300) },
		);
	}
	// Only a lane that was cooling has anything to clear, so the common path
	// (a healthy lane serving a clip) pays no cache round trip for the breaker.
	if (laneOutage) await noteLaneHealthy(providerId);

	if (!out.audio?.length) {
		await refundMetering();
		return error(res, 502, 'upstream_error', `${provider.label} returned empty audio`);
	}

	putObject({
		key: cacheKey,
		body: out.audio,
		contentType: out.contentType,
		metadata: { 'created-at': new Date().toISOString() },
	}).catch((e) => console.warn('[tts/synthesize] R2 cache write failed:', e.message));

	res.statusCode = 200;
	res.setHeader('content-type', out.contentType);
	res.setHeader('content-length', String(out.audio.length));
	res.setHeader('x-tts-provider', providerId);
	res.setHeader('x-tts-voice', out.voiceId);
	res.setHeader('x-tts-model', out.model);
	res.setHeader('x-tts-format', out.format);
	res.setHeader('x-tts-cache', 'miss');
	res.setHeader('x-tts-billing', billing);
	if (out.lane) res.setHeader('x-tts-lane', out.lane);
	if (creditCharge?.chargedUsd > 0)
		res.setHeader('x-tts-charged-usd', creditCharge.chargedUsd.toFixed(6));
	res.setHeader('cache-control', 'private, max-age=86400');
	return res.end(out.audio);
});