Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

15.1 KB
Raw Blame
/**
 * POST /api/tts/eleven
 *
 * Server proxy for ElevenLabs TTS. Keeps the API key server-side. The clip is
 * streamed straight to the client (low TTFB) and, on a clean finish, cached in
 * R2 for 30 days keyed by sha256(voiceId + text + modelId + voice_settings).
 *
 * Billing (header `x-tts-billing` reports which rung served the call). There
 * is no free platform lane: every clip the platform pays ElevenLabs for is
 * paid for by the caller (owner policy 2026-08-06).
 *   byok        - request carried an `x-eleven-key` header: the user's own
 *                 ElevenLabs account pays, so credits never apply.
 *   agent_byok  - `agentId` names an agent whose voice is bound to its owner's
 *                 saved ElevenLabs key: that owner's account pays. This is what
 *                 lets an agent's cloned voice speak to visitors on the chat and
 *                 embed surfaces, not just to the person who cloned it.
 *   credits     - platform key: the request is metered against the user's prepaid
 *                 credit wallet (top up with $THREE or SOL at /credits) at
 *                 TTS_ELEVEN_USD_PER_1K, with a 402 + top_up_url when short.
 *   cached      - R2 cache hit: no upstream call, nothing charged.
 *
 * Body: {
 *   voiceId: string,
 *   text: string,
 *   agentId?: string,                        // speak as this agent's bound voice
 *   modelId?: string,                        // default eleven_flash_v2_5
 *   voice_settings?: {                        // canonical ElevenLabs shape, honored verbatim
 *     stability?, similarity_boost?, style?,  // clamped to 0..1
 *     use_speaker_boost?                       // boolean
 *   }
 *   withTimestamps?: boolean                  // see below
 * }
 * Response: audio/mpeg (chunked), or when `withTimestamps` is true:
 *   application/json { audio_base64, alignment, normalized_alignment }, character-level
 *   timing from ElevenLabs' `/with-timestamps` endpoint, in whatever language/script the
 *   text was written in. The client (src/runtime/speech.js) builds a real per-character
 *   viseme sequence from `alignment` instead of estimating visemes from audio energy.
 *   Same billing and cache lanes as the audio path; the only difference is upstream URL,
 *   accept header, and what gets cached (the JSON body instead of raw MP3 bytes).
 *
 * Auth: a session or bearer token, except on the agent_byok lane — an agent whose
 * owner bound their own key speaks to anonymous visitors too (rate limited per IP),
 * because that is the whole point of binding a voice to a public agent.
 */

import { randomUUID } from 'node:crypto';
import { getSessionUser, authenticateBearer, extractBearer } from '../_lib/auth.js';
import { cors, method, wrap, error, json, readJson, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { sql } from '../_lib/db.js';
import { resolveAgentElevenKey, normalizeKeySource } from '../_lib/agent-voice-key.js';
import { sha256 } from '../_lib/crypto.js';
import { headObject, getObjectBuffer, putObject } from '../_lib/r2.js';
import { chargeCreditsForAction, refundCredits } from '../_lib/credits.js';
import { TTS_ELEVEN_USD_PER_1K } from '../_lib/pricing/catalog.js';
import {
	ELEVEN_BASE,
	DEFAULT_TTS_MODEL,
	resolveElevenKey,
	normalizeVoiceSettings,
} from '../_lib/elevenlabs.js';

// Fire-and-forget R2 cache write. A miss on failure is acceptable. Caches
// exactly the bytes served to the client, so a cache hit is byte-identical
// whether that's raw MP3 or the with-timestamps JSON envelope.
function cacheAudio(key, buffer, contentType) {
	if (!buffer.length) return;
	putObject({
		key,
		body: buffer,
		contentType,
		metadata: { 'created-at': new Date().toISOString() },
	}).catch((e) => console.warn('[tts/eleven] R2 cache write failed:', e.message));
}

export default wrap(async (req, res) => {
	if (cors(req, res, { methods: 'POST,OPTIONS', credentials: true })) return;
	if (!method(req, res, ['POST'])) return;

	const body = await readJson(req);
	const voiceId = String(body.voiceId || '').trim();
	const text = String(body.text || '').trim();
	const modelId = String(body.modelId || DEFAULT_TTS_MODEL).trim();
	const agentId = String(body.agentId || '').trim();
	const withTimestamps = body.withTimestamps === true;

	if (!voiceId) return error(res, 400, 'validation_error', 'voiceId is required');
	if (!text) return error(res, 400, 'validation_error', 'text is required');
	if (text.length > 500)
		return error(res, 400, 'validation_error', 'text exceeds 500 chars per request');

	const session = await getSessionUser(req);
	const bearer = session ? null : await authenticateBearer(extractBearer(req));
	const userId = session?.id ?? bearer?.userId ?? null;

	// ── Agent BYOK lane ───────────────────────────────────────────────────────
	// `agentId` asks to speak as that agent's bound voice. When the binding lives
	// on the owner's own ElevenLabs key, that key serves the clip and the owner's
	// ElevenLabs account is billed, so anyone can hear the agent: the chat page, a
	// cross-origin embed, a signed-out visitor. The requested voiceId must be the
	// agent's own bound voice, so the owner's credential can never be borrowed to
	// synthesize something else.
	let agentKey = null;
	if (agentId) {
		const [agentRow] = await sql`
			SELECT user_id, voice_id, voice_key_source
			FROM agent_identities
			WHERE id = ${agentId} AND deleted_at IS NULL
		`;
		if (agentRow?.voice_id && agentRow.voice_id === voiceId) {
			const pin = normalizeKeySource(agentRow.voice_key_source);
			if (pin === 'owner') {
				const resolved = await resolveAgentElevenKey({ ownerId: agentRow.user_id, pin });
				if (resolved.apiKey) {
					const ipRl = await limits.ttsAgentVoiceIp(`${agentId}:${clientIp(req)}`);
					if (!ipRl.success) return rateLimited(res, ipRl);
					agentKey = resolved.apiKey;
				}
			}
		}
	}

	const requestKey = resolveElevenKey(req);
	// Precedence: an explicit per-request key, then the agent's own bound
	// credential, then the platform key.
	const apiKey = requestKey.byok ? requestKey.apiKey : agentKey || requestKey.apiKey;
	if (!apiKey)
		return error(
			res,
			503,
			'not_configured',
			'ElevenLabs is not configured on this server. Send your own key in the x-eleven-key header to use your account.',
		);

	// Only the agent lane serves anonymous callers; the platform key never does,
	// because every platform clip has to be charged to somebody.
	const servedByAgentKey = !requestKey.byok && !!agentKey;
	if (!servedByAgentKey && !session && !bearer)
		return error(res, 401, 'unauthorized', 'sign in required');

	if (userId) {
		const rl = await limits.ttsSpeakUser(userId);
		if (!rl.success) return rateLimited(res, rl);
	}

	// Honor the canonical ElevenLabs `voice_settings` object the client sends
	// (the ElevenLabsTTS client maps `rate` → `style` and forwards it here);
	// fall back to ElevenLabs' recommended defaults. Folded into the cache key so
	// distinct settings never collide on a shared clip.
	const settings = normalizeVoiceSettings(
		body.voice_settings && typeof body.voice_settings === 'object' ? body.voice_settings : {},
	);

	// ── R2 cache lookup ───────────────────────────────────────────────────────
	// Checked BEFORE any metering: a cached clip costs nothing upstream, so it
	// never consumes the free budget and never charges credits.
	const cacheHash = await sha256(
		`${voiceId}\x00${text}\x00${modelId}\x00${settings.stability}\x00${settings.similarity_boost}\x00${settings.style}\x00${settings.use_speaker_boost}\x00${withTimestamps ? 'ts' : 'audio'}`,
	);
	const responseContentType = withTimestamps ? 'application/json' : 'audio/mpeg';
	const cacheKey = `tts/cache/${cacheHash}.${withTimestamps ? 'json' : 'mp3'}`;

	const cached = await headObject(cacheKey).catch(() => null);
	if (cached) {
		try {
			const buf = await getObjectBuffer(cacheKey);
			res.setHeader('content-type', responseContentType);
			res.setHeader('content-length', String(buf.length));
			res.setHeader('x-tts-cache', 'hit');
			res.setHeader('x-tts-billing', 'cached');
			res.setHeader('cache-control', 'private, max-age=86400');
			return res.end(buf);
		} catch {
			// Cache read failed: fall through to synthesize fresh.
		}
	}

	// ── Metering: no free platform lane ───────────────────────────────────────
	// Both BYOK lanes run on somebody's own ElevenLabs account (the caller's, or
	// the agent owner's), so credits never apply to them. Every platform-key
	// synthesis is charged to the user's credit wallet (topped up with $THREE or
	// SOL) BEFORE the upstream call, and refunded if the clip is never delivered.
	const billing = requestKey.byok ? 'byok' : servedByAgentKey ? 'agent_byok' : 'credits';
	let creditCharge = null;

	if (billing === 'credits') {
		const idempotencyKey = `tts:eleven:${randomUUID()}`;
		const usd = Math.max(0.0001, (text.length / 1000) * TTS_ELEVEN_USD_PER_1K);
		try {
			const charged = await chargeCreditsForAction({
				user: session || { id: userId },
				action: 'tts.eleven',
				usd,
				refType: 'tts',
				refId: cacheHash,
				idempotencyKey,
				meta: { chars: text.length, voiceId, modelId },
			});
			creditCharge = { idempotencyKey, chargedUsd: charged.chargedUsd };
		} catch (err) {
			if (err?.code === 'insufficient_credits') {
				return json(res, 402, {
					error: 'insufficient_credits',
					feature: 'tts.eleven',
					available_usd: err.available_usd,
					required_usd: err.required_usd,
					top_up_url: '/credits',
					message:
						'Speech synthesis is metered to your credit balance. ' +
						'Top up with $THREE at /credits, or bring your own ElevenLabs key.',
				});
			}
			throw err;
		}
	}

	// Refund the charge when synthesis ultimately fails, so a user is never
	// billed for a clip they never received.
	const refundMetering = async () => {
		if (creditCharge?.chargedUsd > 0) {
			await refundCredits({
				userId,
				amountUsd: creditCharge.chargedUsd,
				action: 'tts.eleven',
				refType: 'tts',
				refId: cacheHash,
				idempotencyKey: `${creditCharge.idempotencyKey}:refund`,
				meta: { reason: 'synthesis_failed' },
			}).catch((e) => console.warn('[tts/eleven] credit refund failed:', e?.message || e));
		}
	};

	// ── Synthesize via ElevenLabs ─────────────────────────────────────────────
	// The with-timestamps endpoint returns one JSON object (audio + alignment),
	// not a streamable MP3, so it has no /stream segment and takes a json accept.
	const upstreamPath = withTimestamps
		? `/text-to-speech/${encodeURIComponent(voiceId)}/with-timestamps`
		: `/text-to-speech/${encodeURIComponent(voiceId)}/stream`;
	let elResp;
	try {
		elResp = await fetch(`${ELEVEN_BASE}${upstreamPath}`, {
			method: 'POST',
			headers: {
				'content-type': 'application/json',
				accept: withTimestamps ? 'application/json' : 'audio/mpeg',
				'xi-api-key': apiKey,
			},
			body: JSON.stringify({
				text,
				model_id: modelId,
				voice_settings: settings,
			}),
			// Bound a hung connection (ElevenLabs accepting but never responding)
			// under Vercel's 30s kill, so a stall returns a clean 502 + char refund
			// instead of a 504 with characters charged but no audio delivered. Set
			// well above a real clip's synth+stream time so it never cuts valid audio.
			signal: AbortSignal.timeout(28000),
		});
	} catch (fetchErr) {
		console.error('[tts/eleven] ElevenLabs fetch failed', fetchErr);
		await refundMetering();
		return error(res, 502, 'upstream_error', 'Could not reach ElevenLabs');
	}

	if (!elResp.ok) {
		const msg = await elResp.text().catch(() => '');
		console.error('[tts/eleven] ElevenLabs error', elResp.status, msg);
		await refundMetering();
		// A 400/404 from the text-to-speech endpoint is the caller's voiceId: the
		// model id and the key are resolved server-side, so the voice is the only
		// caller-controlled part of the request left. Answering 502 blamed the
		// platform for a typo the caller can fix, and hid it from the client's own
		// error handling. The charge is already refunded above either way.
		const callerFault = elResp.status === 400 || elResp.status === 404;
		return error(
			res,
			callerFault ? 400 : 502,
			callerFault ? 'validation_error' : 'upstream_error',
			callerFault
				? `ElevenLabs does not have voice "${voiceId}" on this account`
				: `ElevenLabs returned ${elResp.status}`,
		);
	}

	res.setHeader('content-type', responseContentType);
	res.setHeader('x-tts-cache', 'miss');
	res.setHeader('x-tts-billing', billing);
	if (creditCharge?.chargedUsd > 0)
		res.setHeader('x-tts-charged-usd', creditCharge.chargedUsd.toFixed(6));
	res.setHeader('cache-control', 'private, max-age=86400');

	if (withTimestamps) {
		// One JSON object, not a stream: buffer it whole, forward it, cache it.
		let jsonBuffer;
		try {
			jsonBuffer = Buffer.from(await elResp.arrayBuffer());
		} catch (readErr) {
			console.error('[tts/eleven] with-timestamps read failed', readErr);
			await refundMetering();
			return error(res, 502, 'upstream_error', 'ElevenLabs with-timestamps response unreadable');
		}
		res.setHeader('content-length', String(jsonBuffer.length));
		res.end(jsonBuffer);
		cacheAudio(cacheKey, jsonBuffer, responseContentType);
		return;
	}

	if (!elResp.body) {
		// No readable stream (unexpected) — fall back to a single buffered write.
		const audioBuffer = Buffer.from(await elResp.arrayBuffer());
		res.setHeader('content-length', String(audioBuffer.length));
		res.end(audioBuffer);
		cacheAudio(cacheKey, audioBuffer, responseContentType);
		return;
	}

	const chunks = [];
	let completed = false;
	try {
		for await (const chunk of elResp.body) {
			if (res.destroyed) break; // client hung up — stop pulling from upstream
			const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
			chunks.push(buf);
			// Respect write backpressure so a slow client can't pile unbounded data
			// into the socket buffer. Resolve on 'close' too, so a mid-clip
			// disconnect can't hang this await forever.
			if (!res.write(buf)) {
				await new Promise((resolve) => {
					const done = () => {
						res.off('drain', done);
						res.off('close', done);
						resolve();
					};
					res.once('drain', done);
					res.once('close', done);
				});
			}
		}
		// Only a naturally-exhausted iterator (no break/destroy) means every
		// upstream byte arrived — required before we may cache the clip.
		completed = !res.destroyed;
	} catch (streamErr) {
		// Upstream aborted or the client disconnected mid-clip. Don't cache a
		// partial, possibly-corrupt file — just close the response.
		console.error('[tts/eleven] stream interrupted', streamErr);
		if (!res.writableEnded && !res.destroyed) res.end();
		return;
	}

	if (!res.writableEnded && !res.destroyed) res.end();
	// Cache only a complete clip — a truncated one would poison the deterministic
	// cache key and be served as a permanent "hit" for this utterance.
	if (completed) cacheAudio(cacheKey, Buffer.concat(chunks), responseContentType);
});