Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(tts): let speech fall through to the Gemini lane instead of failing with NVIDIA
nirholas48cd4036ed88
diff --git a/api/_lib/ai-speech.js b/api/_lib/ai-speech.jsindex e06cafc3f..3ae5c7232 100644--- a/api/_lib/ai-speech.js+++ b/api/_lib/ai-speech.js@@ -24,6 +24,7 @@ import {parseWav,} from './asr-nvidia.js';import { TTS_VOICES, TTS_VOICE_IDS, DEFAULT_VOICE } from './tts-voices.js';+import { geminiTtsConfigured, synthesizeGeminiTts } from './tts-gemini.js';export { nvidiaTtsConfigured, nvidiaAsrConfigured };@@ -110,11 +111,46 @@ export function readTtsInput(raw, { maxChars = PAID_TTS_MAX_CHARS } = {}) {// Synthesize on the free NVIDIA NIM Magpie lane. Returns a JSON-serializable// payload with base64 audio (uniform across the free + paid lanes so settlement// on the paid rail always emits JSON). Throws an HTTP-typed error on lane failure.+/**+ * True when speech synthesis can serve at all: the NIM Magpie lane, the Gemini+ * TTS lane, or both. Callers gate on this rather than on one vendor's key, so+ * an endpoint is never reported unavailable while a configured lane sits idle.+ */+export function ttsConfigured() {+ return nvidiaTtsConfigured() || geminiTtsConfigured();+}+export async function ttsSynthesize({ text, voice, format, language }) {let out;try {out = await synthesizeNvidiaTts({ text, voice, language, format, timeoutMs: TTS_TIMEOUT_MS });} catch (e) {+ // Magpie is the preferred lane, not the only one it is worth trying. A+ // caller asked for speech, not for a vendor, so a NIM outage falls through+ // to the Gemini TTS lane (platform GCP credits, no vendor quota) before the+ // request is failed. The lane that answered is reported in `model` and+ // `voice`, so nothing pretends the clip came from Magpie, and the voice+ // name changes with it: that is the honest cost of still getting audio.+ if (geminiTtsConfigured()) {+ try {+ const alt = await synthesizeGeminiTts({ text, timeoutMs: TTS_TIMEOUT_MS });+ return {+ audio: alt.audio.toString('base64'),+ encoding: 'base64',+ format: alt.format,+ content_type: alt.contentType,+ sample_rate: alt.sampleRateHz,+ voice: alt.voiceName,+ model: alt.model,+ characters: text.length,+ bytes: alt.audio.length,+ lane: alt.lane,+ fallback_from: 'nvidia',+ };+ } catch (fallbackErr) {+ console.warn(`[ai-speech] gemini tts fallback failed: ${fallbackErr?.message || fallbackErr}`);+ }+ }throw laneErrorToHttp(e, 'Speech synthesis');}return {diff --git a/api/healthz.js b/api/healthz.jsindex 579fe305a..415a2e63a 100644--- a/api/healthz.js+++ b/api/healthz.js@@ -10,7 +10,7 @@import { cors, json, method, wrap } from './_lib/http.js';import { countRecentPayments } from './_lib/x402/audit-log.js';import { gatherSubsystemHealth } from './_lib/ops/subsystem-health.js';-import { nvidiaTtsConfigured, nvidiaAsrConfigured } from './_lib/ai-speech.js';+import { ttsConfigured, nvidiaAsrConfigured } from './_lib/ai-speech.js';import { alertsConfigured } from './_lib/alerts.js';const STARTED_AT = Date.now();@@ -354,7 +354,7 @@ export default wrap(async (req, res) => {// status so a missing NVIDIA key shows here instead of only surfacing as a// 503 on the endpoint. Sync env reads, no upstream probe, no key material.speech: {- tts: { configured: nvidiaTtsConfigured(), route: '/api/v1/ai/tts', requires: ['NVIDIA_API_KEY'] },+ tts: { configured: ttsConfigured(), route: '/api/v1/ai/tts', requires: ['NVIDIA_API_KEY or GOOGLE_CLOUD_PROJECT'] },asr: {configured: nvidiaAsrConfigured(),route: '/api/v1/ai/asr',diff --git a/api/tts/speak.js b/api/tts/speak.jsindex c3b87f862..990716807 100644--- a/api/tts/speak.js+++ b/api/tts/speak.js@@ -43,6 +43,12 @@ const FORMATS = {// Per-attempt budget for the free lane: generous enough for 4096 chars of// synthesis, small enough that the paid backstop still fits in maxDuration.const NVIDIA_TIMEOUT_MS = 30_000;+// The backstop's OWN budget. It used to reuse NVIDIA_TIMEOUT_MS, which reads as+// a shared 30s but is not: lane 1 can spend all of it before lane 2 starts, and+// the platform kills the invocation while OpenAI is still allowed 30 more+// seconds. A shorter, separate budget is what makes the second lane a real+// fallback rather than one that exists only when the first fails instantly.+const OPENAI_TIMEOUT_MS = 20_000;export default wrap(async function handler(req, res) {if (cors(req, res, { methods: 'POST,OPTIONS' })) return;@@ -127,7 +133,7 @@ export default wrap(async function handler(req, res) {body: JSON.stringify({model, voice, input: text, response_format: formatKey, speed,}),- signal: AbortSignal.timeout(NVIDIA_TIMEOUT_MS),+ signal: AbortSignal.timeout(OPENAI_TIMEOUT_MS),});} catch (e) {laneErrors.push(`openai: ${e?.message || 'fetch failed'}`);diff --git a/api/v1/ai/tts.js b/api/v1/ai/tts.jsindex 2dce25275..a3dc8bef0 100644--- a/api/v1/ai/tts.js+++ b/api/v1/ai/tts.js@@ -20,9 +20,9 @@ import { declareHttpDiscovery, withService } from '../../_lib/x402/bazaar-helperimport { installAccessControl } from '../../_lib/x402/access-control.js';import { TTS_VOICE_IDS, DEFAULT_VOICE } from '../../_lib/tts-voices.js';import {- nvidiaTtsConfigured,ttsVoicesPayload,readTtsInput,+ ttsConfigured,ttsSynthesize,FREE_TTS_MAX_CHARS,PAID_TTS_MAX_CHARS,@@ -164,7 +164,7 @@ export default wrap(async function handler(req, res) {data: {endpoint: ROUTE,description: 'POST text to synthesize speech. GET ?voices=1 lists voices.',- configured: nvidiaTtsConfigured(),+ configured: ttsConfigured(),free_tier: { per_day: 10, max_chars: FREE_TTS_MAX_CHARS },paid: { price_usdc: Number(PRICE_ATOMICS) / 1e6, max_chars: PAID_TTS_MAX_CHARS, networks: ['base', 'solana'] },},@@ -174,8 +174,17 @@ export default wrap(async function handler(req, res) {}// ── POST: synthesize ──────────────────────────────────────────────────────- if (!nvidiaTtsConfigured()) {- return error(res, 503, 'not_configured', 'Text-to-speech is not configured (set NVIDIA_API_KEY)');+ // ttsSynthesize falls through to the Gemini TTS lane when Magpie cannot+ // serve, so the endpoint is available whenever EITHER is configured. Gating+ // on the NVIDIA key alone used to 503 a request the Gemini lane could have+ // answered on platform credits.+ if (!ttsConfigured()) {+ return error(+ res,+ 503,+ 'not_configured',+ 'Text-to-speech is not configured (set NVIDIA_API_KEY, or GOOGLE_CLOUD_PROJECT for the Gemini lane)',+ );}// Buffer the body once, then decide the lane — the paid rail reads the same