Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(tts): let speech fall through to the Gemini lane instead of failing with NVIDIA

nirholas48cd4036ed88
diff --git a/api/_lib/ai-speech.js b/api/_lib/ai-speech.js
index e06cafc3f..3ae5c7232 100644
--- a/api/_lib/ai-speech.js
+++ b/api/_lib/ai-speech.js
@@ -24,6 +24,7 @@ import {
parseWav,
} from './asr-nvidia.js';
import { TTS_VOICES, TTS_VOICE_IDS, DEFAULT_VOICE } from './tts-voices.js';
+import { geminiTtsConfigured, synthesizeGeminiTts } from './tts-gemini.js';
export { nvidiaTtsConfigured, nvidiaAsrConfigured };
@@ -110,11 +111,46 @@ export function readTtsInput(raw, { maxChars = PAID_TTS_MAX_CHARS } = {}) {
// Synthesize on the free NVIDIA NIM Magpie lane. Returns a JSON-serializable
// payload with base64 audio (uniform across the free + paid lanes so settlement
// on the paid rail always emits JSON). Throws an HTTP-typed error on lane failure.
+/**
+ * True when speech synthesis can serve at all: the NIM Magpie lane, the Gemini
+ * TTS lane, or both. Callers gate on this rather than on one vendor's key, so
+ * an endpoint is never reported unavailable while a configured lane sits idle.
+ */
+export function ttsConfigured() {
+ return nvidiaTtsConfigured() || geminiTtsConfigured();
+}
+
export async function ttsSynthesize({ text, voice, format, language }) {
let out;
try {
out = await synthesizeNvidiaTts({ text, voice, language, format, timeoutMs: TTS_TIMEOUT_MS });
} catch (e) {
+ // Magpie is the preferred lane, not the only one it is worth trying. A
+ // caller asked for speech, not for a vendor, so a NIM outage falls through
+ // to the Gemini TTS lane (platform GCP credits, no vendor quota) before the
+ // request is failed. The lane that answered is reported in `model` and
+ // `voice`, so nothing pretends the clip came from Magpie, and the voice
+ // name changes with it: that is the honest cost of still getting audio.
+ if (geminiTtsConfigured()) {
+ try {
+ const alt = await synthesizeGeminiTts({ text, timeoutMs: TTS_TIMEOUT_MS });
+ return {
+ audio: alt.audio.toString('base64'),
+ encoding: 'base64',
+ format: alt.format,
+ content_type: alt.contentType,
+ sample_rate: alt.sampleRateHz,
+ voice: alt.voiceName,
+ model: alt.model,
+ characters: text.length,
+ bytes: alt.audio.length,
+ lane: alt.lane,
+ fallback_from: 'nvidia',
+ };
+ } catch (fallbackErr) {
+ console.warn(`[ai-speech] gemini tts fallback failed: ${fallbackErr?.message || fallbackErr}`);
+ }
+ }
throw laneErrorToHttp(e, 'Speech synthesis');
}
return {
diff --git a/api/healthz.js b/api/healthz.js
index 579fe305a..415a2e63a 100644
--- a/api/healthz.js
+++ b/api/healthz.js
@@ -10,7 +10,7 @@
import { cors, json, method, wrap } from './_lib/http.js';
import { countRecentPayments } from './_lib/x402/audit-log.js';
import { gatherSubsystemHealth } from './_lib/ops/subsystem-health.js';
-import { nvidiaTtsConfigured, nvidiaAsrConfigured } from './_lib/ai-speech.js';
+import { ttsConfigured, nvidiaAsrConfigured } from './_lib/ai-speech.js';
import { alertsConfigured } from './_lib/alerts.js';
const STARTED_AT = Date.now();
@@ -354,7 +354,7 @@ export default wrap(async (req, res) => {
// status so a missing NVIDIA key shows here instead of only surfacing as a
// 503 on the endpoint. Sync env reads, no upstream probe, no key material.
speech: {
- tts: { configured: nvidiaTtsConfigured(), route: '/api/v1/ai/tts', requires: ['NVIDIA_API_KEY'] },
+ tts: { configured: ttsConfigured(), route: '/api/v1/ai/tts', requires: ['NVIDIA_API_KEY or GOOGLE_CLOUD_PROJECT'] },
asr: {
configured: nvidiaAsrConfigured(),
route: '/api/v1/ai/asr',
diff --git a/api/tts/speak.js b/api/tts/speak.js
index c3b87f862..990716807 100644
--- a/api/tts/speak.js
+++ b/api/tts/speak.js
@@ -43,6 +43,12 @@ const FORMATS = {
// Per-attempt budget for the free lane: generous enough for 4096 chars of
// synthesis, small enough that the paid backstop still fits in maxDuration.
const NVIDIA_TIMEOUT_MS = 30_000;
+// The backstop's OWN budget. It used to reuse NVIDIA_TIMEOUT_MS, which reads as
+// a shared 30s but is not: lane 1 can spend all of it before lane 2 starts, and
+// the platform kills the invocation while OpenAI is still allowed 30 more
+// seconds. A shorter, separate budget is what makes the second lane a real
+// fallback rather than one that exists only when the first fails instantly.
+const OPENAI_TIMEOUT_MS = 20_000;
export default wrap(async function handler(req, res) {
if (cors(req, res, { methods: 'POST,OPTIONS' })) return;
@@ -127,7 +133,7 @@ export default wrap(async function handler(req, res) {
body: JSON.stringify({
model, voice, input: text, response_format: formatKey, speed,
}),
- signal: AbortSignal.timeout(NVIDIA_TIMEOUT_MS),
+ signal: AbortSignal.timeout(OPENAI_TIMEOUT_MS),
});
} catch (e) {
laneErrors.push(`openai: ${e?.message || 'fetch failed'}`);
diff --git a/api/v1/ai/tts.js b/api/v1/ai/tts.js
index 2dce25275..a3dc8bef0 100644
--- a/api/v1/ai/tts.js
+++ b/api/v1/ai/tts.js
@@ -20,9 +20,9 @@ import { declareHttpDiscovery, withService } from '../../_lib/x402/bazaar-helper
import { installAccessControl } from '../../_lib/x402/access-control.js';
import { TTS_VOICE_IDS, DEFAULT_VOICE } from '../../_lib/tts-voices.js';
import {
- nvidiaTtsConfigured,
ttsVoicesPayload,
readTtsInput,
+ ttsConfigured,
ttsSynthesize,
FREE_TTS_MAX_CHARS,
PAID_TTS_MAX_CHARS,
@@ -164,7 +164,7 @@ export default wrap(async function handler(req, res) {
data: {
endpoint: ROUTE,
description: 'POST text to synthesize speech. GET ?voices=1 lists voices.',
- configured: nvidiaTtsConfigured(),
+ configured: ttsConfigured(),
free_tier: { per_day: 10, max_chars: FREE_TTS_MAX_CHARS },
paid: { price_usdc: Number(PRICE_ATOMICS) / 1e6, max_chars: PAID_TTS_MAX_CHARS, networks: ['base', 'solana'] },
},
@@ -174,8 +174,17 @@ export default wrap(async function handler(req, res) {
}
// ── POST: synthesize ──────────────────────────────────────────────────────
- if (!nvidiaTtsConfigured()) {
- return error(res, 503, 'not_configured', 'Text-to-speech is not configured (set NVIDIA_API_KEY)');
+ // ttsSynthesize falls through to the Gemini TTS lane when Magpie cannot
+ // serve, so the endpoint is available whenever EITHER is configured. Gating
+ // on the NVIDIA key alone used to 503 a request the Gemini lane could have
+ // answered on platform credits.
+ if (!ttsConfigured()) {
+ return error(
+ res,
+ 503,
+ 'not_configured',
+ 'Text-to-speech is not configured (set NVIDIA_API_KEY, or GOOGLE_CLOUD_PROJECT for the Gemini lane)',
+ );
}
// Buffer the body once, then decide the lane — the paid rail reads the same