Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

12.2 KB
Raw Blame
// Free NVIDIA NIM ASR lane — Riva speech-to-text over NVCF gRPC.
//
// This is the voice-IN sibling of api/_lib/tts-nvidia.js (voice-OUT). It closes
// the avatar voice loop: Magpie TTS already gives every avatar a free voice;
// this lane gives users a free, cross-browser way to TALK to the avatar,
// replacing the browser-only window.webkitSpeechRecognition path (Chrome/Edge
// only, ships user audio to Google) with server-side Riva recognition.
//
// Like Magpie, Riva ASR is hosted as an NVCF gRPC function on
// grpc.nvcf.nvidia.com:443, selected by a `function-id` metadata entry with the
// nvapi key as a bearer `authorization` metadata entry. The wire contract is
// the standard Riva SpeechRecognition service; the proto definitions are
// vendored under ./riva-protos/ and loaded from a generated JSON descriptor
// (asr-descriptor.js — kept separate from the TTS descriptor) so no .proto file
// needs to exist on the serverless filesystem (api/ routes are esbuild-bundled
// in place — scripts/bundle-api.mjs).
//
// ── Function id is configuration, never a guessed constant ───────────────────
// Unlike Magpie's id (live-probed and pinned), the hosted ASR model a deployment
// wants can vary (Parakeet CTC/RNN-T, Canary multilingual, different versions),
// so the NVCF function id is read from NVIDIA_ASR_FUNCTION_ID. Discover the live
// id for your account with `node scripts/verify-nvidia-asr.mjs --list`, which
// enumerates GET api.nvcf.nvidia.com/v2/nvcf/functions and prints the ASR
// candidates. Until it is set, nvidiaAsrConfigured() returns false and callers
// treat the lane as absent — no fake default, no silent misroute.
//
// Error codes match the established provider contract (tts-nvidia.js):
//   invalid_key / rate_limited / invalid_argument / timeout /
//   provider_unreachable / provider_error / not_configured.

import { env } from './env.js';
import asrDescriptor from './riva-protos/asr-descriptor.js';

export const NVIDIA_ASR_HOST = 'grpc.nvcf.nvidia.com:443';
const DEFAULT_TIMEOUT_MS = 30_000;

// Riva ASR's offline Recognize accepts mono audio in one of these container/
// codec encodings (riva_audio.proto AudioEncoding). We expose friendly aliases
// so the endpoint can map a request format onto the proto enum. LINEAR_PCM is
// the cross-browser default: every browser can produce raw 16-bit PCM via Web
// Audio even where MediaRecorder codecs differ.
export const ASR_ENCODINGS = new Map([
	['pcm', 'LINEAR_PCM'], ['wav', 'LINEAR_PCM'], ['linear_pcm', 'LINEAR_PCM'], ['linear16', 'LINEAR_PCM'],
	['flac', 'FLAC'],
	['opus', 'OGGOPUS'], ['ogg', 'OGGOPUS'], ['oggopus', 'OGGOPUS'],
	['mulaw', 'MULAW'], ['ulaw', 'MULAW'],
	['alaw', 'ALAW'],
]);

export function resolveAsrEncoding(encoding) {
	if (!encoding) return 'LINEAR_PCM';
	return ASR_ENCODINGS.get(String(encoding).toLowerCase()) || 'LINEAR_PCM';
}

export function nvidiaAsrConfigured() {
	return Boolean(
		env.NVIDIA_API_KEY &&
			(env.NVIDIA_ASR_FUNCTION_ID || env.NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL),
	);
}

// ── Language routing ────────────────────────────────────────────────────────
// One hosted function does not cover every language. The English-tuned Parakeet
// models reject a non-`en` language_code outright (gRPC 3 INVALID_ARGUMENT), so
// a Mandarin utterance sent to the English pin fails rather than degrading. A
// multilingual model (ai-canary-1b-asr on our account) transcribes the languages
// below, and English at the same quality as the English pin. We therefore keep
// the English pin for `en*` (better casing and punctuation on English) and route
// every other language to the multilingual id when one is configured.
//
// The list is empirical, not aspirational: every tag here was verified by
// synthesizing a sentence in that language and reading back a correct
// transcript (`node scripts/verify-nvidia-asr.mjs --languages`). Languages the
// model rejects (Arabic, Malay, Tamil, Filipino, Norwegian) or garbles
// (Indonesian, Vietnamese, Finnish, Danish, Greek, Hungarian) are deliberately
// absent, so the client never offers a language the lane cannot hear.
export const MULTILINGUAL_ASR_LANGUAGES = [
	'zh-CN',
	'zh-TW',
	'ja-JP',
	'ko-KR',
	'es-ES',
	'fr-FR',
	'de-DE',
	'it-IT',
	'pt-BR',
	'ru-RU',
	'hi-IN',
	'nl-NL',
	'pl-PL',
	'cs-CZ',
];

/** True when `language` is an English tag (the English pin's territory). */
function isEnglish(language) {
	return String(language || '').trim().toLowerCase().startsWith('en');
}

/**
 * NVCF function id to use for `language`, or null when nothing is configured.
 * English (and any language with no multilingual id configured) stays on
 * NVIDIA_ASR_FUNCTION_ID; everything else prefers the multilingual id.
 */
export function asrFunctionIdFor(language) {
	const multilingual = env.NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL || null;
	const primary = env.NVIDIA_ASR_FUNCTION_ID || null;
	if (!isEnglish(language) && multilingual) return multilingual;
	return primary || multilingual;
}

/**
 * BCP-47 tags this deployment can actually recognize, for the capability probe.
 * Always includes en-US (the base lane); the multilingual tags appear only when
 * a multilingual function id is configured.
 */
export function asrLanguages() {
	const langs = env.NVIDIA_ASR_FUNCTION_ID ? ['en-US', 'en-GB'] : [];
	if (env.NVIDIA_ASR_FUNCTION_ID_MULTILINGUAL) {
		if (!langs.length) langs.push('en-US');
		langs.push(...MULTILINGUAL_ASR_LANGUAGES);
	}
	return langs;
}

// Parse a RIFF/WAVE header → { sampleRateHz, channels, pcm } so a WAV upload can
// be handed to Riva as raw LINEAR_PCM with the correct rate. Riva's LINEAR_PCM
// expects header-less little-endian s16 samples; passing the 44-byte RIFF header
// through prepends ~44 bytes of garbage to the first samples and skews the rate
// the model is told. Returns null when the bytes are not a WAV (caller then
// trusts the explicitly-supplied rate/encoding).
export function parseWav(buf) {
	if (!Buffer.isBuffer(buf) || buf.length < 44) return null;
	if (buf.toString('ascii', 0, 4) !== 'RIFF' || buf.toString('ascii', 8, 12) !== 'WAVE') return null;
	// Walk the chunk list rather than assuming the canonical 44-byte layout —
	// some encoders insert a LIST/fact chunk before `data`.
	let offset = 12;
	let sampleRateHz = 0;
	let channels = 1;
	let pcm = null;
	while (offset + 8 <= buf.length) {
		const id = buf.toString('ascii', offset, offset + 4);
		const size = buf.readUInt32LE(offset + 4);
		const body = offset + 8;
		if (id === 'fmt ' && body + 16 <= buf.length) {
			channels = buf.readUInt16LE(body + 2) || 1;
			sampleRateHz = buf.readUInt32LE(body + 4);
		} else if (id === 'data') {
			pcm = buf.subarray(body, Math.min(body + size, buf.length));
		}
		// Chunks are word-aligned: an odd size carries a trailing pad byte.
		offset = body + size + (size & 1);
	}
	if (!pcm || !sampleRateHz) return null;
	return { sampleRateHz, channels, pcm };
}

// gRPC status code → platform error code (numeric codes so the mapping does not
// depend on the grpc-js import being resolved). Mirrors tts-nvidia.js.
function normalizeGrpcError(err) {
	const code = typeof err?.code === 'number' ? err.code : null;
	const map = {
		3: 'invalid_argument', // bad encoding/rate/language/audio
		4: 'timeout', // DEADLINE_EXCEEDED
		7: 'invalid_key', // PERMISSION_DENIED
		8: 'rate_limited', // RESOURCE_EXHAUSTED (credit-metered free tier)
		14: 'provider_unreachable', // UNAVAILABLE
		16: 'invalid_key', // UNAUTHENTICATED
	};
	const normalized = new Error(
		`NVIDIA Riva ASR failed${code !== null ? ` (gRPC ${code})` : ''}: ${err?.details || err?.message || 'unknown error'}`,
	);
	normalized.code = (code !== null && map[code]) || 'provider_error';
	normalized.grpcCode = code;
	return normalized;
}

// One cached client per warm lambda (the TLS channel is keyless — auth rides in
// per-call metadata). @grpc/grpc-js is imported lazily so non-ASR importers do
// not pay for the dependency at cold start. See tts-nvidia.js for the NVCF
// warm-channel auth caveat that the verify script works around.
let clientPromise = null;
async function getClient() {
	if (!clientPromise) {
		clientPromise = (async () => {
			const [{ default: grpc }, { default: protoLoader }] = await Promise.all([
				import('@grpc/grpc-js'),
				import('@grpc/proto-loader'),
			]);
			const definition = protoLoader.fromJSON(asrDescriptor, {
				keepCase: true,
				longs: Number,
				enums: String,
				defaults: true,
				oneofs: true,
			});
			const pkg = grpc.loadPackageDefinition(definition);
			const Recognition = pkg.nvidia.riva.asr.RivaSpeechRecognition;
			return {
				grpc,
				client: new Recognition(NVIDIA_ASR_HOST, grpc.credentials.createSsl(), {
					'grpc.max_send_message_length': 64 * 1024 * 1024,
				}),
			};
		})().catch((e) => {
			clientPromise = null; // a failed init must not poison the warm lambda
			throw e;
		});
	}
	return clientPromise;
}

// Transcribe a single utterance on the free NIM lane via the offline Recognize
// RPC (unary — fits a serverless function; no streaming socket to babysit).
//
//   { audio: Buffer, encoding?, sampleRateHz?, language?, maxAlternatives?,
//     automaticPunctuation?, wordTimeOffsets?, model?, timeoutMs?, apiKey? }
//     → { text, confidence, words, language, model, audioProcessed }
//
// `audio` is raw bytes in `encoding`. When a WAV is passed as LINEAR_PCM the
// caller should strip the header first (parseWav) — this function trusts the
// encoding/rate it is given.
//
// Throws Error with .code ∈ not_configured | invalid_key | rate_limited |
// invalid_argument | timeout | provider_unreachable | provider_error.
export async function transcribeNvidiaAsr({
	audio,
	encoding = 'LINEAR_PCM',
	sampleRateHz = 16000,
	language = 'en-US',
	maxAlternatives = 1,
	automaticPunctuation = true,
	wordTimeOffsets = false,
	model = '',
	timeoutMs = DEFAULT_TIMEOUT_MS,
	apiKey,
	functionId: functionIdOverride,
} = {}) {
	const key = apiKey || env.NVIDIA_API_KEY;
	const functionId = functionIdOverride || asrFunctionIdFor(language);
	if (!key || !functionId) {
		const err = new Error(
			!key ? 'NVIDIA_API_KEY not set' : 'NVIDIA_ASR_FUNCTION_ID not set (run scripts/verify-nvidia-asr.mjs --list to discover it)',
		);
		err.code = 'not_configured';
		throw err;
	}
	const bytes = Buffer.isBuffer(audio) ? audio : Buffer.from(audio || []);
	if (!bytes.length) {
		const err = new Error('no audio bytes to transcribe');
		err.code = 'invalid_argument';
		throw err;
	}

	const { grpc, client } = await getClient();
	const metadata = new grpc.Metadata();
	metadata.set('function-id', functionId);
	metadata.set('authorization', `Bearer ${key}`);

	const response = await new Promise((resolveCall, rejectCall) => {
		client.recognize(
			{
				config: {
					encoding: ASR_ENCODINGS.get(String(encoding).toLowerCase()) || encoding,
					sample_rate_hertz: sampleRateHz,
					language_code: language,
					max_alternatives: maxAlternatives,
					enable_automatic_punctuation: automaticPunctuation,
					enable_word_time_offsets: wordTimeOffsets,
					...(model ? { model } : {}),
				},
				audio: bytes,
			},
			metadata,
			{ deadline: new Date(Date.now() + timeoutMs) },
			(err, res) => (err ? rejectCall(normalizeGrpcError(err)) : resolveCall(res)),
		);
	});

	// Riva returns sequential results; the offline path currently yields one.
	// Concatenate the top alternative of each result into a single transcript so
	// the caller gets the complete utterance regardless of internal segmentation.
	const results = Array.isArray(response?.results) ? response.results : [];
	const tops = results.map((r) => r?.alternatives?.[0]).filter(Boolean);
	const text = tops.map((a) => a.transcript || '').join(' ').trim();
	const confidence = tops.length ? tops.reduce((s, a) => s + (a.confidence || 0), 0) / tops.length : 0;
	const words = wordTimeOffsets
		? tops.flatMap((a) => (Array.isArray(a.words) ? a.words : []).map((w) => ({
			word: w.word,
			startMs: w.start_time,
			endMs: w.end_time,
			confidence: w.confidence,
		})))
		: [];
	const audioProcessed = results.reduce((m, r) => Math.max(m, r?.audio_processed || 0), 0);
	const detectedLang = tops[0]?.language_code?.[0] || language;

	return { text, confidence, words, language: detectedLang, model: model || 'riva-asr', audioProcessed };
}