Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: send audio.cpp request fields in the form each model reads (#13791)

Etherl402a405f16a1
diff --git a/studio/backend/core/inference/audio_cpp_backend.py b/studio/backend/core/inference/audio_cpp_backend.py
index 4d952693987..fd7824313ac 100644
--- a/studio/backend/core/inference/audio_cpp_backend.py
+++ b/studio/backend/core/inference/audio_cpp_backend.py
@@ -750,11 +750,23 @@ class AudioCppBackend:
body["voice"] = voice
default_options.update(chosen)
requested: dict[str, Any] = {}
+ # A strict contract refuses undeclared options; the retry would then drop the description too.
+ keys = model.request_keys
if instructions and str(instructions).strip():
- # Voice-design and style-capable families read the description as the instruction.
- requested["instruct"] = str(instructions).strip()
+ # Breeze and Irodori v4 call it instruction, DramaBox has neither.
+ name = next((n for n in ("instruct", "instruction") if keys is None or n in keys), None)
+ if name:
+ requested[name] = str(instructions).strip()
+ else:
+ logger.info(
+ "audio.cpp: %s declares no instruction option; not sending the description",
+ model.family,
+ )
if language and str(language).strip():
- requested["language"] = str(language).strip()
+ if keys is None or "language" in keys:
+ requested["language"] = str(language).strip()
+ else:
+ logger.info("audio.cpp: %s does not declare language; not sending it", model.family)
merged = {**default_options, **requested}
if merged:
body["options"] = merged
diff --git a/studio/backend/core/inference/audio_cpp_models.py b/studio/backend/core/inference/audio_cpp_models.py
index 86629981283..18b905470e0 100644
--- a/studio/backend/core/inference/audio_cpp_models.py
+++ b/studio/backend/core/inference/audio_cpp_models.py
@@ -909,11 +909,21 @@ _FAMILY_LIST: tuple[AudioCppFamily, ...] = (
),
),
# Speech to text.
+ # Full context fails past the ~400 s encoder ceiling; windows drop about a third of the words.
+ AudioCppFamily(
+ "parakeet_tdt",
+ "asr",
+ model_options = {
+ "session_options": {
+ "parakeet_tdt.offline_mode": "auto",
+ "parakeet_tdt.audio_chunk_threshold_sec": "390",
+ }
+ },
+ ),
*(
AudioCppFamily(name, "asr")
for name in (
"qwen3_asr",
- "parakeet_tdt",
"canary_asr",
"citrinet_asr",
"cohere_asr",
@@ -2597,6 +2607,10 @@ def _raw_request_options(spec: Optional[dict], embedded: Optional[dict]) -> list
def _request_keys(spec: Optional[dict], embedded: Optional[dict]) -> Optional[frozenset[str]]:
+ # The server prefers an embedded v1 contract over model_specs/ (model_spec/package.cpp).
+ if isinstance(embedded, dict) and embedded.get("schema_version") is not None:
+ if (embedded.get("options") or {}).get("request"):
+ spec = None
if _request_source(spec, embedded).get("schema_version") is None:
return None
names = frozenset(
diff --git a/studio/backend/core/inference/stt_audiocpp_sidecar.py b/studio/backend/core/inference/stt_audiocpp_sidecar.py
index 908084673e8..fcd224e6ce3 100644
--- a/studio/backend/core/inference/stt_audiocpp_sidecar.py
+++ b/studio/backend/core/inference/stt_audiocpp_sidecar.py
@@ -14,6 +14,7 @@ the timestamps and speakers a model produces.
from __future__ import annotations
+import functools
import json
import subprocess
import tempfile
@@ -55,6 +56,7 @@ from core.inference.audio_cpp_server import (
)
from core.inference.stt_ggml_sidecar import _pcm_to_wav_bytes
from core.inference.stt_sidecar import (
+ QWEN3_ASR_LANGUAGE_NAMES,
STT_KEEP_ALIVE_SECONDS,
SttAudioDecodeError,
SttLoadCancelledError,
@@ -95,6 +97,43 @@ _TIMESTAMP_OPTIONS = {
"qwen3_asr.preserve_punctuation": "true",
}
+# model_specs/nemotron_asr.json "languages"; the prompt dictionary also keys en/es/fr/pt/no.
+_NEMOTRON_ASR_LOCALES = (
+ "ar-AR", "bg-BG", "cs-CZ", "da-DK", "de-DE", "el-GR", "en-GB", "en-US", "es-ES", "es-US",
+ "et-EE", "fi-FI", "fr-CA", "fr-FR", "he-IL", "hi-IN", "hr-HR", "hu-HU", "it-IT", "ja-JP",
+ "ko-KR", "lt-LT", "lv-LV", "mt-MT", "nb-NO", "nl-NL", "nn-NO", "pl-PL", "pt-BR", "pt-PT",
+ "ro-RO", "ru-RU", "sk-SK", "sl-SI", "sv-SE", "th-TH", "tr-TR", "uk-UA", "vi-VN", "zh-CN",
+) # fmt: skip
+
+
[email protected]_cache(maxsize = 256)
+def _nemotron_language(language: str, code: str) -> Optional[str]:
+ """The prompt key Nemotron takes for ``language``; any other string fails the request."""
+ tag = language.strip().replace("_", "-").lower()
+ primary = tag.split("-", 1)[0]
+ for matches in (
+ [locale for locale in _NEMOTRON_ASR_LOCALES if locale.lower() == tag],
+ [locale for locale in _NEMOTRON_ASR_LOCALES if locale.split("-", 1)[0].lower() == primary],
+ [locale for locale in _NEMOTRON_ASR_LOCALES if normalize_whisper_language(locale) == code],
+ ):
+ if len(matches) == 1:
+ return matches[0]
+ if matches:
+ return code
+ return None
+
+
+def _runtime_language(family: str, language: Optional[str]) -> Optional[str]:
+ """``language`` in the form ``family`` reads it, or None to let the model detect it."""
+ code = normalize_whisper_language(language)
+ if not code:
+ return None
+ if family == "qwen3_asr":
+ return QWEN3_ASR_LANGUAGE_NAMES.get(code)
+ if family == "nemotron_asr":
+ return _nemotron_language(language, code)
+ return code
+
class SttEngineUnavailableError(SttUnavailableError):
"""audiocpp_server is not installed or stopped serving; the audio.cpp dictation engine is off."""
@@ -998,7 +1037,10 @@ class AudioCppSttSidecar:
)
_notify(on_phase, "transcribing")
payload = self._post_details(
- path, lang, cancel_event, dict(_TIMESTAMP_OPTIONS) if aligned else {}
+ path,
+ _runtime_language(entry.family, language),
+ cancel_event,
+ dict(_TIMESTAMP_OPTIONS) if aligned else {},
)
if cancel_event is not None and cancel_event.is_set():
raise SttTranscriptionCancelledError("Transcription cancelled.")
@@ -1057,8 +1099,7 @@ class AudioCppSttSidecar:
except AudioCppRequestError as exc:
if "language" not in body or not 400 <= exc.status < 500:
raise
- # English-only families (Moonshine, Nemotron, Parakeet) reject a language option; a
- # dictation language preference is a hint, so transcribe without it rather than fail.
+ # A dictation language is a hint: transcribe without it rather than fail.
logger.info(
"audio.cpp: %s rejected language %r (%s); retrying without it",
server.model.family,
diff --git a/studio/backend/core/inference/stt_sidecar.py b/studio/backend/core/inference/stt_sidecar.py
index 55b72ebb324..8699ae1c751 100644
--- a/studio/backend/core/inference/stt_sidecar.py
+++ b/studio/backend/core/inference/stt_sidecar.py
@@ -266,6 +266,41 @@ def normalize_whisper_language(language: Optional[str]) -> Optional[str]:
return _WHISPER_LANGUAGE_ALIASES.get(primary, primary)
+# Whisper code -> Qwen3-ASR prompt name (config.support_languages; audio.cpp qwen3_asr/assets.cpp).
+QWEN3_ASR_LANGUAGE_NAMES = {
+ "ar": "Arabic",
+ "cs": "Czech",
+ "da": "Danish",
+ "de": "German",
+ "el": "Greek",
+ "en": "English",
+ "es": "Spanish",
+ "fa": "Persian",
+ "fi": "Finnish",
+ "fr": "French",
+ "hi": "Hindi",
+ "hu": "Hungarian",
+ "id": "Indonesian",
+ "it": "Italian",
+ "ja": "Japanese",
+ "ko": "Korean",
+ "mk": "Macedonian",
+ "ms": "Malay",
+ "nl": "Dutch",
+ "pl": "Polish",
+ "pt": "Portuguese",
+ "ro": "Romanian",
+ "ru": "Russian",
+ "sv": "Swedish",
+ "th": "Thai",
+ "tl": "Filipino",
+ "tr": "Turkish",
+ "vi": "Vietnamese",
+ "yue": "Cantonese",
+ "zh": "Chinese",
+}
+
+
def _stt_quiet_window_end(decoded_audio, start: int, window: int) -> Optional[int]:
"""Return a nearby quiet window boundary, or None when speech fills it."""
import numpy as np
diff --git a/studio/backend/core/inference/stt_transformers_worker.py b/studio/backend/core/inference/stt_transformers_worker.py
index cf66290e003..b6172deb9b5 100644
--- a/studio/backend/core/inference/stt_transformers_worker.py
+++ b/studio/backend/core/inference/stt_transformers_worker.py
@@ -197,44 +197,11 @@ def load_mlx_stt(
return _MlxSttModel(model)
-# Whisper language codes -> the names Qwen3-ASR lists in config.support_languages.
-_MLX_LANGUAGE_NAMES = {
- "ar": "Arabic",
- "cs": "Czech",
- "da": "Danish",
- "de": "German",
- "el": "Greek",
- "en": "English",
- "es": "Spanish",
- "fa": "Persian",
- "fi": "Finnish",
- "fr": "French",
- "hi": "Hindi",
- "hu": "Hungarian",
- "id": "Indonesian",
- "it": "Italian",
- "ja": "Japanese",
- "ko": "Korean",
- "mk": "Macedonian",
- "ms": "Malay",
- "nl": "Dutch",
- "pl": "Polish",
- "pt": "Portuguese",
- "ro": "Romanian",
- "ru": "Russian",
- "sv": "Swedish",
- "th": "Thai",
- "tl": "Filipino",
- "tr": "Turkish",
- "vi": "Vietnamese",
- "yue": "Cantonese",
- "zh": "Chinese",
-}
-
-
def _mlx_language(model, code: Optional[str]) -> Optional[str]:
"""The checkpoint's own name for a Whisper language code, or None to let it detect one."""
- name = _MLX_LANGUAGE_NAMES.get(code or "", "").lower()
+ from core.inference.stt_sidecar import QWEN3_ASR_LANGUAGE_NAMES
+
+ name = QWEN3_ASR_LANGUAGE_NAMES.get(code or "", "").lower()
supported = getattr(getattr(model, "config", None), "support_languages", None) or []
return next((lang for lang in supported if name and str(lang).lower() == name), None)
diff --git a/studio/backend/tests/test_audio_cpp_clone_requests.py b/studio/backend/tests/test_audio_cpp_clone_requests.py
index e8af6d05c95..fc3a26e89f7 100644
--- a/studio/backend/tests/test_audio_cpp_clone_requests.py
+++ b/studio/backend/tests/test_audio_cpp_clone_requests.py
@@ -167,6 +167,88 @@ def test_voxcpm2_speak_body_is_unchanged_by_clone_support():
assert server.calls == [(SPEECH, {**body, "seed": "7"})]
+class _StrictRecorder(_Recorder):
+ """Refuses an option the model's contract does not declare, as the runtime does."""
+
+ def post_json(self, path, payload, **kwargs):
+ keys = self.model.request_keys
+ undeclared = sorted(set(payload.get("options") or {}) - set(keys or ()))
+ if keys is not None and undeclared:
+ self.calls.append((path, json.loads(json.dumps(payload))))
+ raise srv.AudioCppRequestError(500, f"unknown request option: {undeclared[0]}")
+ return super().post_json(path, payload, **kwargs)
+
+
+# Request option names each published GGUF's embedded (strict) spec declares.
+STRICT_KEYS = {
+ "Breeze-TTS-2-GGUF": ("breeze_tts", {"instruction", "reference_text", "guidance_scale", "seed"}),
+ "Irodori-TTS-v4-Small-GGUF": ("irodori_tts", {"instruction", "language", "no_ref", "seed"}),
+ "DramaBox-GGUF": ("dramabox", {"negative_prompt", "target_voice", "guidance_scale", "seed"}),
+ "DotTTS-MF-GGUF": ("dots_tts", {"language", "reference_text", "template_name", "seed"}),
+ "DotTTS-Edit-GGUF": ("dots_tts", {"instruct", "instruction", "language", "template_name"}),
+ "Maya1-GGUF": ("maya1", {"instruct", "max_tokens", "temperature", "seed"}),
+}
+
+
+# fmt: off
[email protected]("display, language, options", [
+ ("Breeze-TTS-2-GGUF", None, {"instruction": "warm"}),
+ ("Breeze-TTS-2-GGUF", "en", {"instruction": "warm"}),
+ ("Irodori-TTS-v4-Small-GGUF", None, {"instruction": "warm"}),
+ ("Irodori-TTS-v4-Small-GGUF", "ja", {"instruction": "warm", "language": "ja"}),
+ ("DramaBox-GGUF", "en", ABSENT),
+ ("DotTTS-MF-GGUF", "en", {"language": "en"}),
+ ("DotTTS-Edit-GGUF", None, {"instruct": "warm"}),
+ ("Maya1-GGUF", "en", {"instruct": "warm"}),
+])
+# fmt: on
+def test_speak_sends_the_description_under_the_name_the_contract_declares(display, language, options):
+ family, keys = STRICT_KEYS[display]
+ model = replace(_model(family, display), request_keys = frozenset(keys))
+ backend = acb.AudioCppBackend()
+ server = _StrictRecorder(model)
+ backend._server, backend._model = server, model
+ backend.models, backend.active_model_name = {model.id: {"is_audio": True}}, model.id
+ backend.generate_audio_response(TEXT, language = language, instructions = "warm")
+ # One request: nothing the contract lacks is sent, so there is no refusal to retry after.
+ ((path, body),) = server.calls
+ assert path == SPEECH and body.get("options", ABSENT) == options
+
+
+def _spec(*names, strict = True):
+ spec = {"family": "dots_tts", "options": {"request": [{"name": n, "type": "string"} for n in names]}}
+ return {**spec, "schema_version": 1} if strict else spec
+
+
+# fmt: off
[email protected]("runtime, embedded, keys", [
+ # DotTTS-MF: model_specs/ declares instruct, the GGUF's v1 contract (what the server checks) does not.
+ (_spec("instruct", "instruction", "language"), _spec("language", "seed"), {"language", "seed"}),
+ # A legacy GGUF is checked against model_specs/, so that list still wins.
+ (_spec("instruct", "language"), _spec("language", strict = False), {"instruct", "language"}),
+ # A v1 GGUF that lists no request options leaves the runtime's list in charge, as before.
+ (_spec("instruct", "language"), {**_spec(), "options": {}}, {"instruct", "language"}),
+ (None, _spec("language"), {"language"}),
+ (None, _spec("language", strict = False), None),
+])
+# fmt: on
+def test_request_keys_follow_the_contract_the_server_checks(runtime, embedded, keys):
+ assert acm._request_keys(runtime, embedded) == (frozenset(keys) if keys else None)
+
+
+def test_speak_on_a_runtime_with_model_specs_sends_only_what_the_gguf_declares():
+ runtime = _spec("instruct", "instruction", "language")
+ embedded = _spec("language", "template_name", "seed")
+ model = replace(_model("dots_tts", "DotTTS-MF-GGUF"), request_keys = acm._request_keys(runtime, embedded))
+ backend = acb.AudioCppBackend()
+ server = _StrictRecorder(model)
+ backend._server, backend._model = server, model
+ backend.models, backend.active_model_name = {model.id: {"is_audio": True}}, model.id
+ backend.generate_audio_response(TEXT, language = "en", instructions = "warm")
+ ((path, body),) = server.calls
+ assert path == SPEECH and body["options"] == {"language": "en"}
+
+
# fmt: off
@pytest.mark.parametrize("key, text, sent", [
("vibevoice", "Hello there.", "Speaker 1: Hello there."),
diff --git a/studio/backend/tests/test_stt_audiocpp_details.py b/studio/backend/tests/test_stt_audiocpp_details.py
index e024580a05d..1b1f0681ebc 100644
--- a/studio/backend/tests/test_stt_audiocpp_details.py
+++ b/studio/backend/tests/test_stt_audiocpp_details.py
@@ -24,12 +24,14 @@ from test_audio_inputs import wav_bytes # noqa: E402
QWEN3 = f"{AUDIO_CPP_REPO}/Qwen3-ASR-0.6B-GGUF"
MOSS = f"{AUDIO_CPP_REPO}/MOSS-Transcribe-Diarize-GGUF"
NIAGARA = f"{AUDIO_CPP_REPO}/Niagara-ASR-GGUF"
+NEMOTRON = f"{AUDIO_CPP_REPO}/Nemotron-3.5-ASR-Streaming-0.6B-GGUF"
+PARAKEET = f"{AUDIO_CPP_REPO}/Parakeet-TDT-0.6B-v3-GGUF"
ALIGNER_FILE = "Qwen3-ForcedAligner-0.6B-GGUF/qwen3-forced-aligner-0.6b-q8_0.gguf"
ALIGNER_KEY = "qwen3_asr.forced_aligner_model_path"
FAKE_SERVER = textwrap.dedent(
r"""
- import json, os, sys, time
+ import json, os, socketserver, sys, time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
here = os.path.dirname(os.path.abspath(__file__))
@@ -79,7 +81,13 @@ FAKE_SERVER = textwrap.dedent(
]
self.reply(200, payload)
- ThreadingHTTPServer(("127.0.0.1", cfg["port"]), H).serve_forever()
+ class S(ThreadingHTTPServer):
+ # HTTPServer.server_bind reverse-resolves the host via getfqdn, ~35 s per start on macOS runners.
+ def server_bind(self):
+ socketserver.TCPServer.server_bind(self)
+ self.server_name, self.server_port = "127.0.0.1", self.server_address[1]
+
+ S(("127.0.0.1", cfg["port"]), H).serve_forever()
"""
)
@@ -108,6 +116,12 @@ def fake(tmp_path, monkeypatch, hub):
_add(hub, "Qwen3-ASR-0.6B-GGUF/qwen3-asr-0.6b-q8_0.gguf", "qwen3_asr")
_add(hub, "MOSS-Transcribe-Diarize-GGUF/moss-q8_0.gguf", "moss_transcribe_diarize")
_add(hub, "Niagara-ASR-GGUF/niagara-19m-batch.en-f32.gguf", "niagara_asr")
+ _add(
+ hub,
+ "Nemotron-3.5-ASR-Streaming-0.6B-GGUF/nemotron-3.5-asr-streaming-0.6b-q8_0.gguf",
+ "nemotron_asr",
+ )
+ _add(hub, "Parakeet-TDT-0.6B-v3-GGUF/parakeet-tdt-0.6b-v3-q8_0.gguf", "parakeet_tdt")
root = tmp_path / "fake"
root.mkdir()
for log in ("starts.jsonl", "requests.jsonl"):
@@ -158,16 +172,106 @@ def test_bytes_go_as_json_naming_a_temp_file_removed_after_success_or_error(fake
assert record["path"] == "/v1/audio/transcriptions/details"
assert audio.is_absolute() and audio.parent == stt._stt_tmp_dir() and audio.suffix == ".wav"
assert record["audio_existed"] and not list(audio.parent.glob("*.wav"))
- assert (body["language"], body["model"]) == ("en", fake.starts[0]["entry"]["id"])
+ # Qwen3-ASR prompts with the language's name; the result keeps the code the caller gave.
+ assert (body["language"], body["model"]) == ("English", fake.starts[0]["entry"]["id"])
assert "options" not in body
with pytest.raises(stt.SttEngineUnavailableError):
- side.transcribe(wav_bytes(), QWEN3, "boom")
+ side.transcribe(wav_bytes(), NIAGARA, "boom")
assert not list(audio.parent.glob("*.wav"))
# A family that rejects the language is asked again without it.
- assert side.transcribe(wav_bytes(), QWEN3, "xx")["text"] == "Concord returned."
+ assert side.transcribe(wav_bytes(), NIAGARA, "xx")["text"] == "Concord returned."
assert fake.bodies[-2]["language"] == "xx" and "language" not in fake.bodies[-1]
+ABSENT = object()
+
+
+ ("language", "sent"),
+ [
+ ("en", "English"),
+ ("en-US", "English"),
+ ("EN_us", "English"),
+ ("fr-FR", "French"),
+ ("yue", "Cantonese"),
+ ("fil", "Filipino"),
+ ("cmn", "Chinese"),
+ # Not in Qwen3-ASR's list: the model detects it rather than reading a stray code.
+ ("sw", ABSENT),
+ ("he", ABSENT),
+ ("auto", ABSENT),
+ (None, ABSENT),
+ ],
+)
+def test_qwen3_gets_the_language_name_it_prompts_with(fake, side, source, language, sent):
+ result = side.transcribe_path(source, QWEN3, language)
+ (body,) = fake.bodies
+ assert body.get("language", ABSENT) == sent
+ assert result["language"] == (stt.normalize_whisper_language(language) or "English")
+
+
+ ("language", "sent"),
+ [
+ # The prompt dictionary has no bare key for these; the spec locale is the only one.
+ ("ja", "ja-JP"),
+ ("zh", "zh-CN"),
+ ("zh-TW", "zh-CN"),
+ ("he", "he-IL"),
+ ("iw", "he-IL"),
+ ("vi", "vi-VN"),
+ ("th", "th-TH"),
+ ("mt", "mt-MT"),
+ ("de", "de-DE"),
+ # A locale the model lists goes as it is.
+ ("en-GB", "en-GB"),
+ ("EN_us", "en-US"),
+ ("pt-BR", "pt-BR"),
+ ("nb", "nb-NO"),
+ ("nn-NO", "nn-NO"),
+ # Several locales and none chosen: the model keys the bare code itself.
+ ("en", "en"),
+ ("en-AU", "en"),
+ ("es", "es"),
+ ("fr", "fr"),
+ ("pt", "pt"),
+ ("no", "no"),
+ ("no-NO", "no"),
+ # Outside the model's locales: its default prompt (auto) rather than a refused request.
+ ("sw", ABSENT),
+ ("id", ABSENT),
+ ("auto", ABSENT),
+ (None, ABSENT),
+ ],
+)
+def test_nemotron_gets_a_locale_its_prompt_dictionary_keys(fake, side, source, language, sent):
+ result = side.transcribe_path(source, NEMOTRON, language)
+ (body,) = fake.bodies
+ assert body.get("language", ABSENT) == sent
+ if language not in (None, "auto"):
+ assert result["language"] == stt.normalize_whisper_language(language)
+
+
+def test_other_families_keep_the_bare_code(fake, side, source):
+ side.transcribe_path(source, NIAGARA, "ja-JP")
+ side.transcribe_path(source, NIAGARA, "en")
+ side.transcribe_path(source, PARAKEET, "en-US")
+ assert [body["language"] for body in fake.bodies] == ["ja", "en", "en"]
+
+
+def test_parakeet_switches_to_bounded_windows_on_long_audio(fake, side, source):
+ side.transcribe_path(source, PARAKEET, None)
+ (start,) = fake.starts
+ # Full context below the encoder ceiling: the windows lose words on clips it can encode whole.
+ assert start["entry"]["session_options"] == {
+ "parakeet_tdt.offline_mode": "auto",
+ "parakeet_tdt.audio_chunk_threshold_sec": "390",
+ }
+ side.unload()
+ side.transcribe_path(source, NEMOTRON, None)
+ assert "session_options" not in fake.starts[-1]["entry"]
+
+
def test_moss_reads_the_source_in_place_with_speakers_and_no_options(fake, side, source):
result = side.transcribe_path(source, MOSS, None, timestamps = True)
(body,) = fake.bodies
diff --git a/studio/frontend/src/features/audio/audio-cpp-catalog.ts b/studio/frontend/src/features/audio/audio-cpp-catalog.ts
index 07ffd2253ee..c51d740f40c 100644
--- a/studio/frontend/src/features/audio/audio-cpp-catalog.ts
+++ b/studio/frontend/src/features/audio/audio-cpp-catalog.ts
@@ -89,6 +89,46 @@ const VOXTRAL_LANGUAGES = [
"hi",
"ko",
] as const;
+// Primaries of audio.cpp model_specs/nemotron_asr.json, plus "no", the picker's code for nb/nn.
+const NEMOTRON_ASR_LANGUAGES = [
+ "ar",
+ "bg",
+ "cs",
+ "da",
+ "de",
+ "el",
+ "en",
+ "es",
+ "et",
+ "fi",
+ "fr",
+ "he",
+ "hi",
+ "hr",
+ "hu",
+ "it",
+ "ja",
+ "ko",
+ "lt",
+ "lv",
+ "mt",
+ "nb",
+ "nl",
+ "nn",
+ "no",
+ "pl",
+ "pt",
+ "ro",
+ "ru",
+ "sk",
+ "sl",
+ "sv",
+ "th",
+ "tr",
+ "uk",
+ "vi",
+ "zh",
+] as const;
export const AUDIO_CPP_MODELS: readonly AudioCppModel[] = [
{ id: folder("Kokoro-82M-GGUF"), task: "tts", needsEspeak: true },
@@ -175,7 +215,11 @@ export const AUDIO_CPP_MODELS: readonly AudioCppModel[] = [
{ id: folder("Parakeet-TDT-0.6B-v3-GGUF"), task: "asr" },
{ id: folder("Canary-180M-Flash-GGUF"), task: "asr", languages: ["en", "de", "es", "fr"] },
{ id: folder("Moonshine-Streaming-GGUF"), task: "asr", languages: ENGLISH },
- { id: folder("Nemotron-3.5-ASR-Streaming-0.6B-GGUF"), task: "asr", languages: ENGLISH },
+ {
+ id: folder("Nemotron-3.5-ASR-Streaming-0.6B-GGUF"),
+ task: "asr",
+ languages: NEMOTRON_ASR_LANGUAGES,
+ },
// Diarizes; Transcribe offers it for the Speakers switch.
{ id: folder("MOSS-Transcribe-Diarize-GGUF"), task: "asr" },
{ id: folder("Citrinet-ASR-GGUF"), task: "asr", languages: ENGLISH },
diff --git a/studio/frontend/src/features/settings/stores/stt-model-catalog.ts b/studio/frontend/src/features/settings/stores/stt-model-catalog.ts
index 3c4363ea5da..22aa2bccaa8 100644
--- a/studio/frontend/src/features/settings/stores/stt-model-catalog.ts
+++ b/studio/frontend/src/features/settings/stores/stt-model-catalog.ts
@@ -66,8 +66,8 @@ export const STT_PICKER_MODELS: readonly SttModel[] = [
...AUDIO_CPP_STT_FOLDER_IDS,
];
/** Models that transcribe only these primary languages; every other model is
- * multilingual. Moonshine and Nemotron are English-only, Canary covers
- * en/de/es/fr, and Hviske is Danish. */
+ * multilingual. Moonshine is English-only, Nemotron covers its spec's locales,
+ * Canary covers en/de/es/fr, and Hviske is Danish. */
export const STT_MODEL_LANGUAGES: ReadonlyMap<SttModel, readonly string[]> =
new Map(
[
diff --git a/studio/frontend/tests/audio-cpp-catalog.test.ts b/studio/frontend/tests/audio-cpp-catalog.test.ts
index cb084bcfc2a..3ad308a2e21 100644
--- a/studio/frontend/tests/audio-cpp-catalog.test.ts
+++ b/studio/frontend/tests/audio-cpp-catalog.test.ts
@@ -44,6 +44,7 @@ import {
sttRepoIdForSidecarKey,
sttSidecarKeyFor,
} from "../src/features/audio/stt-artifacts.ts";
+import { transcribeLanguagesFor } from "../src/features/audio/transcribe-languages.ts";
import {
audioPickIsRoutable,
audioPipelineTagFor,
@@ -720,7 +721,6 @@ test("English-only and partial-language ASR models are gated by language", () =>
"audiocpp-moonshine-tiny",
"audiocpp-moonshine-small",
"audiocpp-moonshine-medium",
- "audiocpp-nemotron-3.5-asr-0.6b",
`${AUDIO_CPP_REPO}/Citrinet-ASR-GGUF`,
`${AUDIO_CPP_REPO}/Granite-Speech-5.0-470M-TurboCTC-GGUF`,
`${AUDIO_CPP_REPO}/Higgs-Audio-v3-STT-GGUF`,
@@ -739,6 +739,20 @@ test("English-only and partial-language ASR models are gated by language", () =>
assert.equal(isSttModelLanguageCompatible(canary, language), true, language);
}
assert.equal(isSttModelLanguageCompatible(canary, "ja-JP"), false);
+ // Nemotron lists its spec's 40 locales; the backend sends the one its prompt dictionary keys.
+ const nemotron = "audiocpp-nemotron-3.5-asr-0.6b";
+ assert.ok(!ENGLISH_ONLY_STT_MODELS.has(nemotron));
+ for (const language of ["en-US", "ja-JP", "zh-CN", "nb-NO", "pt-BR", "auto"]) {
+ assert.equal(isSttModelLanguageCompatible(nemotron, language), true, language);
+ }
+ assert.equal(isSttModelLanguageCompatible(nemotron, "id-ID"), false);
+ assert.deepEqual(
+ transcribeLanguagesFor(
+ audioCppModelFor(`${AUDIO_CPP_REPO}/Nemotron-3.5-ASR-Streaming-0.6B-GGUF`)?.languages,
+ ).map((language) => language.code),
+ ["", "en", "zh", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "ar", "hi", "nl", "pl",
+ "tr", "uk", "vi", "th", "sv", "da", "fi", "no", "cs", "el", "he", "hu", "ro"],
+ );
const hviske = `${AUDIO_CPP_REPO}/Hviske-v5.3-GGUF`;
assert.equal(isSttModelLanguageCompatible(hviske, "da-DK"), true);
assert.equal(isSttModelLanguageCompatible(hviske, "auto"), true);