Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: send audio.cpp request fields in the form each model reads (#13791)
Etherl402a405f16a1
diff --git a/studio/backend/core/inference/audio_cpp_backend.py b/studio/backend/core/inference/audio_cpp_backend.pyindex 4d952693987..fd7824313ac 100644--- a/studio/backend/core/inference/audio_cpp_backend.py+++ b/studio/backend/core/inference/audio_cpp_backend.py@@ -750,11 +750,23 @@ class AudioCppBackend:body["voice"] = voicedefault_options.update(chosen)requested: dict[str, Any] = {}+ # A strict contract refuses undeclared options; the retry would then drop the description too.+ keys = model.request_keysif instructions and str(instructions).strip():- # Voice-design and style-capable families read the description as the instruction.- requested["instruct"] = str(instructions).strip()+ # Breeze and Irodori v4 call it instruction, DramaBox has neither.+ name = next((n for n in ("instruct", "instruction") if keys is None or n in keys), None)+ if name:+ requested[name] = str(instructions).strip()+ else:+ logger.info(+ "audio.cpp: %s declares no instruction option; not sending the description",+ model.family,+ )if language and str(language).strip():- requested["language"] = str(language).strip()+ if keys is None or "language" in keys:+ requested["language"] = str(language).strip()+ else:+ logger.info("audio.cpp: %s does not declare language; not sending it", model.family)merged = {**default_options, **requested}if merged:body["options"] = mergeddiff --git a/studio/backend/core/inference/audio_cpp_models.py b/studio/backend/core/inference/audio_cpp_models.pyindex 86629981283..18b905470e0 100644--- a/studio/backend/core/inference/audio_cpp_models.py+++ b/studio/backend/core/inference/audio_cpp_models.py@@ -909,11 +909,21 @@ _FAMILY_LIST: tuple[AudioCppFamily, ...] = (),),# Speech to text.+ # Full context fails past the ~400 s encoder ceiling; windows drop about a third of the words.+ AudioCppFamily(+ "parakeet_tdt",+ "asr",+ model_options = {+ "session_options": {+ "parakeet_tdt.offline_mode": "auto",+ "parakeet_tdt.audio_chunk_threshold_sec": "390",+ }+ },+ ),*(AudioCppFamily(name, "asr")for name in ("qwen3_asr",- "parakeet_tdt","canary_asr","citrinet_asr","cohere_asr",@@ -2597,6 +2607,10 @@ def _raw_request_options(spec: Optional[dict], embedded: Optional[dict]) -> listdef _request_keys(spec: Optional[dict], embedded: Optional[dict]) -> Optional[frozenset[str]]:+ # The server prefers an embedded v1 contract over model_specs/ (model_spec/package.cpp).+ if isinstance(embedded, dict) and embedded.get("schema_version") is not None:+ if (embedded.get("options") or {}).get("request"):+ spec = Noneif _request_source(spec, embedded).get("schema_version") is None:return Nonenames = frozenset(diff --git a/studio/backend/core/inference/stt_audiocpp_sidecar.py b/studio/backend/core/inference/stt_audiocpp_sidecar.pyindex 908084673e8..fcd224e6ce3 100644--- a/studio/backend/core/inference/stt_audiocpp_sidecar.py+++ b/studio/backend/core/inference/stt_audiocpp_sidecar.py@@ -14,6 +14,7 @@ the timestamps and speakers a model produces.from __future__ import annotations+import functoolsimport jsonimport subprocessimport tempfile@@ -55,6 +56,7 @@ from core.inference.audio_cpp_server import ()from core.inference.stt_ggml_sidecar import _pcm_to_wav_bytesfrom core.inference.stt_sidecar import (+ QWEN3_ASR_LANGUAGE_NAMES,STT_KEEP_ALIVE_SECONDS,SttAudioDecodeError,SttLoadCancelledError,@@ -95,6 +97,43 @@ _TIMESTAMP_OPTIONS = {"qwen3_asr.preserve_punctuation": "true",}+# model_specs/nemotron_asr.json "languages"; the prompt dictionary also keys en/es/fr/pt/no.+_NEMOTRON_ASR_LOCALES = (+ "ar-AR", "bg-BG", "cs-CZ", "da-DK", "de-DE", "el-GR", "en-GB", "en-US", "es-ES", "es-US",+ "et-EE", "fi-FI", "fr-CA", "fr-FR", "he-IL", "hi-IN", "hr-HR", "hu-HU", "it-IT", "ja-JP",+ "ko-KR", "lt-LT", "lv-LV", "mt-MT", "nb-NO", "nl-NL", "nn-NO", "pl-PL", "pt-BR", "pt-PT",+ "ro-RO", "ru-RU", "sk-SK", "sl-SI", "sv-SE", "th-TH", "tr-TR", "uk-UA", "vi-VN", "zh-CN",+) # fmt: skip++[email protected]_cache(maxsize = 256)+def _nemotron_language(language: str, code: str) -> Optional[str]:+ """The prompt key Nemotron takes for ``language``; any other string fails the request."""+ tag = language.strip().replace("_", "-").lower()+ primary = tag.split("-", 1)[0]+ for matches in (+ [locale for locale in _NEMOTRON_ASR_LOCALES if locale.lower() == tag],+ [locale for locale in _NEMOTRON_ASR_LOCALES if locale.split("-", 1)[0].lower() == primary],+ [locale for locale in _NEMOTRON_ASR_LOCALES if normalize_whisper_language(locale) == code],+ ):+ if len(matches) == 1:+ return matches[0]+ if matches:+ return code+ return None+++def _runtime_language(family: str, language: Optional[str]) -> Optional[str]:+ """``language`` in the form ``family`` reads it, or None to let the model detect it."""+ code = normalize_whisper_language(language)+ if not code:+ return None+ if family == "qwen3_asr":+ return QWEN3_ASR_LANGUAGE_NAMES.get(code)+ if family == "nemotron_asr":+ return _nemotron_language(language, code)+ return code+class SttEngineUnavailableError(SttUnavailableError):"""audiocpp_server is not installed or stopped serving; the audio.cpp dictation engine is off."""@@ -998,7 +1037,10 @@ class AudioCppSttSidecar:)_notify(on_phase, "transcribing")payload = self._post_details(- path, lang, cancel_event, dict(_TIMESTAMP_OPTIONS) if aligned else {}+ path,+ _runtime_language(entry.family, language),+ cancel_event,+ dict(_TIMESTAMP_OPTIONS) if aligned else {},)if cancel_event is not None and cancel_event.is_set():raise SttTranscriptionCancelledError("Transcription cancelled.")@@ -1057,8 +1099,7 @@ class AudioCppSttSidecar:except AudioCppRequestError as exc:if "language" not in body or not 400 <= exc.status < 500:raise- # English-only families (Moonshine, Nemotron, Parakeet) reject a language option; a- # dictation language preference is a hint, so transcribe without it rather than fail.+ # A dictation language is a hint: transcribe without it rather than fail.logger.info("audio.cpp: %s rejected language %r (%s); retrying without it",server.model.family,diff --git a/studio/backend/core/inference/stt_sidecar.py b/studio/backend/core/inference/stt_sidecar.pyindex 55b72ebb324..8699ae1c751 100644--- a/studio/backend/core/inference/stt_sidecar.py+++ b/studio/backend/core/inference/stt_sidecar.py@@ -266,6 +266,41 @@ def normalize_whisper_language(language: Optional[str]) -> Optional[str]:return _WHISPER_LANGUAGE_ALIASES.get(primary, primary)+# Whisper code -> Qwen3-ASR prompt name (config.support_languages; audio.cpp qwen3_asr/assets.cpp).+QWEN3_ASR_LANGUAGE_NAMES = {+ "ar": "Arabic",+ "cs": "Czech",+ "da": "Danish",+ "de": "German",+ "el": "Greek",+ "en": "English",+ "es": "Spanish",+ "fa": "Persian",+ "fi": "Finnish",+ "fr": "French",+ "hi": "Hindi",+ "hu": "Hungarian",+ "id": "Indonesian",+ "it": "Italian",+ "ja": "Japanese",+ "ko": "Korean",+ "mk": "Macedonian",+ "ms": "Malay",+ "nl": "Dutch",+ "pl": "Polish",+ "pt": "Portuguese",+ "ro": "Romanian",+ "ru": "Russian",+ "sv": "Swedish",+ "th": "Thai",+ "tl": "Filipino",+ "tr": "Turkish",+ "vi": "Vietnamese",+ "yue": "Cantonese",+ "zh": "Chinese",+}++def _stt_quiet_window_end(decoded_audio, start: int, window: int) -> Optional[int]:"""Return a nearby quiet window boundary, or None when speech fills it."""import numpy as npdiff --git a/studio/backend/core/inference/stt_transformers_worker.py b/studio/backend/core/inference/stt_transformers_worker.pyindex cf66290e003..b6172deb9b5 100644--- a/studio/backend/core/inference/stt_transformers_worker.py+++ b/studio/backend/core/inference/stt_transformers_worker.py@@ -197,44 +197,11 @@ def load_mlx_stt(return _MlxSttModel(model)-# Whisper language codes -> the names Qwen3-ASR lists in config.support_languages.-_MLX_LANGUAGE_NAMES = {- "ar": "Arabic",- "cs": "Czech",- "da": "Danish",- "de": "German",- "el": "Greek",- "en": "English",- "es": "Spanish",- "fa": "Persian",- "fi": "Finnish",- "fr": "French",- "hi": "Hindi",- "hu": "Hungarian",- "id": "Indonesian",- "it": "Italian",- "ja": "Japanese",- "ko": "Korean",- "mk": "Macedonian",- "ms": "Malay",- "nl": "Dutch",- "pl": "Polish",- "pt": "Portuguese",- "ro": "Romanian",- "ru": "Russian",- "sv": "Swedish",- "th": "Thai",- "tl": "Filipino",- "tr": "Turkish",- "vi": "Vietnamese",- "yue": "Cantonese",- "zh": "Chinese",-}--def _mlx_language(model, code: Optional[str]) -> Optional[str]:"""The checkpoint's own name for a Whisper language code, or None to let it detect one."""- name = _MLX_LANGUAGE_NAMES.get(code or "", "").lower()+ from core.inference.stt_sidecar import QWEN3_ASR_LANGUAGE_NAMES++ name = QWEN3_ASR_LANGUAGE_NAMES.get(code or "", "").lower()supported = getattr(getattr(model, "config", None), "support_languages", None) or []return next((lang for lang in supported if name and str(lang).lower() == name), None)diff --git a/studio/backend/tests/test_audio_cpp_clone_requests.py b/studio/backend/tests/test_audio_cpp_clone_requests.pyindex e8af6d05c95..fc3a26e89f7 100644--- a/studio/backend/tests/test_audio_cpp_clone_requests.py+++ b/studio/backend/tests/test_audio_cpp_clone_requests.py@@ -167,6 +167,88 @@ def test_voxcpm2_speak_body_is_unchanged_by_clone_support():assert server.calls == [(SPEECH, {**body, "seed": "7"})]+class _StrictRecorder(_Recorder):+ """Refuses an option the model's contract does not declare, as the runtime does."""++ def post_json(self, path, payload, **kwargs):+ keys = self.model.request_keys+ undeclared = sorted(set(payload.get("options") or {}) - set(keys or ()))+ if keys is not None and undeclared:+ self.calls.append((path, json.loads(json.dumps(payload))))+ raise srv.AudioCppRequestError(500, f"unknown request option: {undeclared[0]}")+ return super().post_json(path, payload, **kwargs)+++# Request option names each published GGUF's embedded (strict) spec declares.+STRICT_KEYS = {+ "Breeze-TTS-2-GGUF": ("breeze_tts", {"instruction", "reference_text", "guidance_scale", "seed"}),+ "Irodori-TTS-v4-Small-GGUF": ("irodori_tts", {"instruction", "language", "no_ref", "seed"}),+ "DramaBox-GGUF": ("dramabox", {"negative_prompt", "target_voice", "guidance_scale", "seed"}),+ "DotTTS-MF-GGUF": ("dots_tts", {"language", "reference_text", "template_name", "seed"}),+ "DotTTS-Edit-GGUF": ("dots_tts", {"instruct", "instruction", "language", "template_name"}),+ "Maya1-GGUF": ("maya1", {"instruct", "max_tokens", "temperature", "seed"}),+}+++# fmt: off[email protected]("display, language, options", [+ ("Breeze-TTS-2-GGUF", None, {"instruction": "warm"}),+ ("Breeze-TTS-2-GGUF", "en", {"instruction": "warm"}),+ ("Irodori-TTS-v4-Small-GGUF", None, {"instruction": "warm"}),+ ("Irodori-TTS-v4-Small-GGUF", "ja", {"instruction": "warm", "language": "ja"}),+ ("DramaBox-GGUF", "en", ABSENT),+ ("DotTTS-MF-GGUF", "en", {"language": "en"}),+ ("DotTTS-Edit-GGUF", None, {"instruct": "warm"}),+ ("Maya1-GGUF", "en", {"instruct": "warm"}),+])+# fmt: on+def test_speak_sends_the_description_under_the_name_the_contract_declares(display, language, options):+ family, keys = STRICT_KEYS[display]+ model = replace(_model(family, display), request_keys = frozenset(keys))+ backend = acb.AudioCppBackend()+ server = _StrictRecorder(model)+ backend._server, backend._model = server, model+ backend.models, backend.active_model_name = {model.id: {"is_audio": True}}, model.id+ backend.generate_audio_response(TEXT, language = language, instructions = "warm")+ # One request: nothing the contract lacks is sent, so there is no refusal to retry after.+ ((path, body),) = server.calls+ assert path == SPEECH and body.get("options", ABSENT) == options+++def _spec(*names, strict = True):+ spec = {"family": "dots_tts", "options": {"request": [{"name": n, "type": "string"} for n in names]}}+ return {**spec, "schema_version": 1} if strict else spec+++# fmt: off[email protected]("runtime, embedded, keys", [+ # DotTTS-MF: model_specs/ declares instruct, the GGUF's v1 contract (what the server checks) does not.+ (_spec("instruct", "instruction", "language"), _spec("language", "seed"), {"language", "seed"}),+ # A legacy GGUF is checked against model_specs/, so that list still wins.+ (_spec("instruct", "language"), _spec("language", strict = False), {"instruct", "language"}),+ # A v1 GGUF that lists no request options leaves the runtime's list in charge, as before.+ (_spec("instruct", "language"), {**_spec(), "options": {}}, {"instruct", "language"}),+ (None, _spec("language"), {"language"}),+ (None, _spec("language", strict = False), None),+])+# fmt: on+def test_request_keys_follow_the_contract_the_server_checks(runtime, embedded, keys):+ assert acm._request_keys(runtime, embedded) == (frozenset(keys) if keys else None)+++def test_speak_on_a_runtime_with_model_specs_sends_only_what_the_gguf_declares():+ runtime = _spec("instruct", "instruction", "language")+ embedded = _spec("language", "template_name", "seed")+ model = replace(_model("dots_tts", "DotTTS-MF-GGUF"), request_keys = acm._request_keys(runtime, embedded))+ backend = acb.AudioCppBackend()+ server = _StrictRecorder(model)+ backend._server, backend._model = server, model+ backend.models, backend.active_model_name = {model.id: {"is_audio": True}}, model.id+ backend.generate_audio_response(TEXT, language = "en", instructions = "warm")+ ((path, body),) = server.calls+ assert path == SPEECH and body["options"] == {"language": "en"}++# fmt: off@pytest.mark.parametrize("key, text, sent", [("vibevoice", "Hello there.", "Speaker 1: Hello there."),diff --git a/studio/backend/tests/test_stt_audiocpp_details.py b/studio/backend/tests/test_stt_audiocpp_details.pyindex e024580a05d..1b1f0681ebc 100644--- a/studio/backend/tests/test_stt_audiocpp_details.py+++ b/studio/backend/tests/test_stt_audiocpp_details.py@@ -24,12 +24,14 @@ from test_audio_inputs import wav_bytes # noqa: E402QWEN3 = f"{AUDIO_CPP_REPO}/Qwen3-ASR-0.6B-GGUF"MOSS = f"{AUDIO_CPP_REPO}/MOSS-Transcribe-Diarize-GGUF"NIAGARA = f"{AUDIO_CPP_REPO}/Niagara-ASR-GGUF"+NEMOTRON = f"{AUDIO_CPP_REPO}/Nemotron-3.5-ASR-Streaming-0.6B-GGUF"+PARAKEET = f"{AUDIO_CPP_REPO}/Parakeet-TDT-0.6B-v3-GGUF"ALIGNER_FILE = "Qwen3-ForcedAligner-0.6B-GGUF/qwen3-forced-aligner-0.6b-q8_0.gguf"ALIGNER_KEY = "qwen3_asr.forced_aligner_model_path"FAKE_SERVER = textwrap.dedent(r"""- import json, os, sys, time+ import json, os, socketserver, sys, timefrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServerhere = os.path.dirname(os.path.abspath(__file__))@@ -79,7 +81,13 @@ FAKE_SERVER = textwrap.dedent(]self.reply(200, payload)- ThreadingHTTPServer(("127.0.0.1", cfg["port"]), H).serve_forever()+ class S(ThreadingHTTPServer):+ # HTTPServer.server_bind reverse-resolves the host via getfqdn, ~35 s per start on macOS runners.+ def server_bind(self):+ socketserver.TCPServer.server_bind(self)+ self.server_name, self.server_port = "127.0.0.1", self.server_address[1]++ S(("127.0.0.1", cfg["port"]), H).serve_forever()""")@@ -108,6 +116,12 @@ def fake(tmp_path, monkeypatch, hub):_add(hub, "Qwen3-ASR-0.6B-GGUF/qwen3-asr-0.6b-q8_0.gguf", "qwen3_asr")_add(hub, "MOSS-Transcribe-Diarize-GGUF/moss-q8_0.gguf", "moss_transcribe_diarize")_add(hub, "Niagara-ASR-GGUF/niagara-19m-batch.en-f32.gguf", "niagara_asr")+ _add(+ hub,+ "Nemotron-3.5-ASR-Streaming-0.6B-GGUF/nemotron-3.5-asr-streaming-0.6b-q8_0.gguf",+ "nemotron_asr",+ )+ _add(hub, "Parakeet-TDT-0.6B-v3-GGUF/parakeet-tdt-0.6b-v3-q8_0.gguf", "parakeet_tdt")root = tmp_path / "fake"root.mkdir()for log in ("starts.jsonl", "requests.jsonl"):@@ -158,16 +172,106 @@ def test_bytes_go_as_json_naming_a_temp_file_removed_after_success_or_error(fakeassert record["path"] == "/v1/audio/transcriptions/details"assert audio.is_absolute() and audio.parent == stt._stt_tmp_dir() and audio.suffix == ".wav"assert record["audio_existed"] and not list(audio.parent.glob("*.wav"))- assert (body["language"], body["model"]) == ("en", fake.starts[0]["entry"]["id"])+ # Qwen3-ASR prompts with the language's name; the result keeps the code the caller gave.+ assert (body["language"], body["model"]) == ("English", fake.starts[0]["entry"]["id"])assert "options" not in bodywith pytest.raises(stt.SttEngineUnavailableError):- side.transcribe(wav_bytes(), QWEN3, "boom")+ side.transcribe(wav_bytes(), NIAGARA, "boom")assert not list(audio.parent.glob("*.wav"))# A family that rejects the language is asked again without it.- assert side.transcribe(wav_bytes(), QWEN3, "xx")["text"] == "Concord returned."+ assert side.transcribe(wav_bytes(), NIAGARA, "xx")["text"] == "Concord returned."assert fake.bodies[-2]["language"] == "xx" and "language" not in fake.bodies[-1]+ABSENT = object()+++ ("language", "sent"),+ [+ ("en", "English"),+ ("en-US", "English"),+ ("EN_us", "English"),+ ("fr-FR", "French"),+ ("yue", "Cantonese"),+ ("fil", "Filipino"),+ ("cmn", "Chinese"),+ # Not in Qwen3-ASR's list: the model detects it rather than reading a stray code.+ ("sw", ABSENT),+ ("he", ABSENT),+ ("auto", ABSENT),+ (None, ABSENT),+ ],+)+def test_qwen3_gets_the_language_name_it_prompts_with(fake, side, source, language, sent):+ result = side.transcribe_path(source, QWEN3, language)+ (body,) = fake.bodies+ assert body.get("language", ABSENT) == sent+ assert result["language"] == (stt.normalize_whisper_language(language) or "English")+++ ("language", "sent"),+ [+ # The prompt dictionary has no bare key for these; the spec locale is the only one.+ ("ja", "ja-JP"),+ ("zh", "zh-CN"),+ ("zh-TW", "zh-CN"),+ ("he", "he-IL"),+ ("iw", "he-IL"),+ ("vi", "vi-VN"),+ ("th", "th-TH"),+ ("mt", "mt-MT"),+ ("de", "de-DE"),+ # A locale the model lists goes as it is.+ ("en-GB", "en-GB"),+ ("EN_us", "en-US"),+ ("pt-BR", "pt-BR"),+ ("nb", "nb-NO"),+ ("nn-NO", "nn-NO"),+ # Several locales and none chosen: the model keys the bare code itself.+ ("en", "en"),+ ("en-AU", "en"),+ ("es", "es"),+ ("fr", "fr"),+ ("pt", "pt"),+ ("no", "no"),+ ("no-NO", "no"),+ # Outside the model's locales: its default prompt (auto) rather than a refused request.+ ("sw", ABSENT),+ ("id", ABSENT),+ ("auto", ABSENT),+ (None, ABSENT),+ ],+)+def test_nemotron_gets_a_locale_its_prompt_dictionary_keys(fake, side, source, language, sent):+ result = side.transcribe_path(source, NEMOTRON, language)+ (body,) = fake.bodies+ assert body.get("language", ABSENT) == sent+ if language not in (None, "auto"):+ assert result["language"] == stt.normalize_whisper_language(language)+++def test_other_families_keep_the_bare_code(fake, side, source):+ side.transcribe_path(source, NIAGARA, "ja-JP")+ side.transcribe_path(source, NIAGARA, "en")+ side.transcribe_path(source, PARAKEET, "en-US")+ assert [body["language"] for body in fake.bodies] == ["ja", "en", "en"]+++def test_parakeet_switches_to_bounded_windows_on_long_audio(fake, side, source):+ side.transcribe_path(source, PARAKEET, None)+ (start,) = fake.starts+ # Full context below the encoder ceiling: the windows lose words on clips it can encode whole.+ assert start["entry"]["session_options"] == {+ "parakeet_tdt.offline_mode": "auto",+ "parakeet_tdt.audio_chunk_threshold_sec": "390",+ }+ side.unload()+ side.transcribe_path(source, NEMOTRON, None)+ assert "session_options" not in fake.starts[-1]["entry"]++def test_moss_reads_the_source_in_place_with_speakers_and_no_options(fake, side, source):result = side.transcribe_path(source, MOSS, None, timestamps = True)(body,) = fake.bodiesdiff --git a/studio/frontend/src/features/audio/audio-cpp-catalog.ts b/studio/frontend/src/features/audio/audio-cpp-catalog.tsindex 07ffd2253ee..c51d740f40c 100644--- a/studio/frontend/src/features/audio/audio-cpp-catalog.ts+++ b/studio/frontend/src/features/audio/audio-cpp-catalog.ts@@ -89,6 +89,46 @@ const VOXTRAL_LANGUAGES = ["hi","ko",] as const;+// Primaries of audio.cpp model_specs/nemotron_asr.json, plus "no", the picker's code for nb/nn.+const NEMOTRON_ASR_LANGUAGES = [+ "ar",+ "bg",+ "cs",+ "da",+ "de",+ "el",+ "en",+ "es",+ "et",+ "fi",+ "fr",+ "he",+ "hi",+ "hr",+ "hu",+ "it",+ "ja",+ "ko",+ "lt",+ "lv",+ "mt",+ "nb",+ "nl",+ "nn",+ "no",+ "pl",+ "pt",+ "ro",+ "ru",+ "sk",+ "sl",+ "sv",+ "th",+ "tr",+ "uk",+ "vi",+ "zh",+] as const;export const AUDIO_CPP_MODELS: readonly AudioCppModel[] = [{ id: folder("Kokoro-82M-GGUF"), task: "tts", needsEspeak: true },@@ -175,7 +215,11 @@ export const AUDIO_CPP_MODELS: readonly AudioCppModel[] = [{ id: folder("Parakeet-TDT-0.6B-v3-GGUF"), task: "asr" },{ id: folder("Canary-180M-Flash-GGUF"), task: "asr", languages: ["en", "de", "es", "fr"] },{ id: folder("Moonshine-Streaming-GGUF"), task: "asr", languages: ENGLISH },- { id: folder("Nemotron-3.5-ASR-Streaming-0.6B-GGUF"), task: "asr", languages: ENGLISH },+ {+ id: folder("Nemotron-3.5-ASR-Streaming-0.6B-GGUF"),+ task: "asr",+ languages: NEMOTRON_ASR_LANGUAGES,+ },// Diarizes; Transcribe offers it for the Speakers switch.{ id: folder("MOSS-Transcribe-Diarize-GGUF"), task: "asr" },{ id: folder("Citrinet-ASR-GGUF"), task: "asr", languages: ENGLISH },diff --git a/studio/frontend/src/features/settings/stores/stt-model-catalog.ts b/studio/frontend/src/features/settings/stores/stt-model-catalog.tsindex 3c4363ea5da..22aa2bccaa8 100644--- a/studio/frontend/src/features/settings/stores/stt-model-catalog.ts+++ b/studio/frontend/src/features/settings/stores/stt-model-catalog.ts@@ -66,8 +66,8 @@ export const STT_PICKER_MODELS: readonly SttModel[] = [...AUDIO_CPP_STT_FOLDER_IDS,];/** Models that transcribe only these primary languages; every other model is- * multilingual. Moonshine and Nemotron are English-only, Canary covers- * en/de/es/fr, and Hviske is Danish. */+ * multilingual. Moonshine is English-only, Nemotron covers its spec's locales,+ * Canary covers en/de/es/fr, and Hviske is Danish. */export const STT_MODEL_LANGUAGES: ReadonlyMap<SttModel, readonly string[]> =new Map([diff --git a/studio/frontend/tests/audio-cpp-catalog.test.ts b/studio/frontend/tests/audio-cpp-catalog.test.tsindex cb084bcfc2a..3ad308a2e21 100644--- a/studio/frontend/tests/audio-cpp-catalog.test.ts+++ b/studio/frontend/tests/audio-cpp-catalog.test.ts@@ -44,6 +44,7 @@ import {sttRepoIdForSidecarKey,sttSidecarKeyFor,} from "../src/features/audio/stt-artifacts.ts";+import { transcribeLanguagesFor } from "../src/features/audio/transcribe-languages.ts";import {audioPickIsRoutable,audioPipelineTagFor,@@ -720,7 +721,6 @@ test("English-only and partial-language ASR models are gated by language", () =>"audiocpp-moonshine-tiny","audiocpp-moonshine-small","audiocpp-moonshine-medium",- "audiocpp-nemotron-3.5-asr-0.6b",`${AUDIO_CPP_REPO}/Citrinet-ASR-GGUF`,`${AUDIO_CPP_REPO}/Granite-Speech-5.0-470M-TurboCTC-GGUF`,`${AUDIO_CPP_REPO}/Higgs-Audio-v3-STT-GGUF`,@@ -739,6 +739,20 @@ test("English-only and partial-language ASR models are gated by language", () =>assert.equal(isSttModelLanguageCompatible(canary, language), true, language);}assert.equal(isSttModelLanguageCompatible(canary, "ja-JP"), false);+ // Nemotron lists its spec's 40 locales; the backend sends the one its prompt dictionary keys.+ const nemotron = "audiocpp-nemotron-3.5-asr-0.6b";+ assert.ok(!ENGLISH_ONLY_STT_MODELS.has(nemotron));+ for (const language of ["en-US", "ja-JP", "zh-CN", "nb-NO", "pt-BR", "auto"]) {+ assert.equal(isSttModelLanguageCompatible(nemotron, language), true, language);+ }+ assert.equal(isSttModelLanguageCompatible(nemotron, "id-ID"), false);+ assert.deepEqual(+ transcribeLanguagesFor(+ audioCppModelFor(`${AUDIO_CPP_REPO}/Nemotron-3.5-ASR-Streaming-0.6B-GGUF`)?.languages,+ ).map((language) => language.code),+ ["", "en", "zh", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "ar", "hi", "nl", "pl",+ "tr", "uk", "vi", "th", "sv", "da", "fi", "no", "cs", "el", "he", "hu", "ro"],+ );const hviske = `${AUDIO_CPP_REPO}/Hviske-v5.3-GGUF`;assert.equal(isSttModelLanguageCompatible(hviske, "da-DK"), true);assert.equal(isSttModelLanguageCompatible(hviske, "auto"), true);