Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: add audio translations and list audio workflows in /v1/models (#12817)

Etherl7db04e4e648b
diff --git a/studio/backend/core/inference/stt_sidecar.py b/studio/backend/core/inference/stt_sidecar.py
index 7b0e4c9893e..2682910464f 100644
--- a/studio/backend/core/inference/stt_sidecar.py
+++ b/studio/backend/core/inference/stt_sidecar.py
@@ -343,6 +343,12 @@ def resolve_model_repo(model_id: str) -> str:
return STT_MODELS.get(resolved, resolved)
+def can_translate(model: Optional[str]) -> bool:
+ """Whether Whisper's translate task works on this id. The turbo checkpoints were fine-tuned for transcription
+ only and answer in the source language when asked to translate."""
+ return "turbo" not in (model or DEFAULT_STT_MODEL).lower()
+
+
def _is_whisper_config(config: object) -> bool:
"""True when Hub/local config metadata identifies a Whisper ASR model."""
if not isinstance(config, dict):
@@ -1720,8 +1726,9 @@ class WhisperSttSidecar:
fast: bool = False,
cancel_event: Optional[threading.Event] = None,
on_progress = None,
+ task: str = "transcribe",
) -> dict:
- """Transcribe encoded audio bytes to text.
+ """Transcribe encoded audio bytes to text, or to English text with ``task="translate"``.
Accepts any container PyAV can decode: wav, mp3, opus/webm, ogg,
m4a/aac. Returns {text, language, duration, model}.
@@ -1744,12 +1751,15 @@ class WhisperSttSidecar:
raise SttLanguageError(
f"Language '{language}' is not supported by English-only STT model '{model_id}'."
)
+ if cached.is_multilingual is False and task == "translate":
+ # The checkpoint's generation config pins the task, so it would only transcribe.
+ raise SttLanguageError(f"English-only STT model '{model_id}' cannot translate.")
decoded_audio = _decode_audio_bounded(audio, cancel_event)
if cancel_event is not None and cancel_event.is_set():
raise SttTranscriptionCancelledError("Transcription cancelled.")
# condition_on_prev_tokens=False stops a fresh clip inheriting prior context, which causes runaway repeats.
generate_kwargs = {
- "task": "transcribe",
+ "task": task,
"condition_on_prev_tokens": False,
"num_beams": 5,
}
diff --git a/studio/backend/main.py b/studio/backend/main.py
index 97d1dab44d8..a3c18eaae62 100644
--- a/studio/backend/main.py
+++ b/studio/backend/main.py
@@ -1395,6 +1395,8 @@ _DIFFUSION_DATASET_UPLOAD_PATH = "/api/train/diffusion/dataset"
_STT_MULTIPART_UPLOAD_PATHS = (
"/v1/audio/transcriptions",
"/api/inference/audio/transcriptions",
+ "/v1/audio/translations",
+ "/api/inference/audio/translations",
)
_VIDEO_MULTIPART_UPLOAD_PATHS = (
"/v1/videos",
@@ -1451,7 +1453,7 @@ def _get_request_body_max_bytes(path: str) -> int:
return STT_AUDIO_RAW_MAX_BYTES
if path.startswith("/api/inference/audio/transcribe"):
return STT_AUDIO_JSON_MAX_BYTES
- # multipart headroom over the raw stt cap for the openai transcription route on both mounts
+ # multipart headroom over the raw stt cap for the openai transcription/translation routes
if path.rstrip("/") in _STT_MULTIPART_UPLOAD_PATHS:
return upload_request_limit_bytes(STT_AUDIO_RAW_MAX_BYTES)
if path.rstrip("/") in _VIDEO_MULTIPART_UPLOAD_PATHS:
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 52bde1565f9..1c131b609ed 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -24206,6 +24206,75 @@ async def openai_audio_transcriptions(
return JSONResponse(content = {"text": text})
[email protected]("/audio/translations")
+async def openai_audio_translations(
+ request: Request,
+ file: UploadFile = File(...),
+ model: Optional[str] = Form(None),
+ response_format: str = Form("json"),
+ provider_id: Optional[str] = Form(None),
+ current_subject: str = Depends(get_current_subject),
+):
+ """OpenAI-compatible speech translation to English (POST /v1/audio/translations).
+
+ Runs on the local Whisper sidecar; ``whisper-1`` or nothing selects the default model.
+ ``response_format`` supports ``json``, ``text`` and ``verbose_json``. ``prompt`` and
+ ``temperature`` are accepted and ignored."""
+ fmt = (response_format or "json").strip().lower()
+ if fmt not in ("json", "text", "verbose_json"):
+ _raise_unsupported_openai_parameter(
+ "response_format",
+ f"Unsupported response_format '{response_format}'. Use 'json', 'text' or 'verbose_json'.",
+ )
+ if isinstance(provider_id, str):
+ _raise_unsupported_openai_parameter(
+ "provider_id",
+ "Translations run on a local Whisper model; provider_id is not supported.",
+ )
+ from core.inference.stt_sidecar import can_translate
+
+ sidecar_model = None if model in (None, "", "whisper-1") else model
+ # mtmd / audio.cpp engines and turbo checkpoints would just transcribe.
+ if _stt_engine_for_model(sidecar_model) is not None or not can_translate(sidecar_model):
+ raise HTTPException(
+ status_code = 400,
+ detail = openai_error_body(
+ f"'{model}' cannot translate. Use a multilingual Whisper model such as 'small'.",
+ status = 400,
+ code = "invalid_value",
+ param = "model",
+ ),
+ )
+ raw = await file.read(_MAX_AUDIO_RAW_BYTES + 1)
+ _opening_label = sidecar_model or "whisper-1"
+ async with _monitored_media_request(
+ request,
+ model = public_model_id(_opening_label) or _opening_label,
+ prompt = file.filename or "audio",
+ subject = current_subject,
+ ) as monitor_id:
+ result = await _transcribe_audio_result(
+ raw, sidecar_model, None, fast = False, request = request, translate = True
+ )
+ text = str(result.get("text", ""))
+ _served_model = str(result.get("model") or "")
+ api_monitor.relabel(monitor_id, public_model_id(_served_model) or _served_model)
+ api_monitor.set_reply(monitor_id, text)
+ if fmt == "text":
+ return PlainTextResponse(content = text)
+ if fmt == "verbose_json":
+ duration = result.get("duration")
+ return JSONResponse(
+ content = {
+ "task": "translate",
+ "language": "english",
+ "duration": float(duration) if isinstance(duration, (int, float)) else 0.0,
+ "text": text,
+ }
+ )
+ return JSONResponse(content = {"text": text})
+
+
# =====================================================================
# Speech-to-text (STT) sidecar (/audio/transcribe, /audio/stt/*)
@@ -24938,12 +25007,14 @@ async def _transcribe_audio_result(
*,
source_path: Optional[Path] = None,
timestamps: bool = False,
+ translate: bool = False,
) -> dict:
"""STT for already-decoded bytes, sidecar errors mapped to HTTP statuses.
Returns the sidecar's result dict so callers own the response shape.
``source_path`` (a prepared WAV inside the account) replaces ``raw``: audio.cpp reads it in
- place, with ``timestamps`` when asked; every other engine gets its bytes."""
+ place, with ``timestamps`` when asked; every other engine gets its bytes. ``translate`` is
+ Whisper's English translation; only the Transformers sidecar takes it."""
if account_access.managed_account():
await asyncio.to_thread(
account_access.require_model_access, _stt_repo_reference(model, engine)
@@ -24981,6 +25052,9 @@ async def _transcribe_audio_result(
model,
)
sidecar = _stt_sidecar_for(serving_engine)
+ transcribe = (
+ functools.partial(sidecar.transcribe, task = "translate") if translate else sidecar.transcribe
+ )
cancel_event = threading.Event() if request is not None or on_progress is not None else None
disconnect_watcher = (
asyncio.create_task(_await_stt_disconnect_then_cancel(request, sidecar, cancel_event))
@@ -25027,7 +25101,7 @@ async def _transcribe_audio_result(
)
elif on_progress is not None and serving_engine in ("transformers", "mtmd"):
result = await asyncio.to_thread(
- sidecar.transcribe,
+ transcribe,
raw,
model,
language,
@@ -25036,10 +25110,10 @@ async def _transcribe_audio_result(
on_progress = on_progress,
)
elif cancel_event is None:
- result = await asyncio.to_thread(sidecar.transcribe, raw, model, language, fast)
+ result = await asyncio.to_thread(transcribe, raw, model, language, fast)
else:
result = await asyncio.to_thread(
- sidecar.transcribe,
+ transcribe,
raw,
model,
language,
@@ -35108,6 +35182,9 @@ def _slot_model_objects() -> list[dict]:
Shared by the LIST and RETRIEVE handlers so both report the same ids and
field shape.
"""
+ from core.inference.audio_cpp_models import AUDIO_CPP_SEP_AUDIO_TYPE
+ from core.inference.audio_workflows import status_audio_workflows
+
if (
routed_slot.get() is None
and account_access.managed_account()
@@ -35176,6 +35253,7 @@ def _slot_model_objects() -> list[dict]:
getattr(llama_backend, "_audio_type", None) in _GGUF_TTS_AUDIO_TYPES
):
entry["task"] = _TTS_MODEL_TASK
+ entry["audio_workflows"] = status_audio_workflows(True, llama_backend._audio_type)
models.append(entry)
# Describing residency must not construct an unused orchestrator: its cold
@@ -35201,12 +35279,18 @@ def _slot_model_objects() -> list[dict]:
break
if _ctx is not None:
entry["context_length"] = _ctx
+ audio_type = model_info.get("audio_type")
+ is_sep = audio_type == AUDIO_CPP_SEP_AUDIO_TYPE
if (
not model_info.get("is_mlx")
and model_info.get("is_audio")
- and (model_info.get("audio_type") in _TRANSFORMERS_TTS_AUDIO_TYPES)
+ and (audio_type in _TRANSFORMERS_TTS_AUDIO_TYPES or is_sep)
):
- entry["task"] = _TTS_MODEL_TASK
+ # A resident separation model is not a chat model either.
+ entry["task"] = _SEP_MODEL_TASK if is_sep else _TTS_MODEL_TASK
+ entry["audio_workflows"] = model_info.get("audio_workflows") or (
+ status_audio_workflows(True, audio_type)
+ )
for _field in ("native_context_length", "max_context_length", "context_length_fitted"):
_value = _positive_int_or_none(model_info.get(_field))
@@ -35326,6 +35410,7 @@ def _classified_catalog(models: list) -> list:
_MEDIA_MODEL_TASKS = ("text-to-image", "text-to-video")
_STT_MODEL_TASK = "automatic-speech-recognition"
_TTS_MODEL_TASK = "text-to-speech"
+_SEP_MODEL_TASK = "audio-to-audio"
def _media_owner(task: str) -> str:
@@ -35575,6 +35660,13 @@ def _stt_model_objects(created: int, catalog_at: Optional[float] = None) -> list
"owned_by": _OWNED_BY,
"task": _STT_MODEL_TASK,
"loaded": model_id in loaded,
+ # Custom Whisper repos are not probed here, so only curated ones advertise translate.
+ "audio_workflows": (
+ ["transcribe", "translate"]
+ if model_id in stt_sidecar.STT_MODELS.values()
+ and stt_sidecar.can_translate(model_id)
+ else ["transcribe"]
+ ),
}
spec = stt_mtmd_sidecar.MTMD_STT_MODELS.get(model_id)
if spec is not None:
@@ -35747,6 +35839,8 @@ def _servable_catalog_scan(catalog, catalog_at: Optional[float]):
*_MEDIA_MODEL_TASKS,
_STT_MODEL_TASK,
_TTS_MODEL_TASK,
+ # A separation GGUF: listed by _audio_cpp_speech_model_objects with its workflows.
+ _SEP_MODEL_TASK,
_UNSUPPORTED_DIFFUSION_TASK,
):
continue
@@ -35808,9 +35902,10 @@ def _servable_catalog_rows(
def _audio_cpp_speech_model_objects(created: int) -> list[dict]:
- """Downloaded audio.cpp speech and music GGUFs, found by header in the HF cache. The cache scan
- lists a dedicated repo as a GGUF but cannot key an umbrella folder, and neither loads in
- llama.cpp. /v1/audio/speech serves both kinds, loading one by name when auto-switch is on."""
+ """Downloaded audio.cpp speech, music and separation GGUFs, found by header in the HF cache. The
+ cache scan lists a dedicated repo as a GGUF but cannot key an umbrella folder, and none loads in
+ llama.cpp. /v1/audio/speech serves speech and music, /v1/audio/run every workflow listed in
+ ``audio_workflows``, loading the model by name when auto-switch is on."""
try:
from core.inference import audio_cpp_server
from core.inference.audio_cpp_models import downloaded_models
@@ -35820,7 +35915,7 @@ def _audio_cpp_speech_model_objects(created: int) -> list[dict]:
return []
objects = []
for model in downloaded_models():
- if model.audio_type is None or model.task not in ("tts", "music"):
+ if model.audio_type is None or model.task not in ("tts", "music", "sep"):
continue
if account_access.managed_account() and not account_access.model_visible(
model.repo_id or model.id
@@ -35835,9 +35930,11 @@ def _audio_cpp_speech_model_objects(created: int) -> list[dict]:
"object": "model",
"created": created,
"owned_by": _OWNED_BY,
- "task": _TTS_MODEL_TASK,
+ # Music stays text-to-speech, since /v1/audio/speech serves it.
+ "task": _SEP_MODEL_TASK if model.task == "sep" else _TTS_MODEL_TASK,
"display_name": model.display_name,
"loaded": False,
+ "audio_workflows": list(model.workflows),
}
)
return objects
diff --git a/studio/backend/tests/test_audio_cpp_models.py b/studio/backend/tests/test_audio_cpp_models.py
index 8fe236e3545..f52be09cd0f 100644
--- a/studio/backend/tests/test_audio_cpp_models.py
+++ b/studio/backend/tests/test_audio_cpp_models.py
@@ -1229,7 +1229,7 @@ def test_materialize_refuses_a_repo_file_name_that_climbs_out_of_the_farm(
assert not victim.exists()
-def test_v1_models_lists_speech_but_not_separation_models(hub, monkeypatch):
+def test_v1_models_lists_speech_and_separation_models_with_their_workflows(hub, monkeypatch):
from core.inference import audio_cpp_server
from routes import inference as ri
@@ -1240,8 +1240,13 @@ def test_v1_models_lists_speech_but_not_separation_models(hub, monkeypatch):
] == "sep"
monkeypatch.setattr(audio_cpp_server, "find_audio_cpp_server_binary", lambda: "audiocpp_server")
monkeypatch.setattr(audio_cpp_server, "model_runtime_problem", lambda model, binary = None: None)
- listed = [o["id"] for o in ri._audio_cpp_speech_model_objects(0)]
- assert listed == [f"{AUDIO_CPP_REPO}/Kokoro-82M-GGUF"]
+ listed = {o["id"]: o for o in ri._audio_cpp_speech_model_objects(0)}
+ kokoro = listed[f"{AUDIO_CPP_REPO}/Kokoro-82M-GGUF"]
+ assert kokoro["task"] == "text-to-speech" and "speak" in kokoro["audio_workflows"]
+ # /v1/audio/speech cannot serve it, so it is not text-to-speech; /v1/audio/run loads it by name.
+ demucs = listed[f"{AUDIO_CPP_REPO}/HTDemucs-GGUF"]
+ assert demucs["task"] == "audio-to-audio" and demucs["audio_workflows"] == ["separate"]
+ assert len(listed) == 2
def test_downloaded_models_are_found_by_header(hub):
diff --git a/studio/backend/tests/test_keyless_api_access.py b/studio/backend/tests/test_keyless_api_access.py
index 399f7aac3ec..385ca7ff748 100644
--- a/studio/backend/tests/test_keyless_api_access.py
+++ b/studio/backend/tests/test_keyless_api_access.py
@@ -117,7 +117,7 @@ INFERENCE_POST_PATHS = "/v1/chat/completions /v1/chat/count_tokens /v1/completio
def test_exact_route_matrix_matches_registered_topology():
from routes.inference import router
- denied_posts = "/v1/load /v1/unload /v1/validate /v1/generate/stream /v1/audio/speech /v1/audio/run /v1/audio/inputs /v1/audio/voices /v1/images/generations /v1/external/openai/containers/create /v1x/chat".split()
+ denied_posts = "/v1/load /v1/unload /v1/validate /v1/generate/stream /v1/audio/speech /v1/audio/run /v1/audio/inputs /v1/audio/voices /v1/audio/translations /v1/images/generations /v1/external/openai/containers/create /v1x/chat".split()
allowed = {("POST", path) for path in INFERENCE_POST_PATHS} | {
("GET", "/v1/models"), ("GET", "/v1/models/unsloth/model")}
denied = {("POST", path) for path in denied_posts} | {
diff --git a/studio/backend/tests/test_middleware.py b/studio/backend/tests/test_middleware.py
index e812d9b3ed6..64eb232060e 100644
--- a/studio/backend/tests/test_middleware.py
+++ b/studio/backend/tests/test_middleware.py
@@ -149,8 +149,13 @@ class TestMaxBodyMiddleware:
main_module._get_request_body_max_bytes("/api/inference/audio/transcribe")
== STT_AUDIO_JSON_MAX_BYTES
)
- # The OpenAI transcriptions route is multipart, so it gets headroom over the raw cap, on both mounts.
- for path in ("/v1/audio/transcriptions", "/api/inference/audio/transcriptions"):
+ # The OpenAI transcription/translation routes are multipart, so they get headroom over the raw cap.
+ for path in (
+ "/v1/audio/transcriptions",
+ "/api/inference/audio/transcriptions",
+ "/v1/audio/translations",
+ "/api/inference/audio/translations",
+ ):
assert main_module._get_request_body_max_bytes(path) == upload_request_limit_bytes(
STT_AUDIO_RAW_MAX_BYTES
), path
@@ -373,6 +378,7 @@ class TestMaxBodyMiddleware:
"/v1/audio/generate",
"/v1/audio/speech",
"/v1/audio/transcriptions",
+ "/v1/audio/translations",
"/v1/embeddings",
"/v1/responses",
"/v1/messages",
diff --git a/studio/backend/tests/test_openai_audio_translations_route.py b/studio/backend/tests/test_openai_audio_translations_route.py
new file mode 100644
index 00000000000..ebe08a7648c
--- /dev/null
+++ b/studio/backend/tests/test_openai_audio_translations_route.py
@@ -0,0 +1,192 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FastAPI round-trip tests for the OpenAI-compatible POST /v1/audio/translations.
+
+The sidecar call (_transcribe_audio_result) is faked, so these cover multipart wiring,
+model checks, response formats and error propagation without whisper or a GPU."""
+
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+from fastapi import FastAPI, HTTPException
+from fastapi.testclient import TestClient
+
+import routes.inference as routes_module
+from core.inference.api_monitor import api_monitor
+from auth.authentication import get_current_subject
+from routes.inference import router
+from utils.api_errors import install_api_error_handlers
+
+
+def _make_client(monkeypatch, translate = None):
+ calls = []
+
+ async def _fake_transcribe(
+ raw,
+ model,
+ language,
+ fast,
+ engine = None,
+ request = None,
+ **kwargs,
+ ):
+ calls.append({"raw": raw, "model": model, "language": language, "engine": engine, **kwargs})
+ if translate is not None:
+ return await translate(raw)
+ return {"text": "hello sloth", "language": None, "duration": 1.5, "model": "small"}
+
+ monkeypatch.setattr(routes_module, "_transcribe_audio_result", _fake_transcribe)
+
+ app = FastAPI()
+ install_api_error_handlers(app)
+ app.include_router(router, prefix = "/v1")
+ app.dependency_overrides[get_current_subject] = lambda: "test-user"
+ return TestClient(app), calls
+
+
+def _post(
+ cli,
+ data = None,
+ content = b"RIFFfake",
+):
+ return cli.post(
+ "/v1/audio/translations",
+ files = {"file": ("clip.wav", content, "audio/wav")},
+ data = data or {},
+ )
+
+
+def test_json_response_is_text_only_and_asks_for_a_translation(monkeypatch):
+ cli, calls = _make_client(monkeypatch)
+ resp = _post(cli, data = {"model": "whisper-1", "prompt": "ignored", "temperature": "0.2"})
+ assert resp.status_code == 200
+ assert resp.json() == {"text": "hello sloth"}
+ assert calls[0]["raw"] == b"RIFFfake"
+ assert calls[0]["model"] is None
+ assert calls[0]["language"] is None
+ assert calls[0]["translate"] is True
+
+
+def test_text_response_is_plain_body(monkeypatch):
+ cli, _ = _make_client(monkeypatch)
+ resp = _post(cli, data = {"response_format": "text"})
+ assert resp.status_code == 200
+ assert resp.headers["content-type"].startswith("text/plain")
+ assert resp.text == "hello sloth"
+
+
+def test_verbose_json_validates_against_the_openai_client_model(monkeypatch):
+ openai_types = pytest.importorskip("openai.types.audio.translation_verbose")
+ cli, _ = _make_client(monkeypatch)
+ resp = _post(cli, data = {"response_format": "verbose_json"})
+ assert resp.status_code == 200
+ assert resp.json() == {
+ "task": "translate",
+ "language": "english",
+ "duration": 1.5,
+ "text": "hello sloth",
+ }
+ openai_types.TranslationVerbose.model_validate(resp.json())
+
+
+def test_explicit_whisper_model_passes_through(monkeypatch):
+ cli, calls = _make_client(monkeypatch)
+ assert _post(cli, data = {"model": "large-v3"}).status_code == 200
+ assert calls[0]["model"] == "large-v3"
+
+
+ "model",
+ [
+ "qwen3-asr-0.6b",
+ "audio-cpp/audio.cpp-gguf/Qwen3-ASR-0.6B",
+ # Fine-tuned for transcription only: it answers in the source language.
+ "large-v3-turbo",
+ "unsloth/whisper-large-v3-turbo",
+ ],
+)
+def test_models_without_a_translate_task_are_refused_before_any_work(monkeypatch, model):
+ cli, calls = _make_client(monkeypatch)
+ resp = _post(cli, data = {"model": model})
+ assert resp.status_code == 400
+ assert resp.json()["error"]["param"] == "model"
+ assert calls == []
+
+
+ ("data", "param"),
+ [
+ ({"response_format": "srt"}, "response_format"),
+ ({"response_format": "diarized_json"}, "response_format"),
+ ({"provider_id": "conn-1"}, "provider_id"),
+ ],
+)
+def test_unsupported_fields_are_400(monkeypatch, data, param):
+ cli, calls = _make_client(monkeypatch)
+ api_monitor.clear()
+ resp = _post(cli, data = data)
+ assert resp.status_code == 400
+ assert resp.json()["error"]["param"] == param
+ assert calls == []
+ assert api_monitor.snapshot(include_details = False) == []
+
+
+def test_sidecar_errors_keep_their_status(monkeypatch):
+ # An English-only checkpoint is the sidecar's SttLanguageError, mapped to 422 by the helper.
+ async def _english_only(raw):
+ raise HTTPException(status_code = 422, detail = "English-only STT model 'x' cannot translate.")
+
+ cli, _ = _make_client(monkeypatch, translate = _english_only)
+ resp = _post(cli, data = {"model": "owner/whisper-small.en"})
+ assert resp.status_code == 422
+ assert "cannot translate" in resp.json()["error"]["message"]
+
+
+def test_translation_opens_a_monitor_row(monkeypatch):
+ cli, _ = _make_client(monkeypatch)
+ api_monitor.clear()
+ assert _post(cli).status_code == 200
+ rows = api_monitor.snapshot(include_details = False)
+ assert len(rows) == 1
+ assert rows[0]["endpoint"] == "/v1/audio/translations"
+ assert rows[0]["status"] == "completed"
+ assert rows[0]["prompt_preview"] == "clip.wav"
+ assert rows[0]["reply_preview"] == "hello sloth"
+ assert rows[0]["model"] == "small"
+
+
+def test_the_helper_hands_whisper_the_translate_task(monkeypatch):
+ """The real _transcribe_audio_result, with only the sidecar faked."""
+ seen = {}
+
+ def _transcribe(
+ raw,
+ model,
+ language,
+ fast,
+ cancel_event = None,
+ **kwargs,
+ ):
+ seen.update(kwargs, language = language)
+ return {"text": "hi", "model": "small"}
+
+ sidecar = SimpleNamespace(transcribe = _transcribe, loaded_model = None)
+ monkeypatch.setattr(routes_module, "_stt_sidecar_for", lambda engine: sidecar)
+ monkeypatch.setattr(routes_module, "_prepare_runtime_fallback_checkpoint", lambda *a, **k: None)
+ monkeypatch.setattr(
+ routes_module, "_stt_lifecycle", lambda: (lambda *a, **k: None, lambda *a, **k: [])
+ )
+
+ result = asyncio.run(
+ routes_module._transcribe_audio_result(b"audio", None, None, False, translate = True)
+ )
+ assert result["text"] == "hi"
+ assert seen == {"task": "translate", "language": None}
+
+ seen.clear()
+ asyncio.run(routes_module._transcribe_audio_result(b"audio", None, None, False))
+ assert "task" not in seen
diff --git a/studio/backend/tests/test_openai_audio_upload_bound.py b/studio/backend/tests/test_openai_audio_upload_bound.py
index 2236007247d..3a10c12b485 100644
--- a/studio/backend/tests/test_openai_audio_upload_bound.py
+++ b/studio/backend/tests/test_openai_audio_upload_bound.py
@@ -53,3 +53,31 @@ def test_openai_transcription_reads_only_one_byte_past_limit(monkeypatch):
assert sizes == [5]
assert exc.value.status_code == 413
+
+
+def test_openai_translation_reads_only_one_byte_past_limit(monkeypatch):
+ sizes = []
+
+ class _Upload:
+ filename = "clip.wav"
+
+ async def read(self, size = -1):
+ sizes.append(size)
+ return b"x" * size
+
+ monkeypatch.setattr(inference_route, "_MAX_AUDIO_RAW_BYTES", 4)
+
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(
+ inference_route.openai_audio_translations(
+ request = _request(),
+ file = _Upload(),
+ model = None,
+ response_format = "json",
+ provider_id = None,
+ current_subject = "tester",
+ )
+ )
+
+ assert sizes == [5]
+ assert exc.value.status_code == 413
diff --git a/studio/backend/tests/test_openai_models_media.py b/studio/backend/tests/test_openai_models_media.py
index fa160f46928..3414351dab7 100644
--- a/studio/backend/tests/test_openai_models_media.py
+++ b/studio/backend/tests/test_openai_models_media.py
@@ -11,6 +11,7 @@ from pathlib import Path
from types import SimpleNamespace
from typing import Optional
+import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from pydantic import BaseModel
@@ -443,6 +444,21 @@ def test_stt_models_list_downloaded_and_loaded(monkeypatch):
}
+def test_stt_rows_name_their_workflows(monkeypatch):
+ _stt(
+ monkeypatch,
+ downloaded = ("small", "large-v3-turbo"),
+ mtmd_downloaded = ("qwen3-asr-0.6b",),
+ whisper_loaded = "org/whisper-custom",
+ )
+ assert {o["id"]: o["audio_workflows"] for o in inf._stt_model_objects(7)} == {
+ "unsloth/whisper-small": ["transcribe", "translate"],
+ "unsloth/whisper-large-v3-turbo": ["transcribe"],
+ "qwen3-asr-0.6b": ["transcribe"],
+ "org/whisper-custom": ["transcribe"],
+ }
+
+
def test_curated_stt_alias_reports_canonical_loaded_id(monkeypatch):
_stt(monkeypatch, downloaded = ("small",), whisper_loaded = "small")
assert [(o["id"], o["loaded"]) for o in inf._stt_model_objects(7)] == [
@@ -651,10 +667,47 @@ def test_loaded_tts_model_is_tagged(monkeypatch):
monkeypatch.setattr(inf, "get_inference_backend", lambda: _FakeUnsloth())
(entry,) = inf._openai_model_objects()
assert entry["id"] == "orpheus-3b-Q4" and entry["task"] == "text-to-speech"
+ assert entry["audio_workflows"] == ["speak"]
monkeypatch.setattr(inf, "get_llama_cpp_backend", lambda: _Chat())
(entry,) = inf._openai_model_objects()
assert entry["id"] == "qwen3-Q4" and "task" not in entry
+ assert "audio_workflows" not in entry
+
+
+ ("info", "task", "workflows"),
+ [
+ (
+ {"audio_type": "audiocpp_tts", "audio_workflows": ["speak", "clone"]},
+ "text-to-speech",
+ ["speak", "clone"],
+ ),
+ (
+ {"audio_type": "audiocpp_music", "audio_workflows": ["music"]},
+ "text-to-speech",
+ ["music"],
+ ),
+ # Listed with no task, a resident separation model looked like a chat model.
+ (
+ {"audio_type": "audiocpp_sep", "audio_workflows": ["separate"]},
+ "audio-to-audio",
+ ["separate"],
+ ),
+ ({"audio_type": "csm"}, "text-to-speech", ["speak"]),
+ ],
+)
+def test_resident_audio_models_carry_task_and_workflows(monkeypatch, info, task, workflows):
+ unsloth = type(
+ "_Audio",
+ (_FakeUnsloth,),
+ {"active_model_name": "org/audio", "models": {"org/audio": {"is_audio": True, **info}}},
+ )
+ monkeypatch.setattr(inf, "get_llama_cpp_backend", lambda: _FakeLlama())
+ monkeypatch.setattr(inf, "get_inference_backend", lambda: unsloth())
+ (entry,) = inf._openai_model_objects()
+ assert entry["task"] == task
+ assert entry["audio_workflows"] == workflows
def test_audio_input_models_are_not_tagged_text_to_speech(monkeypatch):
@@ -713,6 +766,35 @@ def test_audio_input_models_are_not_tagged_text_to_speech(monkeypatch):
assert "task" not in entry
+def test_a_dedicated_separation_repo_keeps_its_task_and_workflows(monkeypatch):
+ repo = _Info(
+ "models--someone--htdemucs-GGUF",
+ "htdemucs-GGUF",
+ model_id = "someone/htdemucs-GGUF",
+ task = "audio-to-audio",
+ )
+ _catalog(monkeypatch, [repo], picks = {})
+ monkeypatch.setattr(
+ inf,
+ "_audio_cpp_speech_model_objects",
+ lambda created: [
+ {
+ "id": "someone/htdemucs-GGUF",
+ "object": "model",
+ "created": created,
+ "owned_by": inf._OWNED_BY,
+ "task": "audio-to-audio",
+ "loaded": False,
+ "audio_workflows": ["separate"],
+ }
+ ],
+ )
+
+ ids = {model["id"]: model for model in asyncio.run(inf._openai_catalog_objects())}
+ assert ids["someone/htdemucs-GGUF"]["task"] == "audio-to-audio"
+ assert ids["someone/htdemucs-GGUF"]["audio_workflows"] == ["separate"]
+
+
def test_downloaded_tts_model_is_not_advertised_without_switch_support(monkeypatch):
tts = _Info(
"models--unsloth--csm-1b",
diff --git a/studio/backend/tests/test_stt_sidecar.py b/studio/backend/tests/test_stt_sidecar.py
index fb574a66d9a..688d4f4a29a 100644
--- a/studio/backend/tests/test_stt_sidecar.py
+++ b/studio/backend/tests/test_stt_sidecar.py
@@ -368,6 +368,37 @@ def test_english_only_model_rejects_non_english_before_decode(monkeypatch, tmp_p
)
+def test_translation_asks_whisper_for_its_translate_task(monkeypatch):
+ sidecar = WhisperSttSidecar()
+ infer = _CaptureInference()
+ monkeypatch.setattr(sidecar, "_transcribe_decoded", infer)
+
+ sidecar.transcribe(b"encoded audio", task = "translate")
+
+ assert infer.generate_kwargs["task"] == "translate"
+ assert "language" not in infer.generate_kwargs
+
+
+def test_english_only_model_refuses_to_translate_before_decode(monkeypatch, tmp_path):
+ # Its generation config pins the transcribe task, so the request would quietly transcribe.
+ (tmp_path / "config.json").write_text('{"model_type": "whisper"}')
+ (tmp_path / "generation_config.json").write_text('{"is_multilingual": false}')
+ sidecar = WhisperSttSidecar()
+ monkeypatch.setattr(
+ stt_sidecar_module,
+ "_find_complete_cached_snapshot",
+ lambda _model: tmp_path,
+ )
+ monkeypatch.setattr(
+ stt_sidecar_module,
+ "_decode_audio_bounded",
+ lambda *_: pytest.fail("an English-only translation must be refused before decode"),
+ )
+
+ with pytest.raises(SttLanguageError, match = "cannot translate"):
+ sidecar.transcribe(b"encoded audio", model = "owner/whisper-small.en", task = "translate")
+
+
def test_english_only_model_omits_forbidden_generation_controls(monkeypatch):
calls = []