Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: complete the OpenAI-compatible audio API (#12812)
Etherl0423db52379c
diff --git a/studio/backend/core/inference/audio_inputs.py b/studio/backend/core/inference/audio_inputs.pyindex 4c117baebd1..f5a1e5b4b8e 100644--- a/studio/backend/core/inference/audio_inputs.py+++ b/studio/backend/core/inference/audio_inputs.py@@ -220,6 +220,68 @@ def transcode(return {"duration_s": round(frames / rate, 3), "sample_rate": rate, "channels": channels}+# /v1/audio/speech formats besides WAV: (container, encoder, media type, sample rate it needs).+_SPEECH_ENCODINGS = {+ "mp3": ("mp3", "mp3", "audio/mpeg", None),+ "opus": ("ogg", "libopus", "audio/ogg", 48000),+ "aac": ("adts", "aac", "audio/aac", None),+ "flac": ("flac", "flac", "audio/flac", None),+ # OpenAI's pcm is 24 kHz 16-bit mono; its clients assume that rate.+ "pcm": ("s16le", "pcm_s16le", "audio/pcm", 24000),+}+++def speech_formats() -> list[str]:+ """``wav`` plus each speech format whose encoder this FFmpeg build has."""+ import av++ formats = ["wav"]+ for fmt, (_container, codec, _media_type, _rate) in _SPEECH_ENCODINGS.items():+ try:+ av.codec.Codec(codec, "w")+ except Exception: # noqa: BLE001 - not in this build+ continue+ formats.append(fmt)+ return formats+++def encode_wav(wav_bytes: bytes, fmt: str) -> tuple[bytes, str]:+ """A WAV re-encoded as ``fmt`` (a ``speech_formats`` name but wav); ``(bytes, media type)``.+ ``pcm`` is 24 kHz 16-bit little-endian mono samples, with no header."""+ import io++ import av++ container_format, codec, media_type, fixed_rate = _SPEECH_ENCODINGS[fmt]+ out = io.BytesIO()+ with _av_open(av, io.BytesIO(wav_bytes)) as src:+ audio = src.streams.audio[0]+ rate = fixed_rate or audio.rate+ # A WAV with no channel mask reads as "1 channels", which no encoder takes.+ layout = "stereo" if audio.channels == 2 and fmt != "pcm" else "mono"+ with av.open(out, "w", format = container_format) as dst:+ stream = dst.add_stream(codec, rate = rate, layout = layout)+ stream.codec_context.open(strict = True) # sets frame_size, which mp3, aac and opus need+ resampler = av.AudioResampler(+ format = stream.format.name,+ layout = layout,+ rate = rate,+ frame_size = stream.frame_size or None,+ )++ def mux(frames) -> None:+ for frame in frames:+ for packet in stream.encode(frame):+ dst.mux(packet)++ for frame in src.decode(audio):+ frame.pts = None+ mux(resampler.resample(frame))+ mux(resampler.resample(None))+ mux([None])+ return out.getvalue(), media_type++async def save_stream(chunks: AsyncIterable[bytes],name: Optional[str],@@ -417,7 +479,8 @@ def sweep(age = now - _mtime(path)if name.startswith(".") and name.endswith(".tmp") and age > _STALE_TMP_SECONDS:path.unlink(missing_ok = True)- elif name.startswith(("c-", "v-")) and name.endswith(".wav") and age > ttl:+ # "." = a crashed timed transcription's upload; full TTL (aligner downloads take an hour).+ elif name.startswith(("c-", "v-", ".")) and name.endswith(".wav") and age > ttl:path.unlink(missing_ok = True)# Music run folders a crash left behind.runs = directory / "runs"diff --git a/studio/backend/main.py b/studio/backend/main.pyindex 82a1fad436a..97d1dab44d8 100644--- a/studio/backend/main.py+++ b/studio/backend/main.py@@ -1402,7 +1402,7 @@ _VIDEO_MULTIPART_UPLOAD_PATHS = ()_LIBRARY_UPLOAD_PATH = "/api/library/uploads"# Streamed to disk and capped by the route itself; buffering here would hold 200 MiB in memory.-_AUDIO_INPUT_UPLOAD_PATH = "/api/inference/audio/inputs"+_AUDIO_INPUT_UPLOAD_PATHS = ("/api/inference/audio/inputs", "/v1/audio/inputs")_BODY_UPLOAD_PASSTHROUGH_PREFIXES = (*_DATASET_UPLOAD_PASSTHROUGH_PREFIXES,_DATA_RECIPE_UNSTRUCTURED_UPLOAD_PASSTHROUGH_PREFIX,@@ -1410,7 +1410,7 @@ _BODY_UPLOAD_PASSTHROUGH_PREFIXES = (# Matched by EXACT path (multipart uploads only), so sibling JSON sub-routes keep the normal cap._BODY_UPLOAD_PASSTHROUGH_EXACT_PATHS = (_DIFFUSION_DATASET_UPLOAD_PATH,- _AUDIO_INPUT_UPLOAD_PATH,+ *_AUDIO_INPUT_UPLOAD_PATHS,*_STT_MULTIPART_UPLOAD_PATHS,*_VIDEO_MULTIPART_UPLOAD_PATHS,_LIBRARY_UPLOAD_PATH,@@ -1434,7 +1434,7 @@ def _get_upload_passthrough_request_max_bytes(path: str) -> int:)if path.rstrip("/") == _LIBRARY_UPLOAD_PATH:return upload_request_limit_bytes(LIBRARY_UPLOAD_MAX_BYTES)- if path.rstrip("/") == _AUDIO_INPUT_UPLOAD_PATH:+ if path.rstrip("/") in _AUDIO_INPUT_UPLOAD_PATHS:return AUDIO_INPUT_MAX_BYTES# The trailing-slash variant reaches this middleware BEFORE the router's redirect_slashes# 307, so it must resolve to the same cap. JSON sub-routes keep extra path components.diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 8beb9329e52..055a5ef4442 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -5197,8 +5197,8 @@ class ImageGenerationResponse(BaseModel):class AudioSpeechRequest(BaseModel):"""OpenAI ``CreateSpeechRequest`` for ``POST /v1/audio/speech``.- ``voice`` and ``speed`` are accepted for client compatibility but unused: no loaded- TTS backend has voice or rate plumbing (CSM is fixed to speaker 0)."""+ ``voice`` picks a saved voice or a built-in speaker; OpenAI's own voice names are accepted+ and ignored. ``reference`` clones from uploaded audio, a history clip or a saved voice."""input: str = Field(..., min_length = 1, description = "The text to synthesize.")model: Optional[str] = Field(@@ -5208,11 +5208,31 @@ class AudioSpeechRequest(BaseModel):"model auto-switch is on; otherwise the loaded audio model is used."),)- voice: Optional[str] = Field(None, description = "Voice name (accepted, unused).")+ voice: Optional[Union[str, "AudioSpeechVoice"]] = Field(+ None,+ description = (+ 'A saved voice\'s id (or {"id": ...}), else a built-in speaker of the model. Other '+ "names, such as OpenAI's, are ignored."+ ),+ )+ reference: Optional["AudioSourceRef"] = Field(+ None,+ description = "[x-unsloth] Audio to clone: an uploaded input, a history clip or a saved voice.",+ )+ reference_text: Optional[str] = Field(+ None,+ max_length = 4000,+ description = "[x-unsloth] What is said in the reference, for models that use it.",+ )response_format: Optional[str] = Field(- "wav", description = "Output container. Only 'wav' is supported."+ "wav", description = "Output format: wav, mp3, flac, opus, aac or pcm."+ )+ stream_format: Optional[Literal["audio", "sse"]] = Field(+ None, description = "Only 'audio' (the whole file in one response) is supported."+ )+ speed: Optional[float] = Field(+ None, ge = 0.25, le = 4.0, description = "Speech rate, applied where the model supports it.")- speed: Optional[float] = Field(None, description = "Speech rate (accepted, unused).")instructions: Optional[str] = Field(None,description = "Scene or music-description instructions for compatible audio models.",@@ -5345,6 +5365,12 @@ class AudioSourceRef(BaseModel):return self+class AudioSpeechVoice(BaseModel):+ """OpenAI's custom-voice object: here a saved voice's id."""++ id: str = Field(..., pattern = _AUDIO_ID_PATTERN)++class AudioRunInputs(BaseModel):model_config = ConfigDict(extra = "forbid")@@ -5410,6 +5436,12 @@ class AudioRunRequest(BaseModel):model_config = ConfigDict(extra = "forbid")workflow: Literal["clone", "speak", "edit", "convert", "music", "separate"]+ model: Optional[str] = Field(+ None,+ max_length = 512,+ description = "A downloaded audio model to load first when model auto-switch is on; "+ "otherwise the loaded model runs it",+ )# Required except for a conversion or a separation (the routes answer those given text with a 400).text: Optional[str] = Nonelanguage: Optional[str] = Field(None, max_length = 64)diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 231ab0f9818..52bde1565f9 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -3704,8 +3704,11 @@ from models.inference import (AudioInputRecord,AudioInputTranscribeRequest,AudioInputTranscript,+ AudioRunInputs,AudioRunRequest,AudioRunResponse,+ AudioSourceRef,+ AudioSpeechVoice,AudioVoice,AudioVoiceCreate,AudioVoiceListResponse,@@ -9411,6 +9414,20 @@ def _target_speech_audio_type(return None+def _target_audio_workflows(target_id: str, speech_type: Optional[str]) -> list[str]:+ """The Audio page workflows a switch target runs, answered from the HF cache alone."""+ from core.inference.audio_cpp_models import looks_like_audio_cpp, resolve+ from core.inference.audio_workflows import status_audio_workflows++ if not looks_like_audio_cpp(target_id):+ return status_audio_workflows(speech_type is not None, speech_type)+ try:+ model = resolve(target_id, network = False)+ except Exception: # noqa: BLE001 - unreadable from the cache: it runs nothing+ return []+ return list(model.workflows) if model is not None else []++@dataclass(frozen = True)class _SpeechCodecPreflightResult:cache_environment: Optional[dict[str, str]] = None@@ -10894,6 +10911,7 @@ async def _maybe_auto_switch_model(audio_preflight: Optional[dict] = None,image_preflight: Optional[dict] = None,require_speech: bool = False,+ require_audio_workflow: Optional[str] = None,speech_budget: Optional[dict] = None,tool_images_only: bool = False,) -> None:@@ -10922,7 +10940,8 @@ async def _maybe_auto_switch_model(``audio_preflight`` carries an audio upload whose format-dependentchecks only the resolved target can decide; see:func:`_preflight_audio_for_switch`. ``image_preflight`` does the same for- non-GGUF image count and byte validation.+ non-GGUF image count and byte validation. ``require_audio_workflow`` refuses a target that+ cannot run that Audio page workflow (``separate``, ``clone``...)."""# A text-only GGUF reads tool-result images as a note, so only audio or video needs its mmproj.gguf_requires_vision = (@@ -11382,6 +11401,21 @@ async def _maybe_auto_switch_model(param = "instructions",),)+ if (+ require_audio_workflow+ and resolved is not None+ and require_audio_workflow+ not in await asyncio.to_thread(_target_audio_workflows, target_id, speech_type)+ ):+ raise HTTPException(+ status_code = 400,+ detail = openai_error_body(+ f"The requested model cannot run the {require_audio_workflow} workflow.",+ status = 400,+ code = "invalid_value",+ param = "model",+ ),+ )# resolver branch only, like the probe above: a reload-stash restore changes no format.if audio_preflight is not None and resolved is not None:await _preflight_audio_for_switch(audio_preflight, target_is_gguf)@@ -22241,20 +22275,27 @@ async def _generate_tts_wav(requested_model: str = _RELOAD_ONLY_MODEL,run_inputs: Optional[dict] = None,stats_holder: Optional[dict] = None,+ voice: Optional[str] = None,) -> tuple[bytes, int, str, Optional[str]]:"""Shared core of /audio/generate, /audio/speech and /audio/run. Returns(wav_bytes, sample_rate, model_name, audio_type). ``run_inputs`` is /audio/run's ``workflow``,``audio_inputs`` (role -> server-local WAV path), ``reference_text``, ``speed`` and ``convert``;- a conversion fills ``audio_inputs`` via ``prepare_audio_inputs(model_info)``."""+ a conversion fills ``audio_inputs`` via ``prepare_audio_inputs(model_info)``. ``voice`` is a+ built-in speaker, used when the loaded model lists it and the request set none."""# A named target must be budgeted against its own context after preflight.if requested_model == _RELOAD_ONLY_MODEL:_raise_if_prompt_leaves_no_speech_budget(text)+ workflow = (run_inputs or {}).get("workflow")+ if workflow == "speak" and (run_inputs or {}).get("audio_inputs"):+ # Speaking in a saved voice is cloning, as _audio_request_problem reads it.+ workflow = "clone"await _maybe_auto_switch_model(requested_model,request,current_subject,claim_resident = False,require_speech = True,+ require_audio_workflow = workflow,speech_budget = (Noneif requested_model == _RELOAD_ONLY_MODEL@@ -22318,6 +22359,16 @@ async def _generate_tts_wav(audio_type = model_info.get("audio_type")audio_family = model_info.get("audio_family")supported_audio_types = _TRANSFORMERS_TTS_AUDIO_TYPES+ speakers = next(+ (+ o.get("values")+ for o in model_info.get("audio_options") or ()+ if o.get("name") == "voice"+ ),+ None,+ )+ if voice in (speakers or ()) and "voice" not in (payload.audio_options or {}):+ payload.audio_options = {**(payload.audio_options or {}), "voice": voice}_audio_model_id = getattr(backend, "active_model_name", None) or model_namegen = lambda: backend.generate_audio_response(text = text,@@ -22562,7 +22613,7 @@ async def generate_audio(text = last_user_msg["content"]tts_stats: dict = {}- wav_bytes, sample_rate, model_name, audio_type = await _generate_tts_wav(+ generate = lambda: _generate_tts_wav( # noqa: E731text,payload,request,@@ -22573,6 +22624,18 @@ async def generate_audio(requested_model = _switch_model_for_payload(payload) or _RELOAD_ONLY_MODEL,stats_holder = tts_stats,)+ # Chat completions monitors the turns it hands here, and the chat's read-aloud is no API traffic.+ if getattr(getattr(request, "url", None), "path", "").rstrip("/") != "/v1/audio/generate":+ wav_bytes, sample_rate, model_name, audio_type = await generate()+ else:+ async with _monitored_media_request(+ request,+ model = public_model_id(payload.model) or payload.model,+ prompt = text,+ subject = current_subject,+ ) as monitor_id:+ wav_bytes, sample_rate, model_name, audio_type = await generate()+ api_monitor.relabel(monitor_id, model_name)truncated = bool((tts_stats.get("stats") or {}).get("truncated"))persisted_clip = await asyncio.to_thread(_persist_tts_clip, wav_bytes, sample_rate, text, model_name, audio_type@@ -22604,6 +22667,14 @@ def _audio_source_error(exc) -> HTTPException:return HTTPException(status_code = exc.status, detail = exc.detail)+def _v1_url(request: Request, url: str) -> str:+ """A Studio file URL on the mount the client called, so a /v1 client needs only that base URL."""+ prefix = "/api/inference/"+ if request.url.path.startswith("/v1/") and url.startswith(prefix):+ return "/v1/" + url[len(prefix) :]+ return url++CONVERT_EXPIRED_DETAIL = {"source": "This recording expired. Add it again.","target": "This target voice expired. Add it again.",@@ -22733,6 +22804,7 @@ async def _run_audio_edit(# For the request check; the worker gets the text as the prompt."text": body.text,},+ requested_model = body.model or _RELOAD_ONLY_MODEL,)source_record = Nonetry:@@ -22940,11 +23012,12 @@ async def _run_music_workflow(from core.inference.audio_cpp_music import frames_for, timeout_secondsawait _maybe_auto_switch_model(- _RELOAD_ONLY_MODEL,+ body.model or _RELOAD_ONLY_MODEL,request,current_subject,claim_resident = False,require_speech = True,+ require_audio_workflow = "music",)backend = await asyncio.to_thread(get_inference_backend)if not backend.active_model_name:@@ -23151,6 +23224,7 @@ async def _run_audio_convert("convert_inputs": tuple(resolved),"prepare_audio_inputs": _prepare,},+ requested_model = body.model or _RELOAD_ONLY_MODEL,)builtin = Noneif target is not None:@@ -23202,12 +23276,27 @@ async def run_audio_workflow("""Run one Audio page workflow (Clone, Speak in a saved voice, Edit, Convert, Music orSeparate); clips go to history. Audio is named by id and resolved here in the caller's account;the worker gets a prepared copy's path, never bytes."""- if body.workflow == "separate":- return await _run_separation(body, request, current_subject)- import base64+ if not request.url.path.startswith("/v1/"):+ return await _dispatch_audio_run(body, request, current_subject)+ # Only API runs are monitored, like /v1/audio/speech; the Audio page's are not.+ async with _monitored_media_request(+ request,+ model = public_model_id(body.model) or body.model,+ prompt = body.text or body.workflow,+ subject = current_subject,+ ) as monitor_id:+ response = await _dispatch_audio_run(body, request, current_subject)+ api_monitor.relabel(monitor_id, response.model)+ for clip in response.clips:+ clip.url = _v1_url(request, clip.url)+ return response- from core.inference import audio_inputs+async def _dispatch_audio_run(+ body: AudioRunRequest, request: Request, current_subject: str+) -> AudioRunResponse:+ if body.workflow == "separate":+ return await _run_separation(body, request, current_subject)if body.workflow == "convert":return await _run_audio_convert(body, request, current_subject)if body.workflow == "edit":@@ -23224,6 +23313,18 @@ async def run_audio_workflow(status_code = 400,detail = "A music mode, lyrics, variations, a source clip and edits apply to Music only.",)+ record, wav_bytes, sample_rate, model_name = await _speak(+ body, request, current_subject, requested_model = body.model or _RELOAD_ONLY_MODEL+ )+ return _audio_run_response(record, wav_bytes, sample_rate, model_name)+++async def _speak(+ body: AudioRunRequest, request: Request, current_subject: str, **tts_kwargs+) -> tuple[Optional[dict[str, Any]], bytes, int, str]:+ """Clone, or Speak in a saved voice, and keep the clip in history; shared by /audio/run and+ /audio/speech. ``(record, wav_bytes, sample_rate, model_name)``."""+ from core.inference import audio_inputsif body.workflow == "clone" and body.inputs.reference is None:raise HTTPException(status_code = 400, detail = "Add a reference clip to clone.")@@ -23269,6 +23370,7 @@ async def run_audio_workflow("reference_text": reference_text,"speed": body.speed,},+ **tts_kwargs,)extra_meta: dict[str, Any] = {"role": "output",@@ -23290,7 +23392,7 @@ async def run_audio_workflow(extra_meta,body.workflow,)- return _audio_run_response(record, wav_bytes, sample_rate, model_name)+ return record, wav_bytes, sample_rate, model_name# Stems in the order the Separate page lists them; ids a runtime adds follow in its own order.@@ -23453,7 +23555,11 @@ async def _run_separation(if source_ref.voice_id:raise HTTPException(status_code = 400, detail = "Pick a track to separate, not a saved voice.")await _maybe_auto_switch_model(- _RELOAD_ONLY_MODEL, request, current_subject, claim_resident = False+ body.model or _RELOAD_ONLY_MODEL,+ request,+ current_subject,+ claim_resident = False,+ require_audio_workflow = "separate",)backend = await asyncio.to_thread(get_inference_backend)model_info = (@@ -23553,7 +23659,7 @@ async def _external_tts_speech(body: AudioSpeechRequest, request: Request) -> Re"""Proxy CreateSpeech to a saved connection, so read-aloud skips the local model slot."""provider_id = (body.provider_id or "").strip()external_model = (body.model or "").strip()- external_voice = (body.voice or "").strip()+ external_voice = body.voice.strip() if isinstance(body.voice, str) else ""if not provider_id:raise HTTPException(status_code = 400, detail = "provider_id cannot be empty.")if not external_model:@@ -23666,9 +23772,14 @@ async def openai_audio_speech(With ``provider_id`` the request is proxied. Otherwise an omitted ``model`` serves theresident model. A named downloaded local model is loaded when auto-switch is on; whenit is off, the shared switch guard rejects a different recognized local model.- Local ``voice``/``speed`` are ignored and only WAV is supported."""+ A saved voice (``voice``) or ``reference`` clones; history keeps the clip as WAV."""+ fmt = (body.response_format or "wav").strip().lower()+ # Neither path streams, a saved connection included.+ if body.stream_format == "sse":+ _raise_unsupported_openai_parameter(+ "stream_format", "stream_format 'sse' is not supported. Use 'audio'."+ )if body.provider_id:- fmt = (body.response_format or "wav").strip().lower()if fmt != "wav":raise HTTPException(status_code = 400,@@ -23693,12 +23804,43 @@ async def openai_audio_speech(subject = current_subject,):return await _external_tts_speech(body, request)- fmt = (body.response_format or "wav").strip().lower()- if fmt != "wav":- raise HTTPException(- status_code = 400,- detail = f"Unsupported response_format '{body.response_format}'. Only 'wav' is supported.",+ from core.inference import audio_inputs, audio_voices++ formats = await asyncio.to_thread(audio_inputs.speech_formats)+ if fmt not in formats:+ _raise_unsupported_openai_parameter(+ "response_format",+ f"Unsupported response_format '{body.response_format}'. Use {', '.join(formats)}.",+ )+ voice = body.voice+ voice_id = voice.id if isinstance(voice, AudioSpeechVoice) else None+ if isinstance(voice, str) and await asyncio.to_thread(audio_voices.get, voice):+ voice_id = voice+ if voice_id and body.reference is not None:+ _raise_unsupported_openai_parameter(+ "reference", "Send a saved voice or a reference, not both.")+ run = None+ if voice_id or body.reference is not None:+ from fastapi.exceptions import RequestValidationError+ from pydantic import ValidationError+ try:+ run = AudioRunRequest(+ workflow = "speak" if voice_id else "clone",+ text = body.input,+ language = body.language,+ instructions = body.instructions,+ inputs = AudioRunInputs(+ reference = body.reference or AudioSourceRef(voice_id = voice_id),+ reference_text = body.reference_text,+ ),+ options = body.audio_options,+ speed = body.speed,+ seed = body.seed,+ max_tokens = body.max_new_tokens or AUDIO_GENERATION_MAX_TOKENS,+ )+ except ValidationError as exc:+ raise RequestValidationError(exc.errors()) from None# The tts core reads its sampling knobs from a chat request shape; defaults apply here.# Keep the full speech ceiling unless a client supplies the native max_new_tokens# extension. MiniMax gets its documented 750-frame default deeper in the shared core;@@ -23725,19 +23867,31 @@ async def openai_audio_speech(prompt = body.input,subject = current_subject,) as monitor_id:- wav_bytes, sample_rate, model_name, audio_type = await _generate_tts_wav(- body.input,- payload,- request,- current_subject,- speech_api_default_max_tokens = body.max_new_tokens is None,- requested_model = body.model or _RELOAD_ONLY_MODEL,- )+ tts_kwargs = {+ "speech_api_default_max_tokens": body.max_new_tokens is None,+ "requested_model": body.model or _RELOAD_ONLY_MODEL,+ }+ if run is not None:+ _, wav_bytes, sample_rate, model_name = await _speak(+ run, request, current_subject, **tts_kwargs+ )+ else:+ wav_bytes, sample_rate, model_name, audio_type = await _generate_tts_wav(+ body.input,+ payload,+ request,+ current_subject,+ voice = voice if isinstance(voice, str) else None,+ **tts_kwargs,+ )+ await asyncio.to_thread(+ _persist_tts_clip, wav_bytes, sample_rate, body.input, model_name, audio_type+ )api_monitor.relabel(monitor_id, model_name)- await asyncio.to_thread(- _persist_tts_clip, wav_bytes, sample_rate, body.input, model_name, audio_type- )- return Response(content = wav_bytes, media_type = "audio/wav")+ if fmt == "wav":+ return Response(content = wav_bytes, media_type = "audio/wav")+ content, media_type = await asyncio.to_thread(audio_inputs.encode_wav, wav_bytes, fmt)+ return Response(content = content, media_type = media_type)async def _external_stt_transcription(@@ -23863,14 +24017,15 @@ async def openai_audio_transcriptions(With ``provider_id`` set, the request is proxied to that saved connection.Otherwise ``model`` maps to a sidecar model id, with ``whisper-1`` or nothingselecting the default. ``response_format`` supports ``json``, ``text`` and- ``verbose_json``."""+ ``verbose_json``; an audio.cpp model also gives segments, words (``timestamp_granularities``)+ and ``diarized_json``."""fmt = (response_format or "json").strip().lower()- if fmt not in ("json", "text", "verbose_json"):+ if fmt not in ("json", "text", "verbose_json", "diarized_json"):raise HTTPException(status_code = 400,detail = (f"Unsupported response_format '{response_format}'. "- "Use 'json', 'text' or 'verbose_json'."+ "Use 'json', 'text', 'verbose_json' or 'diarized_json'."),)# UploadFile spools to disk, but an unbounded read materializes the whole upload in@@ -23901,11 +24056,40 @@ async def openai_audio_transcriptions()api_monitor.set_reply(monitor_id, _external_transcript_preview(response))return response- # Refused before any work, like the response_format check above: the sidecar echoes the- # language it was given and detects none, so an auto-detect request has nothing truthful- # for verbose_json's required language field, and guessing would silently label a- # Japanese clip "en". isinstance because a direct call passes the raw Form default.- if fmt == "verbose_json" and not (isinstance(language, str) and language.strip()):+ from core.inference import audio_inputs, stt_capabilities, stt_details++ # openai's whisper-1 placeholder means the default transcription model, not a sidecar id+ sidecar_model = None if model in (None, "", "whisper-1") else model+ engine = _stt_engine_for_model(sidecar_model)+ # isinstance because a direct call passes the raw Form default.+ granularities = timestamp_granularities if isinstance(timestamp_granularities, list) else []+ if granularities and fmt != "verbose_json":+ # OpenAI's contract; also spares an aligner download whose timings would be dropped.+ _raise_unsupported_openai_parameter(+ "timestamp_granularities",+ "timestamp_granularities needs response_format 'verbose_json'.",+ )+ has_language = isinstance(language, str) and language.strip()+ serving_engine = _resolve_serving_stt_engine(engine)+ timed = serving_engine == "audiocpp" and (+ fmt in ("verbose_json", "diarized_json") or bool(granularities)+ )+ if timed or fmt == "diarized_json":+ caps = await asyncio.to_thread(+ stt_capabilities.capabilities_for, sidecar_model, serving_engine+ )+ _refuse_stt_caps(+ caps, timestamps = timed and bool(granularities), speakers = fmt == "diarized_json"+ )+ if timed:+ # The prepared copy is read in place, past the sidecar's own size check.+ if len(raw) > _MAX_AUDIO_RAW_BYTES:+ raise HTTPException(status_code = 413, detail = "Audio is too large.")+ elif fmt == "verbose_json" and not has_language:+ # Refused before any work, like the response_format check above: the sidecar echoes the+ # language it was given and detects none, so an auto-detect request has nothing truthful+ # for verbose_json's required language field, and guessing would silently label a+ # Japanese clip "en".raise HTTPException(status_code = 501,detail = (@@ -23914,9 +24098,9 @@ async def openai_audio_transcriptions("'json' or 'text'."),)- if isinstance(timestamp_granularities, list) and timestamp_granularities:+ elif granularities:# Accepting the parameter and returning neither words nor segments would look like- # the audio simply had none. The sidecar reports no per-token timing today.+ # the audio simply had none. Only audio.cpp models report timings.raise HTTPException(status_code = 400,detail = (@@ -23924,8 +24108,6 @@ async def openai_audio_transcriptions("Use a saved STT connection with provider_id for word or segment timings."),)- # openai's whisper-1 placeholder means the default transcription model, not a sidecar id- sidecar_model = None if model in (None, "", "whisper-1") else model# The relabel below only lands on success; a sidecar failure would otherwise strand the# raw client string, which may be a host path, on the terminal row._opening_label = sidecar_model or "whisper-1"@@ -23935,15 +24117,36 @@ async def openai_audio_transcriptions(prompt = file.filename or "audio",subject = current_subject,) as monitor_id:- result = await _transcribe_audio_result(- raw,- sidecar_model,- language,- fast = False,- engine = _stt_engine_for_model(sidecar_model),- request = request,- )+ path = None+ try:+ if timed:+ rate = stt_details.FIXED_SPAN_RATES.get(caps["family"], 16000)+ try:+ path = await asyncio.to_thread(_prepared_upload, raw, rate)+ except audio_inputs.AudioInputError as exc:+ raise _audio_source_error(exc) from None+ result = await _transcribe_audio_result(+ raw,+ sidecar_model,+ language,+ fast = False,+ engine = engine,+ request = request,+ source_path = path,+ timestamps = bool(granularities),+ )+ finally:+ if path is not None:+ await asyncio.to_thread(path.unlink, True)text = str(result.get("text", ""))+ # Required by OpenAI's schema: a null fails validation inside the official clients.+ detected = str(result.get("language") or (language if has_language else "")).strip()+ if fmt == "verbose_json" and not detected:+ raise HTTPException(+ status_code = 501,+ detail = "This model did not report the language of the audio. Send the "+ "'language' parameter, or use response_format 'json' or 'text'.",+ )# Same path-free treatment the image route gives its label: sidecar ids are# curated or owner/model today, but the row goes out over the tunnel._served_model = str(result.get("model") or "")@@ -23952,20 +24155,54 @@ async def openai_audio_transcriptions(if fmt == "text":return PlainTextResponse(content = text)- if fmt == "verbose_json":- _duration = result.get("duration")+ _duration = result.get("duration")+ # Unlike the language, this is not a guess: duration is None only for a clip that decoded+ # to no samples, and that clip really is zero seconds.+ duration = float(_duration) if isinstance(_duration, (int, float)) else 0.0+ segments = result.get("segments") or []+ if fmt == "diarized_json":+ labels = {s["id"]: s["label"] for s in result.get("speakers") or ()}return JSONResponse(content = {"task": "transcribe",- # Both are required by OpenAI's schema, so a null fails validation inside- # the official clients. The request is refused above with no language.- "language": str(result.get("language") or language).strip(),- # Unlike the language, this is not a guess: duration is None only for a- # clip that decoded to no samples, and that clip really is zero seconds.- "duration": float(_duration) if isinstance(_duration, (int, float)) else 0.0,+ "duration": duration,"text": text,+ "segments": [+ {+ "type": "transcript.text.segment",+ "id": f"seg_{index}",+ "start": s["start"],+ "end": s["end"],+ "text": s["text"],+ "speaker": labels.get(s.get("speaker"), s.get("speaker") or ""),+ }+ for index, s in enumerate(segments)+ ],})+ if fmt == "verbose_json":+ content = {"task": "transcribe", "language": detected, "duration": duration, "text": text}+ if segments:+ content["segments"] = [+ {+ "id": index,+ "seek": 0,+ "start": s["start"],+ "end": s["end"],+ "text": s["text"],+ "tokens": [],+ "temperature": 0.0,+ "avg_logprob": 0.0,+ "compression_ratio": 0.0,+ "no_speech_prob": 0.0,+ }+ for index, s in enumerate(segments)+ ]+ if "word" in granularities and result.get("words"):+ content["words"] = [+ {"word": w["word"], "start": w["start"], "end": w["end"]} for w in result["words"]+ ]+ return JSONResponse(content = content)return JSONResponse(content = {"text": text})@@ -24928,6 +25165,40 @@ async def transcribe_audio_raw()+def _refuse_stt_caps(caps: dict, *, timestamps: bool, speakers: bool) -> None:+ """422 when the model cannot add what the transcript asks for."""+ if timestamps and caps["timestamps"] == "unsupported":+ raise HTTPException(+ status_code = 422,+ detail = "This model cannot add timestamps. Pick Qwen3-ASR (audio.cpp), Parakeet-TDT, "+ "MOSS-Transcribe-Diarize or VibeVoice-ASR.",+ )+ if speakers and not caps["speakers"]:+ raise HTTPException(+ status_code = 422,+ detail = "This model cannot tell speakers apart. Pick MOSS-Transcribe-Diarize or "+ "VibeVoice-ASR.",+ )+++def _prepared_upload(raw: bytes, rate: int) -> Path:+ """An upload decoded to a mono WAV at ``rate`` in this account's inputs, for audio.cpp to read+ in place. The caller removes it."""+ from core.inference import audio_inputs++ directory = audio_inputs.inputs_dir()+ stem = uuid.uuid4().hex+ src = directory / f".{stem}.upload.tmp"+ dst = directory / f".{stem}.{rate}.wav"+ try:+ src.write_bytes(raw)+ # Refused past 30 minutes like every other transcription, never cut short.+ audio_inputs.transcode(src, dst, rate = rate, mono = True)+ finally:+ src.unlink(missing_ok = True)+ return dst++def _source_transcript(result: dict, source, speakers: bool) -> dict:out = dict(result)if not (speakers and out.get("speakers")):@@ -24952,18 +25223,7 @@ async def transcribe_audio_source(engine = body.engine or await asyncio.to_thread(_stt_engine_for_model, body.model)serving_engine = _resolve_serving_stt_engine(engine)caps = await asyncio.to_thread(stt_capabilities.capabilities_for, body.model, serving_engine)- if body.timestamps and caps["timestamps"] == "unsupported":- raise HTTPException(- status_code = 422,- detail = "This model cannot add timestamps. Pick Qwen3-ASR (audio.cpp), Parakeet-TDT, "- "MOSS-Transcribe-Diarize or VibeVoice-ASR.",- )- if body.speakers and not caps["speakers"]:- raise HTTPException(- status_code = 422,- detail = "This model cannot tell speakers apart. Pick MOSS-Transcribe-Diarize or "- "VibeVoice-ASR.",- )+ _refuse_stt_caps(caps, timestamps = body.timestamps, speakers = body.speakers)rate = stt_details.FIXED_SPAN_RATES.get(caps["family"], 16000)def _prepare():@@ -46322,7 +46582,7 @@ async def list_gallery_audio()-@studio_router.get("/audio/gallery/{audio_id}/file")[email protected]("/audio/gallery/{audio_id}/file")async def get_gallery_audio_file(audio_id: str, current_subject: str = Depends(get_current_subject)):@@ -46478,7 +46738,7 @@ async def clear_gallery_audio(return {"removed": removed}-@studio_router.post("/audio/inputs", status_code = 201, response_model = AudioInputRecord)[email protected]("/audio/inputs", status_code = 201, response_model = AudioInputRecord)async def upload_audio_input(request: Request,response: Response,@@ -46502,10 +46762,10 @@ async def upload_audio_input(raise _audio_source_error(exc) from Noneif not created:response.status_code = 200- return AudioInputRecord(**record)+ return AudioInputRecord(**{**record, "url": _v1_url(request, record["url"])})-@studio_router.get("/audio/inputs/{input_id}/file")[email protected]("/audio/inputs/{input_id}/file")async def get_audio_input_file(input_id: str, current_subject: str = Depends(get_current_subject)):from core.inference import audio_inputsfrom fastapi.responses import FileResponse@@ -46518,7 +46778,7 @@ async def get_audio_input_file(input_id: str, current_subject: str = Depends(get)-@studio_router.delete("/audio/inputs/{input_id}")[email protected]("/audio/inputs/{input_id}")async def delete_audio_input(input_id: str, current_subject: str = Depends(get_current_subject)):from core.inference import audio_inputsif not await asyncio.to_thread(audio_inputs.delete, input_id):@@ -46571,17 +46831,23 @@ async def transcribe_audio_input()-@studio_router.get("/audio/voices", response_model = AudioVoiceListResponse)-async def list_audio_voices(current_subject: str = Depends(get_current_subject)):[email protected]("/audio/voices", response_model = AudioVoiceListResponse)+async def list_audio_voices(request: Request, current_subject: str = Depends(get_current_subject)):from core.inference import audio_voices- return AudioVoiceListResponse(voices = await asyncio.to_thread(audio_voices.list_voices))+ voices = await asyncio.to_thread(audio_voices.list_voices)+ return AudioVoiceListResponse(+ voices = [{**voice, "url": _v1_url(request, voice["url"])} for voice in voices]+ )-@studio_router.post("/audio/voices", status_code = 201, response_model = AudioVoice)[email protected]("/audio/voices", status_code = 201, response_model = AudioVoice)async def create_audio_voice(- body: AudioVoiceCreate, current_subject: str = Depends(get_current_subject)+ body: AudioVoiceCreate,+ request: Request,+ current_subject: str = Depends(get_current_subject),):from core.inference import audio_inputs, audio_voices+def _create() -> dict:source = audio_inputs.resolve_source(body.source.model_dump(exclude_none = True))return audio_voices.create(@@ -46590,15 +46856,17 @@ async def create_audio_voice()try:- return AudioVoice(**await asyncio.to_thread(_create))+ record = await asyncio.to_thread(_create)except audio_inputs.AudioInputError as exc:raise _audio_source_error(exc) from None+ return AudioVoice(**{**record, "url": _v1_url(request, record["url"])})-@studio_router.patch("/audio/voices/{voice_id}", response_model = AudioVoice)[email protected]("/audio/voices/{voice_id}", response_model = AudioVoice)async def update_audio_voice(voice_id: str,body: AudioVoicePatch,+ request: Request,current_subject: str = Depends(get_current_subject),):from core.inference import audio_inputs, audio_voices@@ -46611,10 +46879,10 @@ async def update_audio_voice(raise _audio_source_error(exc) from Noneif record is None:raise HTTPException(status_code = 404, detail = "Voice not found.")- return AudioVoice(**record)+ return AudioVoice(**{**record, "url": _v1_url(request, record["url"])})-@studio_router.delete("/audio/voices/{voice_id}")[email protected]("/audio/voices/{voice_id}")async def delete_audio_voice(voice_id: str, current_subject: str = Depends(get_current_subject)):from core.inference import audio_voicesif not await asyncio.to_thread(audio_voices.delete, voice_id):@@ -46622,7 +46890,7 @@ async def delete_audio_voice(voice_id: str, current_subject: str = Depends(get_creturn {"removed": True}-@studio_router.get("/audio/voices/{voice_id}/file")[email protected]("/audio/voices/{voice_id}/file")async def get_audio_voice_file(voice_id: str, current_subject: str = Depends(get_current_subject)):from core.inference import audio_voicesfrom fastapi.responses import FileResponsediff --git a/studio/backend/tests/test_account_media_isolation.py b/studio/backend/tests/test_account_media_isolation.pyindex df2ec1802d8..3934b81a174 100644--- a/studio/backend/tests/test_account_media_isolation.py+++ b/studio/backend/tests/test_account_media_isolation.py@@ -79,6 +79,8 @@ def _client(account):app.dependency_overrides[get_current_subject] = subjectapp.dependency_overrides[authenticated_via_api_key] = lambda: False+ # Mounted like main.py: the audio file, input and voice routes are on both.+ app.include_router(inference.router, prefix = "/api/inference")app.include_router(inference.studio_router, prefix = "/api/inference")app.include_router(video.router, prefix = "/api/inference")app.include_router(video.openai_router, prefix = "/v1")diff --git a/studio/backend/tests/test_audio_inputs.py b/studio/backend/tests/test_audio_inputs.pyindex 6e55f30c140..47cbaa36d43 100644--- a/studio/backend/tests/test_audio_inputs.py+++ b/studio/backend/tests/test_audio_inputs.py@@ -21,6 +21,7 @@ import core.inference.audio_gallery as galleryimport utils.upload_limits as limitsfrom auth.authentication import get_current_subjectfrom core.inference import audio_inputs+from utils.api_errors import install_api_error_handlersINPUTS = "/api/inference/audio/inputs"@@ -89,8 +90,11 @@ def client():from routes import inferenceapp = FastAPI()+ install_api_error_handlers(app)app.dependency_overrides[get_current_subject] = lambda: "tester"+ app.include_router(inference.router, prefix = "/api/inference")app.include_router(inference.studio_router, prefix = "/api/inference")+ app.include_router(inference.router, prefix = "/v1")yield TestClient(app)@@ -320,3 +324,34 @@ def test_the_byte_cap_counts_only_kept_inputs(client, monkeypatch):False,True,]+++def test_an_upload_prepared_for_audio_cpp_is_mono_at_the_rate_and_leaves_no_copy():+ from routes.inference import _prepared_upload++ path = _prepared_upload(encode("mp3", "libmp3lame", 44100, "stereo", 1.0), 24000)+ assert path.parent == audio_inputs.inputs_dir()+ rate, channels, frames = _frames(path)+ assert (rate, channels) == (24000, 1) and abs(frames - 24000) < 2400+ assert [p.name for p in path.parent.iterdir()] == [path.name]+++def test_an_upload_too_long_for_audio_cpp_is_refused_not_cut(monkeypatch):+ from routes.inference import _prepared_upload++ monkeypatch.setattr(audio_inputs, "MAX_SECONDS", 0.5)+ with pytest.raises(audio_inputs.AudioInputError) as refused:+ _prepared_upload(wav_bytes(1.0), 16000)+ assert refused.value.status == 413+ assert list(audio_inputs.inputs_dir().iterdir()) == []+++def test_the_sweep_reaps_a_prepared_upload_a_crash_left_behind():+ from routes.inference import _prepared_upload++ stale = _prepared_upload(wav_bytes(0.2), 16000)+ waiting = _prepared_upload(wav_bytes(0.2), 16000)+ for path, age in ((stale, audio_inputs.TTL_SECONDS + 60), (waiting, 2 * 60 * 60)):+ os.utime(path, (time.time() - age, time.time() - age))+ audio_inputs.sweep()+ assert not stale.exists() and waiting.exists()diff --git a/studio/backend/tests/test_audio_run_route.py b/studio/backend/tests/test_audio_run_route.pyindex 69cd178037e..7385e3a5826 100644--- a/studio/backend/tests/test_audio_run_route.py+++ b/studio/backend/tests/test_audio_run_route.py@@ -203,7 +203,6 @@ def test_a_clone_run_hands_the_worker_an_account_path_and_saves_the_clip(stub, t{"voice_ref": "/etc/passwd"},{"file": "x.wav"},{"url": "http://x"},- {"model": "other/model"},{"inputs": {"reference": {"path": "/etc/passwd"}}},{"inputs": {"reference": {"input_id": "a" * 32, "voice_ref": "/etc/passwd"}}},{"inputs": {"reference": {"input_id": "a" * 32}, "voice_ref": "/x.wav"}},@@ -1809,3 +1808,173 @@ def test_a_convert_expiry_names_the_side_that_expired():assert _convert_role_error(gone, "target").detail == CONVERT_EXPIRED_DETAIL["target"]other = AudioInputError(404, "That clip is gone.")assert _convert_role_error(other, "target") is other+++def _v1(account):+ from utils.api_errors import install_api_error_handlers++ client = _client(account)+ install_api_error_handlers(client.app)+ client.app.include_router(inference.router, prefix = "/v1")+ return client+++def switches(stub, monkeypatch):+ seen = []++ async def record(model, *_args, **kwargs):+ seen.append((model, kwargs.get("require_audio_workflow")))++ monkeypatch.setattr(inference, "_maybe_auto_switch_model", record)+ return seen+++def test_a_named_model_is_switched_to_with_the_workflow_it_must_run(stub, switches):+ voice = _voice(ALICE, _input(ALICE), transcript = "hi")+ with _client(ALICE) as client:+ clone = _run(client, model = QWEN3_BASE, inputs = {"reference": {"voice_id": voice["id"]}})+ speak = _run(+ client,+ workflow = "speak",+ model = QWEN3_BASE,+ inputs = {"reference": {"voice_id": voice["id"]}},+ )+ unnamed = _run(client, inputs = {"reference": {"voice_id": voice["id"]}})+ assert [r.status_code for r in (clone, speak, unnamed)] == [200, 200, 200]+ assert switches == [+ (QWEN3_BASE, "clone"),+ (QWEN3_BASE, "clone"),+ (inference._RELOAD_ONLY_MODEL, "clone"),+ ]+++def test_a_named_separation_model_is_switched_to_before_the_track_is_read(sep, switches):+ with _client(ALICE) as client:+ response = _separate(client, {"input_id": _input(ALICE)}, model = HTDEMUCS_6)+ assert response.status_code == 200, response.text+ assert switches == [(HTDEMUCS_6, "separate")]+++def test_a_v1_run_is_monitored_and_its_clips_are_served_under_v1(stub):+ from core.inference.api_monitor import api_monitor++ voice = _voice(ALICE, _input(ALICE), transcript = "hi")+ body = {+ "workflow": "clone",+ "text": "Hello.",+ "inputs": {"reference": {"voice_id": voice["id"]}},+ }+ api_monitor.clear()+ with _v1(ALICE) as client:+ studio = client.post("/api/inference/audio/run", json = body)+ assert api_monitor.snapshot(include_details = False) == []+ api = client.post("/v1/audio/run", json = body)+ assert api.status_code == 200, api.text+ url = api.json()["clips"][0]["url"]+ assert url.startswith("/v1/audio/gallery/")+ assert client.get(url).content[:4] == b"RIFF"+ assert studio.json()["clips"][0]["url"].startswith("/api/inference/audio/gallery/")+ (row,) = api_monitor.snapshot(include_details = False)+ assert (row["endpoint"], row["status"], row["model"]) == (+ "/v1/audio/run",+ "completed",+ QWEN3_BASE,+ )++[email protected]("as_object", [False, True])+def test_speech_in_a_saved_voice_clones_with_its_transcript(stub, tmp_path, as_object):+ voice = _voice(ALICE, _input(ALICE), transcript = "Okay, I'm Cemo.")+ ref = {"id": voice["id"]} if as_object else voice["id"]+ with _v1(ALICE) as client:+ response = client.post("/v1/audio/speech", json = {"input": "Hello.", "voice": ref})+ assert response.status_code == 200, response.text+ assert response.content[:4] == b"RIFF"+ (call,) = stub["backend"].calls+ assert (call["workflow"], call["reference_text"]) == ("speak", "Okay, I'm Cemo.")+ assert Path(call["audio_inputs"]["reference"]).name.startswith(f"v-{voice['id']}.")+ (meta,) = [json.loads(p.read_text()) for p in _gallery_root(tmp_path).glob("*.json")]+ assert (meta["voice_id"], meta["workflow"]) == (voice["id"], "speak")+++def test_speech_clones_a_reference_and_refuses_it_beside_a_saved_voice(stub):+ input_id = _input(ALICE)+ voice = _voice(ALICE, input_id)+ reference = {"reference": {"input_id": input_id}, "reference_text": "Hi there."}+ with _v1(ALICE) as client:+ both = client.post(+ "/v1/audio/speech", json = {"input": "Hello.", "voice": voice["id"], **reference}+ )+ clone = client.post(+ "/v1/audio/speech", json = {"input": "Hello.", "voice": "alloy", **reference}+ )+ assert (both.status_code, both.json()["error"]["param"]) == (400, "reference")+ assert clone.status_code == 200, clone.text+ (call,) = stub["backend"].calls+ assert (call["workflow"], call["reference_text"]) == ("clone", "Hi there.")+ assert Path(call["audio_inputs"]["reference"]).name.startswith(f"{input_id}.24000.mono")+++def test_speech_uses_a_builtin_speaker_the_model_lists(stub):+ voices = {"name": "voice", "type": "enum", "values": ["Ryan", "Vivian"]}+ backend = stub["use"](QWEN3_BASE, {**_speak_info(), "audio_options": [voices]})+ with _v1(ALICE) as client:+ for body in (+ {"voice": "Ryan"},+ {"voice": "alloy"},+ {"voice": "Ryan", "audio_options": {"voice": "Vivian"}},+ ):+ response = client.post("/v1/audio/speech", json = {"input": "Hello.", **body})+ assert response.status_code == 200, response.text+ assert [call.get("audio_options") for call in backend.calls] == [+ {"voice": "Ryan"},+ None,+ {"voice": "Vivian"},+ ]+++ "fmt, media_type, magic",+ [+ ("mp3", "audio/mpeg", (b"ID3", b"\xff\xfb", b"\xff\xf3")),+ ("flac", "audio/flac", (b"fLaC",)),+ ("opus", "audio/ogg", (b"OggS",)),+ ("aac", "audio/aac", (b"\xff\xf1", b"\xff\xf9")),+ ],+)+def test_speech_encodes_each_format_and_history_keeps_the_wav(+ stub, tmp_path, fmt, media_type, magic+):+ stub["use"](QWEN3_BASE, _speak_info())+ with _v1(ALICE) as client:+ response = client.post("/v1/audio/speech", json = {"input": "Hello.", "response_format": fmt})+ assert response.status_code == 200, response.text+ assert response.headers["content-type"] == media_type+ assert response.content.startswith(magic)+ (clip,) = _gallery_root(tmp_path).glob("*.wav")+ assert clip.read_bytes() == _wav()++[email protected]("rate, channels", [(24000, 1), (44100, 2)])+def test_speech_pcm_is_openais_24k_mono_samples(stub, monkeypatch, rate, channels):+ stub["use"](QWEN3_BASE, _speak_info())+ buf = io.BytesIO()+ with wave.open(buf, "wb") as w:+ w.setnchannels(channels)+ w.setsampwidth(2)+ w.setframerate(rate)+ w.writeframes(b"\x01\x00" * rate * channels)+ monkeypatch.setattr(+ _Backend, "generate_audio_response", lambda self, **kw: (buf.getvalue(), rate)+ )+ with _v1(ALICE) as client:+ response = client.post(+ "/v1/audio/speech", json = {"input": "Hello.", "response_format": "pcm"}+ )+ assert response.status_code == 200, response.text+ assert response.headers["content-type"] == "audio/pcm"+ # One second of audio: 24000 two-byte mono samples, whatever the model's rate and layout.+ assert abs(len(response.content) - 48000) <= 2 * 64+ if (rate, channels) == (24000, 1):+ assert response.content == buf.getvalue()[44:]diff --git a/studio/backend/tests/test_audio_voices.py b/studio/backend/tests/test_audio_voices.pyindex 27e39b58454..0fafd1f451f 100644--- a/studio/backend/tests/test_audio_voices.py+++ b/studio/backend/tests/test_audio_voices.py@@ -126,3 +126,26 @@ def test_deleting_a_voice_drops_its_prepared_copies():assert prepared.is_file() and prepared.name.startswith(f"v-{voice['id']}.")assert audio_voices.delete(voice["id"])assert not prepared.exists()+++def test_a_script_uploads_saves_a_voice_and_fetches_it_under_v1(client):+ headers = {"Content-Type": "application/octet-stream"}+ upload = client.post(+ "/v1/audio/inputs", params = {"name": "me.wav"}, content = wav_bytes(2.0), headers = headers+ )+ assert upload.status_code == 201, upload.text+ source = {"input_id": upload.json()["id"]}+ assert client.get(upload.json()["url"]).content[:4] == b"RIFF"+ created = client.post("/v1/audio/voices", json = {"source": source, "name": "Me"})+ assert created.status_code == 201, created.text+ voice = created.json()+ (listed,) = client.get("/v1/audio/voices").json()["voices"]+ renamed = client.patch(f"/v1/audio/voices/{voice['id']}", json = {"name": "Me 2"}).json()+ for url in (upload.json()["url"], voice["url"], listed["url"], renamed["url"]):+ assert url.startswith("/v1/audio/"), url+ assert client.get(voice["url"]).content[:4] == b"RIFF"+ assert client.get(VOICES).json()["voices"][0]["url"].startswith(VOICES)+ assert client.delete(f"/v1/audio/inputs/{source['input_id']}").json() == {"removed": True}+ missing = client.patch("/v1/audio/voices/" + "a" * 32, json = {"name": "x"})+ assert missing.status_code == 404+ assert missing.json()["error"]["message"] == "Voice not found."diff --git a/studio/backend/tests/test_keyless_api_access.py b/studio/backend/tests/test_keyless_api_access.pyindex c74ec779d14..399f7aac3ec 100644--- a/studio/backend/tests/test_keyless_api_access.py+++ b/studio/backend/tests/test_keyless_api_access.py@@ -117,7 +117,7 @@ INFERENCE_POST_PATHS = "/v1/chat/completions /v1/chat/count_tokens /v1/completiodef test_exact_route_matrix_matches_registered_topology():from routes.inference import router- denied_posts = "/v1/load /v1/unload /v1/validate /v1/generate/stream /v1/audio/speech /v1/images/generations /v1/external/openai/containers/create /v1x/chat".split()+ denied_posts = "/v1/load /v1/unload /v1/validate /v1/generate/stream /v1/audio/speech /v1/audio/run /v1/audio/inputs /v1/audio/voices /v1/images/generations /v1/external/openai/containers/create /v1x/chat".split()allowed = {("POST", path) for path in INFERENCE_POST_PATHS} | {("GET", "/v1/models"), ("GET", "/v1/models/unsloth/model")}denied = {("POST", path) for path in denied_posts} | {diff --git a/studio/backend/tests/test_middleware.py b/studio/backend/tests/test_middleware.pyindex c26d3b07d7d..e812d9b3ed6 100644--- a/studio/backend/tests/test_middleware.py+++ b/studio/backend/tests/test_middleware.py@@ -161,6 +161,13 @@ class TestMaxBodyMiddleware:assert main_module._get_upload_passthrough_request_max_bytes(path + "/") == (upload_request_limit_bytes(STT_AUDIO_RAW_MAX_BYTES)), path+ from utils.upload_limits import AUDIO_INPUT_MAX_BYTES++ for path in ("/v1/audio/inputs", "/api/inference/audio/inputs"):+ assert path in main_module._BODY_UPLOAD_PASSTHROUGH_EXACT_PATHS, path+ assert main_module._get_upload_passthrough_request_max_bytes(path) == (+ AUDIO_INPUT_MAX_BYTES+ ), pathfrom utils.upload_limits import (VIDEO_INPUT_REFERENCE_JSON_MAX_BYTES,VIDEO_INPUT_REFERENCE_MAX_BYTES,diff --git a/studio/backend/tests/test_openai_audio_speech_route.py b/studio/backend/tests/test_openai_audio_speech_route.pyindex 8da84c27a5c..310f956f829 100644--- a/studio/backend/tests/test_openai_audio_speech_route.py+++ b/studio/backend/tests/test_openai_audio_speech_route.py@@ -107,14 +107,27 @@ def test_voice_and_speed_accepted_and_ignored(monkeypatch):assert resp.status_code == 200-def test_non_wav_response_format_is_400(monkeypatch):+def test_unknown_response_format_is_400(monkeypatch):cli, calls, saved = _make_client(monkeypatch)- resp = cli.post("/v1/audio/speech", json = {"input": "hi", "response_format": "mp3"})+ resp = cli.post("/v1/audio/speech", json = {"input": "hi", "response_format": "wma"})assert resp.status_code == 400- assert "mp3" in resp.json()["error"]["message"]+ error = resp.json()["error"]+ assert error["param"] == "response_format"+ assert "'wma'" in error["message"] and "mp3" in error["message"]assert calls == [] # rejected before any generation+def test_sse_stream_format_is_400(monkeypatch):+ cli, calls, saved = _make_client(monkeypatch)+ resp = cli.post("/v1/audio/speech", json = {"input": "hi", "stream_format": "sse"})+ assert (resp.status_code, resp.json()["error"]["param"]) == (400, "stream_format")+ assert calls == []+ assert (+ cli.post("/v1/audio/speech", json = {"input": "hi", "stream_format": "audio"}).status_code+ == 200+ )++def test_null_response_format_means_wav(monkeypatch):cli, calls, saved = _make_client(monkeypatch)resp = cli.post("/v1/audio/speech", json = {"input": "hi", "response_format": None})@@ -436,6 +449,23 @@ def test_external_rejects_non_wav_response_format(monkeypatch):assert speech_calls == []+def test_external_rejects_sse_before_the_upstream_call(monkeypatch):+ cli, _calls, _saved = _make_client(monkeypatch)+ created, speech_calls = _install_external(monkeypatch)+ resp = cli.post(+ "/v1/audio/speech",+ json = {+ "input": "hi",+ "provider_id": "conn-1",+ "model": "kokoro",+ "voice": "alloy",+ "stream_format": "sse",+ },+ )+ assert (resp.status_code, resp.json()["error"]["param"]) == (400, "stream_format")+ assert created == [] and speech_calls == []++def test_external_missing_model_is_400(monkeypatch):cli, calls, saved = _make_client(monkeypatch)_install_external(monkeypatch)@@ -931,6 +961,22 @@ def test_speech_opens_a_monitor_row(monkeypatch):assert rows[0]["model"] == "unsloth/orpheus-3b-0.1-ft"+def test_v1_audio_generate_opens_a_monitor_row_and_the_chat_mount_does_not(monkeypatch):+ cli, calls, saved = _make_client(monkeypatch)+ cli.app.include_router(router, prefix = "/api/inference")+ api_monitor.clear()+ body = {"messages": [{"role": "user", "content": "read me"}]}+ assert cli.post("/api/inference/audio/generate", json = body).status_code == 200+ assert api_monitor.snapshot(include_details = False) == []+ assert cli.post("/v1/audio/generate", json = body).status_code == 200+ (row,) = api_monitor.snapshot(include_details = False)+ assert (row["endpoint"], row["status"], row["model"]) == (+ "/v1/audio/generate",+ "completed",+ "unsloth/orpheus-3b-0.1-ft",+ )++def test_tts_failure_records_an_error_row(monkeypatch):cli, calls, saved = _make_client(monkeypatch, generate = _boom)api_monitor.clear()@@ -945,7 +991,7 @@ def test_rejected_response_format_records_nothing(monkeypatch):# Refused before any work, so it is not traffic the monitor should show.cli, calls, saved = _make_client(monkeypatch)api_monitor.clear()- resp = cli.post("/v1/audio/speech", json = {"input": "hi", "response_format": "mp3"})+ resp = cli.post("/v1/audio/speech", json = {"input": "hi", "response_format": "wma"})assert resp.status_code == 400assert api_monitor.snapshot(include_details = False) == []diff --git a/studio/backend/tests/test_openai_audio_transcriptions_route.py b/studio/backend/tests/test_openai_audio_transcriptions_route.pyindex 78ee761ed3f..31f7f883672 100644--- a/studio/backend/tests/test_openai_audio_transcriptions_route.py+++ b/studio/backend/tests/test_openai_audio_transcriptions_route.py@@ -32,6 +32,7 @@ def _make_client(monkeypatch, transcribe = None):engine = None,request = None,device = None,+ **kwargs,):calls.append({@@ -41,6 +42,7 @@ def _make_client(monkeypatch, transcribe = None):"fast": fast,"engine": engine,"request": request,+ **kwargs,})if transcribe is not None:@@ -911,3 +913,146 @@ def test_the_proxied_row_never_carries_a_local_path(monkeypatch):assert row["model"] == "whisper-v3"# Only the label is redacted; the provider is still asked for what the client sent.assert transcription_calls[-1]["model"] == "/home/ana/models/whisper-v3"+++_TIMED = {+ "text": "Hi there. Bye.",+ "language": "English",+ "duration": 2.5,+ "model": "audio-cpp/audio.cpp-gguf/VibeVoice-ASR-GGUF",+ "segments": [+ {"start": 0.0, "end": 1.0, "text": "Hi there.", "speaker": "0"},+ {"start": 1.2, "end": 2.5, "text": "Bye.", "speaker": "1"},+ ],+ "words": [+ {"start": 0.0, "end": 0.4, "word": "Hi"},+ {"start": 0.4, "end": 1.0, "word": "there."},+ ],+ "speakers": [{"id": "0", "label": "Speaker 1"}, {"id": "1", "label": "Speaker 2"}],+}+++def _audio_cpp(+ monkeypatch,+ tmp_path,+ result = _TIMED,+ **caps,+):+ """An audio.cpp ASR model; the upload's prepared copy is a file the route must remove."""+ from core.inference import stt_capabilities++ async def _result(raw):+ return dict(result)++ cli, calls = _make_client(monkeypatch, transcribe = _result)+ monkeypatch.setattr(routes_module, "_stt_engine_for_model", lambda model: "audiocpp")+ monkeypatch.setattr(routes_module, "_resolve_serving_stt_engine", lambda engine: engine)+ caps = {"family": "vibevoice_asr", "timestamps": "always", "speakers": True, **caps}+ monkeypatch.setattr(stt_capabilities, "capabilities_for", lambda model, engine: caps)+ prepared = []++ def _prepare(raw, rate):+ path = tmp_path / f"upload.{rate}.wav"+ path.write_bytes(raw)+ prepared.append(path)+ return path++ monkeypatch.setattr(routes_module, "_prepared_upload", _prepare)+ return cli, calls, prepared+++def test_audio_cpp_verbose_json_has_openai_segments_and_words(monkeypatch, tmp_path):+ openai_types = pytest.importorskip("openai.types.audio.transcription_verbose")+ cli, calls, prepared = _audio_cpp(monkeypatch, tmp_path)+ data = {"response_format": "verbose_json", "timestamp_granularities[]": ["segment", "word"]}+ resp = _post(cli, data = data)+ assert resp.status_code == 200, resp.text+ body = resp.json()+ openai_types.TranscriptionVerbose.model_validate(body)+ assert body["language"] == "English"+ assert [(s["id"], s["start"], s["end"], s["text"]) for s in body["segments"]] == [+ (0, 0.0, 1.0, "Hi there."),+ (1, 1.2, 2.5, "Bye."),+ ]+ assert body["words"] == [+ {"word": "Hi", "start": 0.0, "end": 0.4},+ {"word": "there.", "start": 0.4, "end": 1.0},+ ]+ # VibeVoice times its spans at 24 kHz, so the upload is prepared at that rate and removed.+ (path,) = prepared+ assert path.name == "upload.24000.wav" and not path.exists()+ assert calls[0]["source_path"] == path and calls[0]["timestamps"] is True+++def test_audio_cpp_diarized_json_names_speakers(monkeypatch, tmp_path):+ cli, calls, prepared = _audio_cpp(monkeypatch, tmp_path)+ resp = _post(cli, data = {"response_format": "diarized_json"})+ assert resp.status_code == 200, resp.text+ body = resp.json()+ assert (body["task"], body["duration"], body["text"]) == ("transcribe", 2.5, "Hi there. Bye.")+ assert body["segments"] == [+ {+ "type": "transcript.text.segment",+ "id": "seg_0",+ "start": 0.0,+ "end": 1.0,+ "text": "Hi there.",+ "speaker": "Speaker 1",+ },+ {+ "type": "transcript.text.segment",+ "id": "seg_1",+ "start": 1.2,+ "end": 2.5,+ "text": "Bye.",+ "speaker": "Speaker 2",+ },+ ]+ assert calls[0]["timestamps"] is False+++ "caps, data, detail",+ [+ ({"speakers": False}, {"response_format": "diarized_json"}, "cannot tell speakers apart"),+ (+ {"timestamps": "unsupported"},+ {"response_format": "verbose_json", "timestamp_granularities[]": "word"},+ "cannot add timestamps",+ ),+ ],+)+def test_audio_cpp_refuses_what_the_model_cannot_add(monkeypatch, tmp_path, caps, data, detail):+ cli, calls, prepared = _audio_cpp(monkeypatch, tmp_path, **caps)+ resp = _post(cli, data = data)+ assert resp.status_code == 422+ assert detail in resp.json()["error"]["message"]+ assert calls == [] and prepared == []++[email protected]("fmt", ["json", "text", "diarized_json"])+def test_audio_cpp_granularities_need_verbose_json(monkeypatch, tmp_path, fmt):+ cli, calls, prepared = _audio_cpp(monkeypatch, tmp_path)+ resp = _post(cli, data = {"response_format": fmt, "timestamp_granularities[]": "word"})+ assert (resp.status_code, resp.json()["error"]["param"]) == (400, "timestamp_granularities")+ assert calls == [] and prepared == []+++def test_diarized_json_on_whisper_is_refused_before_any_work(monkeypatch):+ cli, calls = _make_client(monkeypatch)+ resp = _post(cli, data = {"response_format": "diarized_json"})+ assert resp.status_code == 422+ assert calls == []++[email protected]("reported, status", [("English", 200), (None, 501)])+def test_audio_cpp_verbose_json_needs_no_language_when_the_model_reports_one(+ monkeypatch, tmp_path, reported, status+):+ cli, calls, prepared = _audio_cpp(+ monkeypatch, tmp_path, result = {**_TIMED, "language": reported}+ )+ resp = _post(cli, data = {"response_format": "verbose_json"})+ assert resp.status_code == status, resp.text+ if status == 200:+ assert resp.json()["language"] == "English"diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 019666615af..0a16be9fde1 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -11821,6 +11821,52 @@ def test_speech_switch_admission(monkeypatch, audio_type, context, text, instrucassert len(recorder.calls) == 1+ "workflow, speech_type, workflows, admitted",+ [+ ("separate", None, ["separate"], True),+ ("separate", None, [], False),+ ("clone", "audiocpp_tts", ["speak"], False),+ ("clone", "audiocpp_tts", ["speak", "clone"], True),+ ],+)+def test_audio_workflow_switch_admission(monkeypatch, workflow, speech_type, workflows, admitted):+ backend, recorder = _speech_case(monkeypatch, speech_type, gguf = False)+ seen = []+ monkeypatch.setattr(+ inference_route, "_target_audio_workflows", lambda *a: seen.append(a) or workflows+ )+ call = inference_route._maybe_auto_switch_model(+ "org/B-GGUF",+ object(),+ "tester",+ require_speech = speech_type is not None,+ require_audio_workflow = workflow,+ )+ if admitted:+ asyncio.run(call)+ assert len(recorder.calls) == 1+ else:+ with pytest.raises(HTTPException) as error:+ asyncio.run(call)+ assert error.value.status_code == 400+ assert error.value.detail["error"]["param"] == "model"+ # Refused before the resident model is evicted.+ assert recorder.calls == [] and backend.model_identifier == "org/A-GGUF"+ assert seen == [("/local/B", speech_type)]+++def test_target_audio_workflows_reads_audio_cpp_targets_from_the_cache(monkeypatch):+ from core.inference import audio_cpp_models++ sep = SimpleNamespace(workflows = {"separate": None})+ monkeypatch.setattr(audio_cpp_models, "resolve", lambda target, network: sep)+ workflows = inference_route._target_audio_workflows+ assert workflows("audio-cpp/audio.cpp-gguf/HTDemucs-6stems-GGUF", None) == ["separate"]+ assert workflows("/local/csm", "csm") == ["speak"]+ assert workflows("/local/chat", None) == []++@pytest.mark.parametrize("audio_type", ["snac", "bicodec", "dac", "higgs_tts2"])@pytest.mark.parametrize("token", [None, "caller-token"])def test_speech_switch_preserves_staged_assets_and_identity(monkeypatch, audio_type, token):diff --git a/tests/test_runtime_text_encoding.py b/tests/test_runtime_text_encoding.pyindex 5bd40b5dcef..32c764b846c 100644--- a/tests/test_runtime_text_encoding.py+++ b/tests/test_runtime_text_encoding.py@@ -67,7 +67,7 @@ REVIEWED_VENDORED_OFFENDERS = {# keyed on path + expression, not line, so unrelated edits above it do not break the scan.REVIEWED_NON_FILE_OPEN = ("studio/backend/core/inference/audio_inputs.py",- "stream.codec_context.open()",+ "stream.codec_context.open(strict=True)",)GUARDED_METHODS = {"read_text", "write_text"}# path classes, so an unbound `Path.open(p)` shifts every argument one right.@@ -431,12 +431,12 @@ def test_skips_foreign_openers_and_readers():def test_skips_only_the_reviewed_pyav_codec_open():- pyav = "\n" * 263 + "stream.codec_context.open()\n"+ pyav = "\n" * 263 + "stream.codec_context.open(strict = True)\n"path = "studio/backend/core/inference/audio_inputs.py"assert not _offenders_in(pyav, path)assert not _offenders_in("\n" + pyav, path)assert _offenders_in(pyav, "studio/backend/core/inference/other.py")- assert _offenders_in("\n" * 263 + "config.codec_context.open()\n", path)+ assert _offenders_in("\n" * 263 + "config.codec_context.open(strict = True)\n", path)def test_test_trees_are_out_of_scope():