Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: complete the OpenAI-compatible audio API (#12812)

Etherl0423db52379c
diff --git a/studio/backend/core/inference/audio_inputs.py b/studio/backend/core/inference/audio_inputs.py
index 4c117baebd1..f5a1e5b4b8e 100644
--- a/studio/backend/core/inference/audio_inputs.py
+++ b/studio/backend/core/inference/audio_inputs.py
@@ -220,6 +220,68 @@ def transcode(
return {"duration_s": round(frames / rate, 3), "sample_rate": rate, "channels": channels}
+# /v1/audio/speech formats besides WAV: (container, encoder, media type, sample rate it needs).
+_SPEECH_ENCODINGS = {
+ "mp3": ("mp3", "mp3", "audio/mpeg", None),
+ "opus": ("ogg", "libopus", "audio/ogg", 48000),
+ "aac": ("adts", "aac", "audio/aac", None),
+ "flac": ("flac", "flac", "audio/flac", None),
+ # OpenAI's pcm is 24 kHz 16-bit mono; its clients assume that rate.
+ "pcm": ("s16le", "pcm_s16le", "audio/pcm", 24000),
+}
+
+
+def speech_formats() -> list[str]:
+ """``wav`` plus each speech format whose encoder this FFmpeg build has."""
+ import av
+
+ formats = ["wav"]
+ for fmt, (_container, codec, _media_type, _rate) in _SPEECH_ENCODINGS.items():
+ try:
+ av.codec.Codec(codec, "w")
+ except Exception: # noqa: BLE001 - not in this build
+ continue
+ formats.append(fmt)
+ return formats
+
+
+def encode_wav(wav_bytes: bytes, fmt: str) -> tuple[bytes, str]:
+ """A WAV re-encoded as ``fmt`` (a ``speech_formats`` name but wav); ``(bytes, media type)``.
+ ``pcm`` is 24 kHz 16-bit little-endian mono samples, with no header."""
+ import io
+
+ import av
+
+ container_format, codec, media_type, fixed_rate = _SPEECH_ENCODINGS[fmt]
+ out = io.BytesIO()
+ with _av_open(av, io.BytesIO(wav_bytes)) as src:
+ audio = src.streams.audio[0]
+ rate = fixed_rate or audio.rate
+ # A WAV with no channel mask reads as "1 channels", which no encoder takes.
+ layout = "stereo" if audio.channels == 2 and fmt != "pcm" else "mono"
+ with av.open(out, "w", format = container_format) as dst:
+ stream = dst.add_stream(codec, rate = rate, layout = layout)
+ stream.codec_context.open(strict = True) # sets frame_size, which mp3, aac and opus need
+ resampler = av.AudioResampler(
+ format = stream.format.name,
+ layout = layout,
+ rate = rate,
+ frame_size = stream.frame_size or None,
+ )
+
+ def mux(frames) -> None:
+ for frame in frames:
+ for packet in stream.encode(frame):
+ dst.mux(packet)
+
+ for frame in src.decode(audio):
+ frame.pts = None
+ mux(resampler.resample(frame))
+ mux(resampler.resample(None))
+ mux([None])
+ return out.getvalue(), media_type
+
+
async def save_stream(
chunks: AsyncIterable[bytes],
name: Optional[str],
@@ -417,7 +479,8 @@ def sweep(
age = now - _mtime(path)
if name.startswith(".") and name.endswith(".tmp") and age > _STALE_TMP_SECONDS:
path.unlink(missing_ok = True)
- elif name.startswith(("c-", "v-")) and name.endswith(".wav") and age > ttl:
+ # "." = a crashed timed transcription's upload; full TTL (aligner downloads take an hour).
+ elif name.startswith(("c-", "v-", ".")) and name.endswith(".wav") and age > ttl:
path.unlink(missing_ok = True)
# Music run folders a crash left behind.
runs = directory / "runs"
diff --git a/studio/backend/main.py b/studio/backend/main.py
index 82a1fad436a..97d1dab44d8 100644
--- a/studio/backend/main.py
+++ b/studio/backend/main.py
@@ -1402,7 +1402,7 @@ _VIDEO_MULTIPART_UPLOAD_PATHS = (
)
_LIBRARY_UPLOAD_PATH = "/api/library/uploads"
# Streamed to disk and capped by the route itself; buffering here would hold 200 MiB in memory.
-_AUDIO_INPUT_UPLOAD_PATH = "/api/inference/audio/inputs"
+_AUDIO_INPUT_UPLOAD_PATHS = ("/api/inference/audio/inputs", "/v1/audio/inputs")
_BODY_UPLOAD_PASSTHROUGH_PREFIXES = (
*_DATASET_UPLOAD_PASSTHROUGH_PREFIXES,
_DATA_RECIPE_UNSTRUCTURED_UPLOAD_PASSTHROUGH_PREFIX,
@@ -1410,7 +1410,7 @@ _BODY_UPLOAD_PASSTHROUGH_PREFIXES = (
# Matched by EXACT path (multipart uploads only), so sibling JSON sub-routes keep the normal cap.
_BODY_UPLOAD_PASSTHROUGH_EXACT_PATHS = (
_DIFFUSION_DATASET_UPLOAD_PATH,
- _AUDIO_INPUT_UPLOAD_PATH,
+ *_AUDIO_INPUT_UPLOAD_PATHS,
*_STT_MULTIPART_UPLOAD_PATHS,
*_VIDEO_MULTIPART_UPLOAD_PATHS,
_LIBRARY_UPLOAD_PATH,
@@ -1434,7 +1434,7 @@ def _get_upload_passthrough_request_max_bytes(path: str) -> int:
)
if path.rstrip("/") == _LIBRARY_UPLOAD_PATH:
return upload_request_limit_bytes(LIBRARY_UPLOAD_MAX_BYTES)
- if path.rstrip("/") == _AUDIO_INPUT_UPLOAD_PATH:
+ if path.rstrip("/") in _AUDIO_INPUT_UPLOAD_PATHS:
return AUDIO_INPUT_MAX_BYTES
# The trailing-slash variant reaches this middleware BEFORE the router's redirect_slashes
# 307, so it must resolve to the same cap. JSON sub-routes keep extra path components.
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 8beb9329e52..055a5ef4442 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -5197,8 +5197,8 @@ class ImageGenerationResponse(BaseModel):
class AudioSpeechRequest(BaseModel):
"""OpenAI ``CreateSpeechRequest`` for ``POST /v1/audio/speech``.
- ``voice`` and ``speed`` are accepted for client compatibility but unused: no loaded
- TTS backend has voice or rate plumbing (CSM is fixed to speaker 0)."""
+ ``voice`` picks a saved voice or a built-in speaker; OpenAI's own voice names are accepted
+ and ignored. ``reference`` clones from uploaded audio, a history clip or a saved voice."""
input: str = Field(..., min_length = 1, description = "The text to synthesize.")
model: Optional[str] = Field(
@@ -5208,11 +5208,31 @@ class AudioSpeechRequest(BaseModel):
"model auto-switch is on; otherwise the loaded audio model is used."
),
)
- voice: Optional[str] = Field(None, description = "Voice name (accepted, unused).")
+ voice: Optional[Union[str, "AudioSpeechVoice"]] = Field(
+ None,
+ description = (
+ 'A saved voice\'s id (or {"id": ...}), else a built-in speaker of the model. Other '
+ "names, such as OpenAI's, are ignored."
+ ),
+ )
+ reference: Optional["AudioSourceRef"] = Field(
+ None,
+ description = "[x-unsloth] Audio to clone: an uploaded input, a history clip or a saved voice.",
+ )
+ reference_text: Optional[str] = Field(
+ None,
+ max_length = 4000,
+ description = "[x-unsloth] What is said in the reference, for models that use it.",
+ )
response_format: Optional[str] = Field(
- "wav", description = "Output container. Only 'wav' is supported."
+ "wav", description = "Output format: wav, mp3, flac, opus, aac or pcm."
+ )
+ stream_format: Optional[Literal["audio", "sse"]] = Field(
+ None, description = "Only 'audio' (the whole file in one response) is supported."
+ )
+ speed: Optional[float] = Field(
+ None, ge = 0.25, le = 4.0, description = "Speech rate, applied where the model supports it."
)
- speed: Optional[float] = Field(None, description = "Speech rate (accepted, unused).")
instructions: Optional[str] = Field(
None,
description = "Scene or music-description instructions for compatible audio models.",
@@ -5345,6 +5365,12 @@ class AudioSourceRef(BaseModel):
return self
+class AudioSpeechVoice(BaseModel):
+ """OpenAI's custom-voice object: here a saved voice's id."""
+
+ id: str = Field(..., pattern = _AUDIO_ID_PATTERN)
+
+
class AudioRunInputs(BaseModel):
model_config = ConfigDict(extra = "forbid")
@@ -5410,6 +5436,12 @@ class AudioRunRequest(BaseModel):
model_config = ConfigDict(extra = "forbid")
workflow: Literal["clone", "speak", "edit", "convert", "music", "separate"]
+ model: Optional[str] = Field(
+ None,
+ max_length = 512,
+ description = "A downloaded audio model to load first when model auto-switch is on; "
+ "otherwise the loaded model runs it",
+ )
# Required except for a conversion or a separation (the routes answer those given text with a 400).
text: Optional[str] = None
language: Optional[str] = Field(None, max_length = 64)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 231ab0f9818..52bde1565f9 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3704,8 +3704,11 @@ from models.inference import (
AudioInputRecord,
AudioInputTranscribeRequest,
AudioInputTranscript,
+ AudioRunInputs,
AudioRunRequest,
AudioRunResponse,
+ AudioSourceRef,
+ AudioSpeechVoice,
AudioVoice,
AudioVoiceCreate,
AudioVoiceListResponse,
@@ -9411,6 +9414,20 @@ def _target_speech_audio_type(
return None
+def _target_audio_workflows(target_id: str, speech_type: Optional[str]) -> list[str]:
+ """The Audio page workflows a switch target runs, answered from the HF cache alone."""
+ from core.inference.audio_cpp_models import looks_like_audio_cpp, resolve
+ from core.inference.audio_workflows import status_audio_workflows
+
+ if not looks_like_audio_cpp(target_id):
+ return status_audio_workflows(speech_type is not None, speech_type)
+ try:
+ model = resolve(target_id, network = False)
+ except Exception: # noqa: BLE001 - unreadable from the cache: it runs nothing
+ return []
+ return list(model.workflows) if model is not None else []
+
+
@dataclass(frozen = True)
class _SpeechCodecPreflightResult:
cache_environment: Optional[dict[str, str]] = None
@@ -10894,6 +10911,7 @@ async def _maybe_auto_switch_model(
audio_preflight: Optional[dict] = None,
image_preflight: Optional[dict] = None,
require_speech: bool = False,
+ require_audio_workflow: Optional[str] = None,
speech_budget: Optional[dict] = None,
tool_images_only: bool = False,
) -> None:
@@ -10922,7 +10940,8 @@ async def _maybe_auto_switch_model(
``audio_preflight`` carries an audio upload whose format-dependent
checks only the resolved target can decide; see
:func:`_preflight_audio_for_switch`. ``image_preflight`` does the same for
- non-GGUF image count and byte validation.
+ non-GGUF image count and byte validation. ``require_audio_workflow`` refuses a target that
+ cannot run that Audio page workflow (``separate``, ``clone``...).
"""
# A text-only GGUF reads tool-result images as a note, so only audio or video needs its mmproj.
gguf_requires_vision = (
@@ -11382,6 +11401,21 @@ async def _maybe_auto_switch_model(
param = "instructions",
),
)
+ if (
+ require_audio_workflow
+ and resolved is not None
+ and require_audio_workflow
+ not in await asyncio.to_thread(_target_audio_workflows, target_id, speech_type)
+ ):
+ raise HTTPException(
+ status_code = 400,
+ detail = openai_error_body(
+ f"The requested model cannot run the {require_audio_workflow} workflow.",
+ status = 400,
+ code = "invalid_value",
+ param = "model",
+ ),
+ )
# resolver branch only, like the probe above: a reload-stash restore changes no format.
if audio_preflight is not None and resolved is not None:
await _preflight_audio_for_switch(audio_preflight, target_is_gguf)
@@ -22241,20 +22275,27 @@ async def _generate_tts_wav(
requested_model: str = _RELOAD_ONLY_MODEL,
run_inputs: Optional[dict] = None,
stats_holder: Optional[dict] = None,
+ voice: Optional[str] = None,
) -> tuple[bytes, int, str, Optional[str]]:
"""Shared core of /audio/generate, /audio/speech and /audio/run. Returns
(wav_bytes, sample_rate, model_name, audio_type). ``run_inputs`` is /audio/run's ``workflow``,
``audio_inputs`` (role -> server-local WAV path), ``reference_text``, ``speed`` and ``convert``;
- a conversion fills ``audio_inputs`` via ``prepare_audio_inputs(model_info)``."""
+ a conversion fills ``audio_inputs`` via ``prepare_audio_inputs(model_info)``. ``voice`` is a
+ built-in speaker, used when the loaded model lists it and the request set none."""
# A named target must be budgeted against its own context after preflight.
if requested_model == _RELOAD_ONLY_MODEL:
_raise_if_prompt_leaves_no_speech_budget(text)
+ workflow = (run_inputs or {}).get("workflow")
+ if workflow == "speak" and (run_inputs or {}).get("audio_inputs"):
+ # Speaking in a saved voice is cloning, as _audio_request_problem reads it.
+ workflow = "clone"
await _maybe_auto_switch_model(
requested_model,
request,
current_subject,
claim_resident = False,
require_speech = True,
+ require_audio_workflow = workflow,
speech_budget = (
None
if requested_model == _RELOAD_ONLY_MODEL
@@ -22318,6 +22359,16 @@ async def _generate_tts_wav(
audio_type = model_info.get("audio_type")
audio_family = model_info.get("audio_family")
supported_audio_types = _TRANSFORMERS_TTS_AUDIO_TYPES
+ speakers = next(
+ (
+ o.get("values")
+ for o in model_info.get("audio_options") or ()
+ if o.get("name") == "voice"
+ ),
+ None,
+ )
+ if voice in (speakers or ()) and "voice" not in (payload.audio_options or {}):
+ payload.audio_options = {**(payload.audio_options or {}), "voice": voice}
_audio_model_id = getattr(backend, "active_model_name", None) or model_name
gen = lambda: backend.generate_audio_response(
text = text,
@@ -22562,7 +22613,7 @@ async def generate_audio(
text = last_user_msg["content"]
tts_stats: dict = {}
- wav_bytes, sample_rate, model_name, audio_type = await _generate_tts_wav(
+ generate = lambda: _generate_tts_wav( # noqa: E731
text,
payload,
request,
@@ -22573,6 +22624,18 @@ async def generate_audio(
requested_model = _switch_model_for_payload(payload) or _RELOAD_ONLY_MODEL,
stats_holder = tts_stats,
)
+ # Chat completions monitors the turns it hands here, and the chat's read-aloud is no API traffic.
+ if getattr(getattr(request, "url", None), "path", "").rstrip("/") != "/v1/audio/generate":
+ wav_bytes, sample_rate, model_name, audio_type = await generate()
+ else:
+ async with _monitored_media_request(
+ request,
+ model = public_model_id(payload.model) or payload.model,
+ prompt = text,
+ subject = current_subject,
+ ) as monitor_id:
+ wav_bytes, sample_rate, model_name, audio_type = await generate()
+ api_monitor.relabel(monitor_id, model_name)
truncated = bool((tts_stats.get("stats") or {}).get("truncated"))
persisted_clip = await asyncio.to_thread(
_persist_tts_clip, wav_bytes, sample_rate, text, model_name, audio_type
@@ -22604,6 +22667,14 @@ def _audio_source_error(exc) -> HTTPException:
return HTTPException(status_code = exc.status, detail = exc.detail)
+def _v1_url(request: Request, url: str) -> str:
+ """A Studio file URL on the mount the client called, so a /v1 client needs only that base URL."""
+ prefix = "/api/inference/"
+ if request.url.path.startswith("/v1/") and url.startswith(prefix):
+ return "/v1/" + url[len(prefix) :]
+ return url
+
+
CONVERT_EXPIRED_DETAIL = {
"source": "This recording expired. Add it again.",
"target": "This target voice expired. Add it again.",
@@ -22733,6 +22804,7 @@ async def _run_audio_edit(
# For the request check; the worker gets the text as the prompt.
"text": body.text,
},
+ requested_model = body.model or _RELOAD_ONLY_MODEL,
)
source_record = None
try:
@@ -22940,11 +23012,12 @@ async def _run_music_workflow(
from core.inference.audio_cpp_music import frames_for, timeout_seconds
await _maybe_auto_switch_model(
- _RELOAD_ONLY_MODEL,
+ body.model or _RELOAD_ONLY_MODEL,
request,
current_subject,
claim_resident = False,
require_speech = True,
+ require_audio_workflow = "music",
)
backend = await asyncio.to_thread(get_inference_backend)
if not backend.active_model_name:
@@ -23151,6 +23224,7 @@ async def _run_audio_convert(
"convert_inputs": tuple(resolved),
"prepare_audio_inputs": _prepare,
},
+ requested_model = body.model or _RELOAD_ONLY_MODEL,
)
builtin = None
if target is not None:
@@ -23202,12 +23276,27 @@ async def run_audio_workflow(
"""Run one Audio page workflow (Clone, Speak in a saved voice, Edit, Convert, Music or
Separate); clips go to history. Audio is named by id and resolved here in the caller's account;
the worker gets a prepared copy's path, never bytes."""
- if body.workflow == "separate":
- return await _run_separation(body, request, current_subject)
- import base64
+ if not request.url.path.startswith("/v1/"):
+ return await _dispatch_audio_run(body, request, current_subject)
+ # Only API runs are monitored, like /v1/audio/speech; the Audio page's are not.
+ async with _monitored_media_request(
+ request,
+ model = public_model_id(body.model) or body.model,
+ prompt = body.text or body.workflow,
+ subject = current_subject,
+ ) as monitor_id:
+ response = await _dispatch_audio_run(body, request, current_subject)
+ api_monitor.relabel(monitor_id, response.model)
+ for clip in response.clips:
+ clip.url = _v1_url(request, clip.url)
+ return response
- from core.inference import audio_inputs
+async def _dispatch_audio_run(
+ body: AudioRunRequest, request: Request, current_subject: str
+) -> AudioRunResponse:
+ if body.workflow == "separate":
+ return await _run_separation(body, request, current_subject)
if body.workflow == "convert":
return await _run_audio_convert(body, request, current_subject)
if body.workflow == "edit":
@@ -23224,6 +23313,18 @@ async def run_audio_workflow(
status_code = 400,
detail = "A music mode, lyrics, variations, a source clip and edits apply to Music only.",
)
+ record, wav_bytes, sample_rate, model_name = await _speak(
+ body, request, current_subject, requested_model = body.model or _RELOAD_ONLY_MODEL
+ )
+ return _audio_run_response(record, wav_bytes, sample_rate, model_name)
+
+
+async def _speak(
+ body: AudioRunRequest, request: Request, current_subject: str, **tts_kwargs
+) -> tuple[Optional[dict[str, Any]], bytes, int, str]:
+ """Clone, or Speak in a saved voice, and keep the clip in history; shared by /audio/run and
+ /audio/speech. ``(record, wav_bytes, sample_rate, model_name)``."""
+ from core.inference import audio_inputs
if body.workflow == "clone" and body.inputs.reference is None:
raise HTTPException(status_code = 400, detail = "Add a reference clip to clone.")
@@ -23269,6 +23370,7 @@ async def run_audio_workflow(
"reference_text": reference_text,
"speed": body.speed,
},
+ **tts_kwargs,
)
extra_meta: dict[str, Any] = {
"role": "output",
@@ -23290,7 +23392,7 @@ async def run_audio_workflow(
extra_meta,
body.workflow,
)
- return _audio_run_response(record, wav_bytes, sample_rate, model_name)
+ return record, wav_bytes, sample_rate, model_name
# Stems in the order the Separate page lists them; ids a runtime adds follow in its own order.
@@ -23453,7 +23555,11 @@ async def _run_separation(
if source_ref.voice_id:
raise HTTPException(status_code = 400, detail = "Pick a track to separate, not a saved voice.")
await _maybe_auto_switch_model(
- _RELOAD_ONLY_MODEL, request, current_subject, claim_resident = False
+ body.model or _RELOAD_ONLY_MODEL,
+ request,
+ current_subject,
+ claim_resident = False,
+ require_audio_workflow = "separate",
)
backend = await asyncio.to_thread(get_inference_backend)
model_info = (
@@ -23553,7 +23659,7 @@ async def _external_tts_speech(body: AudioSpeechRequest, request: Request) -> Re
"""Proxy CreateSpeech to a saved connection, so read-aloud skips the local model slot."""
provider_id = (body.provider_id or "").strip()
external_model = (body.model or "").strip()
- external_voice = (body.voice or "").strip()
+ external_voice = body.voice.strip() if isinstance(body.voice, str) else ""
if not provider_id:
raise HTTPException(status_code = 400, detail = "provider_id cannot be empty.")
if not external_model:
@@ -23666,9 +23772,14 @@ async def openai_audio_speech(
With ``provider_id`` the request is proxied. Otherwise an omitted ``model`` serves the
resident model. A named downloaded local model is loaded when auto-switch is on; when
it is off, the shared switch guard rejects a different recognized local model.
- Local ``voice``/``speed`` are ignored and only WAV is supported."""
+ A saved voice (``voice``) or ``reference`` clones; history keeps the clip as WAV."""
+ fmt = (body.response_format or "wav").strip().lower()
+ # Neither path streams, a saved connection included.
+ if body.stream_format == "sse":
+ _raise_unsupported_openai_parameter(
+ "stream_format", "stream_format 'sse' is not supported. Use 'audio'."
+ )
if body.provider_id:
- fmt = (body.response_format or "wav").strip().lower()
if fmt != "wav":
raise HTTPException(
status_code = 400,
@@ -23693,12 +23804,43 @@ async def openai_audio_speech(
subject = current_subject,
):
return await _external_tts_speech(body, request)
- fmt = (body.response_format or "wav").strip().lower()
- if fmt != "wav":
- raise HTTPException(
- status_code = 400,
- detail = f"Unsupported response_format '{body.response_format}'. Only 'wav' is supported.",
+ from core.inference import audio_inputs, audio_voices
+
+ formats = await asyncio.to_thread(audio_inputs.speech_formats)
+ if fmt not in formats:
+ _raise_unsupported_openai_parameter(
+ "response_format",
+ f"Unsupported response_format '{body.response_format}'. Use {', '.join(formats)}.",
+ )
+ voice = body.voice
+ voice_id = voice.id if isinstance(voice, AudioSpeechVoice) else None
+ if isinstance(voice, str) and await asyncio.to_thread(audio_voices.get, voice):
+ voice_id = voice
+ if voice_id and body.reference is not None:
+ _raise_unsupported_openai_parameter(
+ "reference", "Send a saved voice or a reference, not both."
)
+ run = None
+ if voice_id or body.reference is not None:
+ from fastapi.exceptions import RequestValidationError
+ from pydantic import ValidationError
+ try:
+ run = AudioRunRequest(
+ workflow = "speak" if voice_id else "clone",
+ text = body.input,
+ language = body.language,
+ instructions = body.instructions,
+ inputs = AudioRunInputs(
+ reference = body.reference or AudioSourceRef(voice_id = voice_id),
+ reference_text = body.reference_text,
+ ),
+ options = body.audio_options,
+ speed = body.speed,
+ seed = body.seed,
+ max_tokens = body.max_new_tokens or AUDIO_GENERATION_MAX_TOKENS,
+ )
+ except ValidationError as exc:
+ raise RequestValidationError(exc.errors()) from None
# The tts core reads its sampling knobs from a chat request shape; defaults apply here.
# Keep the full speech ceiling unless a client supplies the native max_new_tokens
# extension. MiniMax gets its documented 750-frame default deeper in the shared core;
@@ -23725,19 +23867,31 @@ async def openai_audio_speech(
prompt = body.input,
subject = current_subject,
) as monitor_id:
- wav_bytes, sample_rate, model_name, audio_type = await _generate_tts_wav(
- body.input,
- payload,
- request,
- current_subject,
- speech_api_default_max_tokens = body.max_new_tokens is None,
- requested_model = body.model or _RELOAD_ONLY_MODEL,
- )
+ tts_kwargs = {
+ "speech_api_default_max_tokens": body.max_new_tokens is None,
+ "requested_model": body.model or _RELOAD_ONLY_MODEL,
+ }
+ if run is not None:
+ _, wav_bytes, sample_rate, model_name = await _speak(
+ run, request, current_subject, **tts_kwargs
+ )
+ else:
+ wav_bytes, sample_rate, model_name, audio_type = await _generate_tts_wav(
+ body.input,
+ payload,
+ request,
+ current_subject,
+ voice = voice if isinstance(voice, str) else None,
+ **tts_kwargs,
+ )
+ await asyncio.to_thread(
+ _persist_tts_clip, wav_bytes, sample_rate, body.input, model_name, audio_type
+ )
api_monitor.relabel(monitor_id, model_name)
- await asyncio.to_thread(
- _persist_tts_clip, wav_bytes, sample_rate, body.input, model_name, audio_type
- )
- return Response(content = wav_bytes, media_type = "audio/wav")
+ if fmt == "wav":
+ return Response(content = wav_bytes, media_type = "audio/wav")
+ content, media_type = await asyncio.to_thread(audio_inputs.encode_wav, wav_bytes, fmt)
+ return Response(content = content, media_type = media_type)
async def _external_stt_transcription(
@@ -23863,14 +24017,15 @@ async def openai_audio_transcriptions(
With ``provider_id`` set, the request is proxied to that saved connection.
Otherwise ``model`` maps to a sidecar model id, with ``whisper-1`` or nothing
selecting the default. ``response_format`` supports ``json``, ``text`` and
- ``verbose_json``."""
+ ``verbose_json``; an audio.cpp model also gives segments, words (``timestamp_granularities``)
+ and ``diarized_json``."""
fmt = (response_format or "json").strip().lower()
- if fmt not in ("json", "text", "verbose_json"):
+ if fmt not in ("json", "text", "verbose_json", "diarized_json"):
raise HTTPException(
status_code = 400,
detail = (
f"Unsupported response_format '{response_format}'. "
- "Use 'json', 'text' or 'verbose_json'."
+ "Use 'json', 'text', 'verbose_json' or 'diarized_json'."
),
)
# UploadFile spools to disk, but an unbounded read materializes the whole upload in
@@ -23901,11 +24056,40 @@ async def openai_audio_transcriptions(
)
api_monitor.set_reply(monitor_id, _external_transcript_preview(response))
return response
- # Refused before any work, like the response_format check above: the sidecar echoes the
- # language it was given and detects none, so an auto-detect request has nothing truthful
- # for verbose_json's required language field, and guessing would silently label a
- # Japanese clip "en". isinstance because a direct call passes the raw Form default.
- if fmt == "verbose_json" and not (isinstance(language, str) and language.strip()):
+ from core.inference import audio_inputs, stt_capabilities, stt_details
+
+ # openai's whisper-1 placeholder means the default transcription model, not a sidecar id
+ sidecar_model = None if model in (None, "", "whisper-1") else model
+ engine = _stt_engine_for_model(sidecar_model)
+ # isinstance because a direct call passes the raw Form default.
+ granularities = timestamp_granularities if isinstance(timestamp_granularities, list) else []
+ if granularities and fmt != "verbose_json":
+ # OpenAI's contract; also spares an aligner download whose timings would be dropped.
+ _raise_unsupported_openai_parameter(
+ "timestamp_granularities",
+ "timestamp_granularities needs response_format 'verbose_json'.",
+ )
+ has_language = isinstance(language, str) and language.strip()
+ serving_engine = _resolve_serving_stt_engine(engine)
+ timed = serving_engine == "audiocpp" and (
+ fmt in ("verbose_json", "diarized_json") or bool(granularities)
+ )
+ if timed or fmt == "diarized_json":
+ caps = await asyncio.to_thread(
+ stt_capabilities.capabilities_for, sidecar_model, serving_engine
+ )
+ _refuse_stt_caps(
+ caps, timestamps = timed and bool(granularities), speakers = fmt == "diarized_json"
+ )
+ if timed:
+ # The prepared copy is read in place, past the sidecar's own size check.
+ if len(raw) > _MAX_AUDIO_RAW_BYTES:
+ raise HTTPException(status_code = 413, detail = "Audio is too large.")
+ elif fmt == "verbose_json" and not has_language:
+ # Refused before any work, like the response_format check above: the sidecar echoes the
+ # language it was given and detects none, so an auto-detect request has nothing truthful
+ # for verbose_json's required language field, and guessing would silently label a
+ # Japanese clip "en".
raise HTTPException(
status_code = 501,
detail = (
@@ -23914,9 +24098,9 @@ async def openai_audio_transcriptions(
"'json' or 'text'."
),
)
- if isinstance(timestamp_granularities, list) and timestamp_granularities:
+ elif granularities:
# Accepting the parameter and returning neither words nor segments would look like
- # the audio simply had none. The sidecar reports no per-token timing today.
+ # the audio simply had none. Only audio.cpp models report timings.
raise HTTPException(
status_code = 400,
detail = (
@@ -23924,8 +24108,6 @@ async def openai_audio_transcriptions(
"Use a saved STT connection with provider_id for word or segment timings."
),
)
- # openai's whisper-1 placeholder means the default transcription model, not a sidecar id
- sidecar_model = None if model in (None, "", "whisper-1") else model
# The relabel below only lands on success; a sidecar failure would otherwise strand the
# raw client string, which may be a host path, on the terminal row.
_opening_label = sidecar_model or "whisper-1"
@@ -23935,15 +24117,36 @@ async def openai_audio_transcriptions(
prompt = file.filename or "audio",
subject = current_subject,
) as monitor_id:
- result = await _transcribe_audio_result(
- raw,
- sidecar_model,
- language,
- fast = False,
- engine = _stt_engine_for_model(sidecar_model),
- request = request,
- )
+ path = None
+ try:
+ if timed:
+ rate = stt_details.FIXED_SPAN_RATES.get(caps["family"], 16000)
+ try:
+ path = await asyncio.to_thread(_prepared_upload, raw, rate)
+ except audio_inputs.AudioInputError as exc:
+ raise _audio_source_error(exc) from None
+ result = await _transcribe_audio_result(
+ raw,
+ sidecar_model,
+ language,
+ fast = False,
+ engine = engine,
+ request = request,
+ source_path = path,
+ timestamps = bool(granularities),
+ )
+ finally:
+ if path is not None:
+ await asyncio.to_thread(path.unlink, True)
text = str(result.get("text", ""))
+ # Required by OpenAI's schema: a null fails validation inside the official clients.
+ detected = str(result.get("language") or (language if has_language else "")).strip()
+ if fmt == "verbose_json" and not detected:
+ raise HTTPException(
+ status_code = 501,
+ detail = "This model did not report the language of the audio. Send the "
+ "'language' parameter, or use response_format 'json' or 'text'.",
+ )
# Same path-free treatment the image route gives its label: sidecar ids are
# curated or owner/model today, but the row goes out over the tunnel.
_served_model = str(result.get("model") or "")
@@ -23952,20 +24155,54 @@ async def openai_audio_transcriptions(
if fmt == "text":
return PlainTextResponse(content = text)
- if fmt == "verbose_json":
- _duration = result.get("duration")
+ _duration = result.get("duration")
+ # Unlike the language, this is not a guess: duration is None only for a clip that decoded
+ # to no samples, and that clip really is zero seconds.
+ duration = float(_duration) if isinstance(_duration, (int, float)) else 0.0
+ segments = result.get("segments") or []
+ if fmt == "diarized_json":
+ labels = {s["id"]: s["label"] for s in result.get("speakers") or ()}
return JSONResponse(
content = {
"task": "transcribe",
- # Both are required by OpenAI's schema, so a null fails validation inside
- # the official clients. The request is refused above with no language.
- "language": str(result.get("language") or language).strip(),
- # Unlike the language, this is not a guess: duration is None only for a
- # clip that decoded to no samples, and that clip really is zero seconds.
- "duration": float(_duration) if isinstance(_duration, (int, float)) else 0.0,
+ "duration": duration,
"text": text,
+ "segments": [
+ {
+ "type": "transcript.text.segment",
+ "id": f"seg_{index}",
+ "start": s["start"],
+ "end": s["end"],
+ "text": s["text"],
+ "speaker": labels.get(s.get("speaker"), s.get("speaker") or ""),
+ }
+ for index, s in enumerate(segments)
+ ],
}
)
+ if fmt == "verbose_json":
+ content = {"task": "transcribe", "language": detected, "duration": duration, "text": text}
+ if segments:
+ content["segments"] = [
+ {
+ "id": index,
+ "seek": 0,
+ "start": s["start"],
+ "end": s["end"],
+ "text": s["text"],
+ "tokens": [],
+ "temperature": 0.0,
+ "avg_logprob": 0.0,
+ "compression_ratio": 0.0,
+ "no_speech_prob": 0.0,
+ }
+ for index, s in enumerate(segments)
+ ]
+ if "word" in granularities and result.get("words"):
+ content["words"] = [
+ {"word": w["word"], "start": w["start"], "end": w["end"]} for w in result["words"]
+ ]
+ return JSONResponse(content = content)
return JSONResponse(content = {"text": text})
@@ -24928,6 +25165,40 @@ async def transcribe_audio_raw(
)
+def _refuse_stt_caps(caps: dict, *, timestamps: bool, speakers: bool) -> None:
+ """422 when the model cannot add what the transcript asks for."""
+ if timestamps and caps["timestamps"] == "unsupported":
+ raise HTTPException(
+ status_code = 422,
+ detail = "This model cannot add timestamps. Pick Qwen3-ASR (audio.cpp), Parakeet-TDT, "
+ "MOSS-Transcribe-Diarize or VibeVoice-ASR.",
+ )
+ if speakers and not caps["speakers"]:
+ raise HTTPException(
+ status_code = 422,
+ detail = "This model cannot tell speakers apart. Pick MOSS-Transcribe-Diarize or "
+ "VibeVoice-ASR.",
+ )
+
+
+def _prepared_upload(raw: bytes, rate: int) -> Path:
+ """An upload decoded to a mono WAV at ``rate`` in this account's inputs, for audio.cpp to read
+ in place. The caller removes it."""
+ from core.inference import audio_inputs
+
+ directory = audio_inputs.inputs_dir()
+ stem = uuid.uuid4().hex
+ src = directory / f".{stem}.upload.tmp"
+ dst = directory / f".{stem}.{rate}.wav"
+ try:
+ src.write_bytes(raw)
+ # Refused past 30 minutes like every other transcription, never cut short.
+ audio_inputs.transcode(src, dst, rate = rate, mono = True)
+ finally:
+ src.unlink(missing_ok = True)
+ return dst
+
+
def _source_transcript(result: dict, source, speakers: bool) -> dict:
out = dict(result)
if not (speakers and out.get("speakers")):
@@ -24952,18 +25223,7 @@ async def transcribe_audio_source(
engine = body.engine or await asyncio.to_thread(_stt_engine_for_model, body.model)
serving_engine = _resolve_serving_stt_engine(engine)
caps = await asyncio.to_thread(stt_capabilities.capabilities_for, body.model, serving_engine)
- if body.timestamps and caps["timestamps"] == "unsupported":
- raise HTTPException(
- status_code = 422,
- detail = "This model cannot add timestamps. Pick Qwen3-ASR (audio.cpp), Parakeet-TDT, "
- "MOSS-Transcribe-Diarize or VibeVoice-ASR.",
- )
- if body.speakers and not caps["speakers"]:
- raise HTTPException(
- status_code = 422,
- detail = "This model cannot tell speakers apart. Pick MOSS-Transcribe-Diarize or "
- "VibeVoice-ASR.",
- )
+ _refuse_stt_caps(caps, timestamps = body.timestamps, speakers = body.speakers)
rate = stt_details.FIXED_SPAN_RATES.get(caps["family"], 16000)
def _prepare():
@@ -46322,7 +46582,7 @@ async def list_gallery_audio(
)
-@studio_router.get("/audio/gallery/{audio_id}/file")
[email protected]("/audio/gallery/{audio_id}/file")
async def get_gallery_audio_file(
audio_id: str, current_subject: str = Depends(get_current_subject)
):
@@ -46478,7 +46738,7 @@ async def clear_gallery_audio(
return {"removed": removed}
-@studio_router.post("/audio/inputs", status_code = 201, response_model = AudioInputRecord)
[email protected]("/audio/inputs", status_code = 201, response_model = AudioInputRecord)
async def upload_audio_input(
request: Request,
response: Response,
@@ -46502,10 +46762,10 @@ async def upload_audio_input(
raise _audio_source_error(exc) from None
if not created:
response.status_code = 200
- return AudioInputRecord(**record)
+ return AudioInputRecord(**{**record, "url": _v1_url(request, record["url"])})
-@studio_router.get("/audio/inputs/{input_id}/file")
[email protected]("/audio/inputs/{input_id}/file")
async def get_audio_input_file(input_id: str, current_subject: str = Depends(get_current_subject)):
from core.inference import audio_inputs
from fastapi.responses import FileResponse
@@ -46518,7 +46778,7 @@ async def get_audio_input_file(input_id: str, current_subject: str = Depends(get
)
-@studio_router.delete("/audio/inputs/{input_id}")
[email protected]("/audio/inputs/{input_id}")
async def delete_audio_input(input_id: str, current_subject: str = Depends(get_current_subject)):
from core.inference import audio_inputs
if not await asyncio.to_thread(audio_inputs.delete, input_id):
@@ -46571,17 +46831,23 @@ async def transcribe_audio_input(
)
-@studio_router.get("/audio/voices", response_model = AudioVoiceListResponse)
-async def list_audio_voices(current_subject: str = Depends(get_current_subject)):
[email protected]("/audio/voices", response_model = AudioVoiceListResponse)
+async def list_audio_voices(request: Request, current_subject: str = Depends(get_current_subject)):
from core.inference import audio_voices
- return AudioVoiceListResponse(voices = await asyncio.to_thread(audio_voices.list_voices))
+ voices = await asyncio.to_thread(audio_voices.list_voices)
+ return AudioVoiceListResponse(
+ voices = [{**voice, "url": _v1_url(request, voice["url"])} for voice in voices]
+ )
-@studio_router.post("/audio/voices", status_code = 201, response_model = AudioVoice)
[email protected]("/audio/voices", status_code = 201, response_model = AudioVoice)
async def create_audio_voice(
- body: AudioVoiceCreate, current_subject: str = Depends(get_current_subject)
+ body: AudioVoiceCreate,
+ request: Request,
+ current_subject: str = Depends(get_current_subject),
):
from core.inference import audio_inputs, audio_voices
+
def _create() -> dict:
source = audio_inputs.resolve_source(body.source.model_dump(exclude_none = True))
return audio_voices.create(
@@ -46590,15 +46856,17 @@ async def create_audio_voice(
)
try:
- return AudioVoice(**await asyncio.to_thread(_create))
+ record = await asyncio.to_thread(_create)
except audio_inputs.AudioInputError as exc:
raise _audio_source_error(exc) from None
+ return AudioVoice(**{**record, "url": _v1_url(request, record["url"])})
-@studio_router.patch("/audio/voices/{voice_id}", response_model = AudioVoice)
[email protected]("/audio/voices/{voice_id}", response_model = AudioVoice)
async def update_audio_voice(
voice_id: str,
body: AudioVoicePatch,
+ request: Request,
current_subject: str = Depends(get_current_subject),
):
from core.inference import audio_inputs, audio_voices
@@ -46611,10 +46879,10 @@ async def update_audio_voice(
raise _audio_source_error(exc) from None
if record is None:
raise HTTPException(status_code = 404, detail = "Voice not found.")
- return AudioVoice(**record)
+ return AudioVoice(**{**record, "url": _v1_url(request, record["url"])})
-@studio_router.delete("/audio/voices/{voice_id}")
[email protected]("/audio/voices/{voice_id}")
async def delete_audio_voice(voice_id: str, current_subject: str = Depends(get_current_subject)):
from core.inference import audio_voices
if not await asyncio.to_thread(audio_voices.delete, voice_id):
@@ -46622,7 +46890,7 @@ async def delete_audio_voice(voice_id: str, current_subject: str = Depends(get_c
return {"removed": True}
-@studio_router.get("/audio/voices/{voice_id}/file")
[email protected]("/audio/voices/{voice_id}/file")
async def get_audio_voice_file(voice_id: str, current_subject: str = Depends(get_current_subject)):
from core.inference import audio_voices
from fastapi.responses import FileResponse
diff --git a/studio/backend/tests/test_account_media_isolation.py b/studio/backend/tests/test_account_media_isolation.py
index df2ec1802d8..3934b81a174 100644
--- a/studio/backend/tests/test_account_media_isolation.py
+++ b/studio/backend/tests/test_account_media_isolation.py
@@ -79,6 +79,8 @@ def _client(account):
app.dependency_overrides[get_current_subject] = subject
app.dependency_overrides[authenticated_via_api_key] = lambda: False
+ # Mounted like main.py: the audio file, input and voice routes are on both.
+ app.include_router(inference.router, prefix = "/api/inference")
app.include_router(inference.studio_router, prefix = "/api/inference")
app.include_router(video.router, prefix = "/api/inference")
app.include_router(video.openai_router, prefix = "/v1")
diff --git a/studio/backend/tests/test_audio_inputs.py b/studio/backend/tests/test_audio_inputs.py
index 6e55f30c140..47cbaa36d43 100644
--- a/studio/backend/tests/test_audio_inputs.py
+++ b/studio/backend/tests/test_audio_inputs.py
@@ -21,6 +21,7 @@ import core.inference.audio_gallery as gallery
import utils.upload_limits as limits
from auth.authentication import get_current_subject
from core.inference import audio_inputs
+from utils.api_errors import install_api_error_handlers
INPUTS = "/api/inference/audio/inputs"
@@ -89,8 +90,11 @@ def client():
from routes import inference
app = FastAPI()
+ install_api_error_handlers(app)
app.dependency_overrides[get_current_subject] = lambda: "tester"
+ app.include_router(inference.router, prefix = "/api/inference")
app.include_router(inference.studio_router, prefix = "/api/inference")
+ app.include_router(inference.router, prefix = "/v1")
yield TestClient(app)
@@ -320,3 +324,34 @@ def test_the_byte_cap_counts_only_kept_inputs(client, monkeypatch):
False,
True,
]
+
+
+def test_an_upload_prepared_for_audio_cpp_is_mono_at_the_rate_and_leaves_no_copy():
+ from routes.inference import _prepared_upload
+
+ path = _prepared_upload(encode("mp3", "libmp3lame", 44100, "stereo", 1.0), 24000)
+ assert path.parent == audio_inputs.inputs_dir()
+ rate, channels, frames = _frames(path)
+ assert (rate, channels) == (24000, 1) and abs(frames - 24000) < 2400
+ assert [p.name for p in path.parent.iterdir()] == [path.name]
+
+
+def test_an_upload_too_long_for_audio_cpp_is_refused_not_cut(monkeypatch):
+ from routes.inference import _prepared_upload
+
+ monkeypatch.setattr(audio_inputs, "MAX_SECONDS", 0.5)
+ with pytest.raises(audio_inputs.AudioInputError) as refused:
+ _prepared_upload(wav_bytes(1.0), 16000)
+ assert refused.value.status == 413
+ assert list(audio_inputs.inputs_dir().iterdir()) == []
+
+
+def test_the_sweep_reaps_a_prepared_upload_a_crash_left_behind():
+ from routes.inference import _prepared_upload
+
+ stale = _prepared_upload(wav_bytes(0.2), 16000)
+ waiting = _prepared_upload(wav_bytes(0.2), 16000)
+ for path, age in ((stale, audio_inputs.TTL_SECONDS + 60), (waiting, 2 * 60 * 60)):
+ os.utime(path, (time.time() - age, time.time() - age))
+ audio_inputs.sweep()
+ assert not stale.exists() and waiting.exists()
diff --git a/studio/backend/tests/test_audio_run_route.py b/studio/backend/tests/test_audio_run_route.py
index 69cd178037e..7385e3a5826 100644
--- a/studio/backend/tests/test_audio_run_route.py
+++ b/studio/backend/tests/test_audio_run_route.py
@@ -203,7 +203,6 @@ def test_a_clone_run_hands_the_worker_an_account_path_and_saves_the_clip(stub, t
{"voice_ref": "/etc/passwd"},
{"file": "x.wav"},
{"url": "http://x"},
- {"model": "other/model"},
{"inputs": {"reference": {"path": "/etc/passwd"}}},
{"inputs": {"reference": {"input_id": "a" * 32, "voice_ref": "/etc/passwd"}}},
{"inputs": {"reference": {"input_id": "a" * 32}, "voice_ref": "/x.wav"}},
@@ -1809,3 +1808,173 @@ def test_a_convert_expiry_names_the_side_that_expired():
assert _convert_role_error(gone, "target").detail == CONVERT_EXPIRED_DETAIL["target"]
other = AudioInputError(404, "That clip is gone.")
assert _convert_role_error(other, "target") is other
+
+
+def _v1(account):
+ from utils.api_errors import install_api_error_handlers
+
+ client = _client(account)
+ install_api_error_handlers(client.app)
+ client.app.include_router(inference.router, prefix = "/v1")
+ return client
+
+
+def switches(stub, monkeypatch):
+ seen = []
+
+ async def record(model, *_args, **kwargs):
+ seen.append((model, kwargs.get("require_audio_workflow")))
+
+ monkeypatch.setattr(inference, "_maybe_auto_switch_model", record)
+ return seen
+
+
+def test_a_named_model_is_switched_to_with_the_workflow_it_must_run(stub, switches):
+ voice = _voice(ALICE, _input(ALICE), transcript = "hi")
+ with _client(ALICE) as client:
+ clone = _run(client, model = QWEN3_BASE, inputs = {"reference": {"voice_id": voice["id"]}})
+ speak = _run(
+ client,
+ workflow = "speak",
+ model = QWEN3_BASE,
+ inputs = {"reference": {"voice_id": voice["id"]}},
+ )
+ unnamed = _run(client, inputs = {"reference": {"voice_id": voice["id"]}})
+ assert [r.status_code for r in (clone, speak, unnamed)] == [200, 200, 200]
+ assert switches == [
+ (QWEN3_BASE, "clone"),
+ (QWEN3_BASE, "clone"),
+ (inference._RELOAD_ONLY_MODEL, "clone"),
+ ]
+
+
+def test_a_named_separation_model_is_switched_to_before_the_track_is_read(sep, switches):
+ with _client(ALICE) as client:
+ response = _separate(client, {"input_id": _input(ALICE)}, model = HTDEMUCS_6)
+ assert response.status_code == 200, response.text
+ assert switches == [(HTDEMUCS_6, "separate")]
+
+
+def test_a_v1_run_is_monitored_and_its_clips_are_served_under_v1(stub):
+ from core.inference.api_monitor import api_monitor
+
+ voice = _voice(ALICE, _input(ALICE), transcript = "hi")
+ body = {
+ "workflow": "clone",
+ "text": "Hello.",
+ "inputs": {"reference": {"voice_id": voice["id"]}},
+ }
+ api_monitor.clear()
+ with _v1(ALICE) as client:
+ studio = client.post("/api/inference/audio/run", json = body)
+ assert api_monitor.snapshot(include_details = False) == []
+ api = client.post("/v1/audio/run", json = body)
+ assert api.status_code == 200, api.text
+ url = api.json()["clips"][0]["url"]
+ assert url.startswith("/v1/audio/gallery/")
+ assert client.get(url).content[:4] == b"RIFF"
+ assert studio.json()["clips"][0]["url"].startswith("/api/inference/audio/gallery/")
+ (row,) = api_monitor.snapshot(include_details = False)
+ assert (row["endpoint"], row["status"], row["model"]) == (
+ "/v1/audio/run",
+ "completed",
+ QWEN3_BASE,
+ )
+
+
[email protected]("as_object", [False, True])
+def test_speech_in_a_saved_voice_clones_with_its_transcript(stub, tmp_path, as_object):
+ voice = _voice(ALICE, _input(ALICE), transcript = "Okay, I'm Cemo.")
+ ref = {"id": voice["id"]} if as_object else voice["id"]
+ with _v1(ALICE) as client:
+ response = client.post("/v1/audio/speech", json = {"input": "Hello.", "voice": ref})
+ assert response.status_code == 200, response.text
+ assert response.content[:4] == b"RIFF"
+ (call,) = stub["backend"].calls
+ assert (call["workflow"], call["reference_text"]) == ("speak", "Okay, I'm Cemo.")
+ assert Path(call["audio_inputs"]["reference"]).name.startswith(f"v-{voice['id']}.")
+ (meta,) = [json.loads(p.read_text()) for p in _gallery_root(tmp_path).glob("*.json")]
+ assert (meta["voice_id"], meta["workflow"]) == (voice["id"], "speak")
+
+
+def test_speech_clones_a_reference_and_refuses_it_beside_a_saved_voice(stub):
+ input_id = _input(ALICE)
+ voice = _voice(ALICE, input_id)
+ reference = {"reference": {"input_id": input_id}, "reference_text": "Hi there."}
+ with _v1(ALICE) as client:
+ both = client.post(
+ "/v1/audio/speech", json = {"input": "Hello.", "voice": voice["id"], **reference}
+ )
+ clone = client.post(
+ "/v1/audio/speech", json = {"input": "Hello.", "voice": "alloy", **reference}
+ )
+ assert (both.status_code, both.json()["error"]["param"]) == (400, "reference")
+ assert clone.status_code == 200, clone.text
+ (call,) = stub["backend"].calls
+ assert (call["workflow"], call["reference_text"]) == ("clone", "Hi there.")
+ assert Path(call["audio_inputs"]["reference"]).name.startswith(f"{input_id}.24000.mono")
+
+
+def test_speech_uses_a_builtin_speaker_the_model_lists(stub):
+ voices = {"name": "voice", "type": "enum", "values": ["Ryan", "Vivian"]}
+ backend = stub["use"](QWEN3_BASE, {**_speak_info(), "audio_options": [voices]})
+ with _v1(ALICE) as client:
+ for body in (
+ {"voice": "Ryan"},
+ {"voice": "alloy"},
+ {"voice": "Ryan", "audio_options": {"voice": "Vivian"}},
+ ):
+ response = client.post("/v1/audio/speech", json = {"input": "Hello.", **body})
+ assert response.status_code == 200, response.text
+ assert [call.get("audio_options") for call in backend.calls] == [
+ {"voice": "Ryan"},
+ None,
+ {"voice": "Vivian"},
+ ]
+
+
+ "fmt, media_type, magic",
+ [
+ ("mp3", "audio/mpeg", (b"ID3", b"\xff\xfb", b"\xff\xf3")),
+ ("flac", "audio/flac", (b"fLaC",)),
+ ("opus", "audio/ogg", (b"OggS",)),
+ ("aac", "audio/aac", (b"\xff\xf1", b"\xff\xf9")),
+ ],
+)
+def test_speech_encodes_each_format_and_history_keeps_the_wav(
+ stub, tmp_path, fmt, media_type, magic
+):
+ stub["use"](QWEN3_BASE, _speak_info())
+ with _v1(ALICE) as client:
+ response = client.post("/v1/audio/speech", json = {"input": "Hello.", "response_format": fmt})
+ assert response.status_code == 200, response.text
+ assert response.headers["content-type"] == media_type
+ assert response.content.startswith(magic)
+ (clip,) = _gallery_root(tmp_path).glob("*.wav")
+ assert clip.read_bytes() == _wav()
+
+
[email protected]("rate, channels", [(24000, 1), (44100, 2)])
+def test_speech_pcm_is_openais_24k_mono_samples(stub, monkeypatch, rate, channels):
+ stub["use"](QWEN3_BASE, _speak_info())
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as w:
+ w.setnchannels(channels)
+ w.setsampwidth(2)
+ w.setframerate(rate)
+ w.writeframes(b"\x01\x00" * rate * channels)
+ monkeypatch.setattr(
+ _Backend, "generate_audio_response", lambda self, **kw: (buf.getvalue(), rate)
+ )
+ with _v1(ALICE) as client:
+ response = client.post(
+ "/v1/audio/speech", json = {"input": "Hello.", "response_format": "pcm"}
+ )
+ assert response.status_code == 200, response.text
+ assert response.headers["content-type"] == "audio/pcm"
+ # One second of audio: 24000 two-byte mono samples, whatever the model's rate and layout.
+ assert abs(len(response.content) - 48000) <= 2 * 64
+ if (rate, channels) == (24000, 1):
+ assert response.content == buf.getvalue()[44:]
diff --git a/studio/backend/tests/test_audio_voices.py b/studio/backend/tests/test_audio_voices.py
index 27e39b58454..0fafd1f451f 100644
--- a/studio/backend/tests/test_audio_voices.py
+++ b/studio/backend/tests/test_audio_voices.py
@@ -126,3 +126,26 @@ def test_deleting_a_voice_drops_its_prepared_copies():
assert prepared.is_file() and prepared.name.startswith(f"v-{voice['id']}.")
assert audio_voices.delete(voice["id"])
assert not prepared.exists()
+
+
+def test_a_script_uploads_saves_a_voice_and_fetches_it_under_v1(client):
+ headers = {"Content-Type": "application/octet-stream"}
+ upload = client.post(
+ "/v1/audio/inputs", params = {"name": "me.wav"}, content = wav_bytes(2.0), headers = headers
+ )
+ assert upload.status_code == 201, upload.text
+ source = {"input_id": upload.json()["id"]}
+ assert client.get(upload.json()["url"]).content[:4] == b"RIFF"
+ created = client.post("/v1/audio/voices", json = {"source": source, "name": "Me"})
+ assert created.status_code == 201, created.text
+ voice = created.json()
+ (listed,) = client.get("/v1/audio/voices").json()["voices"]
+ renamed = client.patch(f"/v1/audio/voices/{voice['id']}", json = {"name": "Me 2"}).json()
+ for url in (upload.json()["url"], voice["url"], listed["url"], renamed["url"]):
+ assert url.startswith("/v1/audio/"), url
+ assert client.get(voice["url"]).content[:4] == b"RIFF"
+ assert client.get(VOICES).json()["voices"][0]["url"].startswith(VOICES)
+ assert client.delete(f"/v1/audio/inputs/{source['input_id']}").json() == {"removed": True}
+ missing = client.patch("/v1/audio/voices/" + "a" * 32, json = {"name": "x"})
+ assert missing.status_code == 404
+ assert missing.json()["error"]["message"] == "Voice not found."
diff --git a/studio/backend/tests/test_keyless_api_access.py b/studio/backend/tests/test_keyless_api_access.py
index c74ec779d14..399f7aac3ec 100644
--- a/studio/backend/tests/test_keyless_api_access.py
+++ b/studio/backend/tests/test_keyless_api_access.py
@@ -117,7 +117,7 @@ INFERENCE_POST_PATHS = "/v1/chat/completions /v1/chat/count_tokens /v1/completio
def test_exact_route_matrix_matches_registered_topology():
from routes.inference import router
- denied_posts = "/v1/load /v1/unload /v1/validate /v1/generate/stream /v1/audio/speech /v1/images/generations /v1/external/openai/containers/create /v1x/chat".split()
+ denied_posts = "/v1/load /v1/unload /v1/validate /v1/generate/stream /v1/audio/speech /v1/audio/run /v1/audio/inputs /v1/audio/voices /v1/images/generations /v1/external/openai/containers/create /v1x/chat".split()
allowed = {("POST", path) for path in INFERENCE_POST_PATHS} | {
("GET", "/v1/models"), ("GET", "/v1/models/unsloth/model")}
denied = {("POST", path) for path in denied_posts} | {
diff --git a/studio/backend/tests/test_middleware.py b/studio/backend/tests/test_middleware.py
index c26d3b07d7d..e812d9b3ed6 100644
--- a/studio/backend/tests/test_middleware.py
+++ b/studio/backend/tests/test_middleware.py
@@ -161,6 +161,13 @@ class TestMaxBodyMiddleware:
assert main_module._get_upload_passthrough_request_max_bytes(path + "/") == (
upload_request_limit_bytes(STT_AUDIO_RAW_MAX_BYTES)
), path
+ from utils.upload_limits import AUDIO_INPUT_MAX_BYTES
+
+ for path in ("/v1/audio/inputs", "/api/inference/audio/inputs"):
+ assert path in main_module._BODY_UPLOAD_PASSTHROUGH_EXACT_PATHS, path
+ assert main_module._get_upload_passthrough_request_max_bytes(path) == (
+ AUDIO_INPUT_MAX_BYTES
+ ), path
from utils.upload_limits import (
VIDEO_INPUT_REFERENCE_JSON_MAX_BYTES,
VIDEO_INPUT_REFERENCE_MAX_BYTES,
diff --git a/studio/backend/tests/test_openai_audio_speech_route.py b/studio/backend/tests/test_openai_audio_speech_route.py
index 8da84c27a5c..310f956f829 100644
--- a/studio/backend/tests/test_openai_audio_speech_route.py
+++ b/studio/backend/tests/test_openai_audio_speech_route.py
@@ -107,14 +107,27 @@ def test_voice_and_speed_accepted_and_ignored(monkeypatch):
assert resp.status_code == 200
-def test_non_wav_response_format_is_400(monkeypatch):
+def test_unknown_response_format_is_400(monkeypatch):
cli, calls, saved = _make_client(monkeypatch)
- resp = cli.post("/v1/audio/speech", json = {"input": "hi", "response_format": "mp3"})
+ resp = cli.post("/v1/audio/speech", json = {"input": "hi", "response_format": "wma"})
assert resp.status_code == 400
- assert "mp3" in resp.json()["error"]["message"]
+ error = resp.json()["error"]
+ assert error["param"] == "response_format"
+ assert "'wma'" in error["message"] and "mp3" in error["message"]
assert calls == [] # rejected before any generation
+def test_sse_stream_format_is_400(monkeypatch):
+ cli, calls, saved = _make_client(monkeypatch)
+ resp = cli.post("/v1/audio/speech", json = {"input": "hi", "stream_format": "sse"})
+ assert (resp.status_code, resp.json()["error"]["param"]) == (400, "stream_format")
+ assert calls == []
+ assert (
+ cli.post("/v1/audio/speech", json = {"input": "hi", "stream_format": "audio"}).status_code
+ == 200
+ )
+
+
def test_null_response_format_means_wav(monkeypatch):
cli, calls, saved = _make_client(monkeypatch)
resp = cli.post("/v1/audio/speech", json = {"input": "hi", "response_format": None})
@@ -436,6 +449,23 @@ def test_external_rejects_non_wav_response_format(monkeypatch):
assert speech_calls == []
+def test_external_rejects_sse_before_the_upstream_call(monkeypatch):
+ cli, _calls, _saved = _make_client(monkeypatch)
+ created, speech_calls = _install_external(monkeypatch)
+ resp = cli.post(
+ "/v1/audio/speech",
+ json = {
+ "input": "hi",
+ "provider_id": "conn-1",
+ "model": "kokoro",
+ "voice": "alloy",
+ "stream_format": "sse",
+ },
+ )
+ assert (resp.status_code, resp.json()["error"]["param"]) == (400, "stream_format")
+ assert created == [] and speech_calls == []
+
+
def test_external_missing_model_is_400(monkeypatch):
cli, calls, saved = _make_client(monkeypatch)
_install_external(monkeypatch)
@@ -931,6 +961,22 @@ def test_speech_opens_a_monitor_row(monkeypatch):
assert rows[0]["model"] == "unsloth/orpheus-3b-0.1-ft"
+def test_v1_audio_generate_opens_a_monitor_row_and_the_chat_mount_does_not(monkeypatch):
+ cli, calls, saved = _make_client(monkeypatch)
+ cli.app.include_router(router, prefix = "/api/inference")
+ api_monitor.clear()
+ body = {"messages": [{"role": "user", "content": "read me"}]}
+ assert cli.post("/api/inference/audio/generate", json = body).status_code == 200
+ assert api_monitor.snapshot(include_details = False) == []
+ assert cli.post("/v1/audio/generate", json = body).status_code == 200
+ (row,) = api_monitor.snapshot(include_details = False)
+ assert (row["endpoint"], row["status"], row["model"]) == (
+ "/v1/audio/generate",
+ "completed",
+ "unsloth/orpheus-3b-0.1-ft",
+ )
+
+
def test_tts_failure_records_an_error_row(monkeypatch):
cli, calls, saved = _make_client(monkeypatch, generate = _boom)
api_monitor.clear()
@@ -945,7 +991,7 @@ def test_rejected_response_format_records_nothing(monkeypatch):
# Refused before any work, so it is not traffic the monitor should show.
cli, calls, saved = _make_client(monkeypatch)
api_monitor.clear()
- resp = cli.post("/v1/audio/speech", json = {"input": "hi", "response_format": "mp3"})
+ resp = cli.post("/v1/audio/speech", json = {"input": "hi", "response_format": "wma"})
assert resp.status_code == 400
assert api_monitor.snapshot(include_details = False) == []
diff --git a/studio/backend/tests/test_openai_audio_transcriptions_route.py b/studio/backend/tests/test_openai_audio_transcriptions_route.py
index 78ee761ed3f..31f7f883672 100644
--- a/studio/backend/tests/test_openai_audio_transcriptions_route.py
+++ b/studio/backend/tests/test_openai_audio_transcriptions_route.py
@@ -32,6 +32,7 @@ def _make_client(monkeypatch, transcribe = None):
engine = None,
request = None,
device = None,
+ **kwargs,
):
calls.append(
{
@@ -41,6 +42,7 @@ def _make_client(monkeypatch, transcribe = None):
"fast": fast,
"engine": engine,
"request": request,
+ **kwargs,
}
)
if transcribe is not None:
@@ -911,3 +913,146 @@ def test_the_proxied_row_never_carries_a_local_path(monkeypatch):
assert row["model"] == "whisper-v3"
# Only the label is redacted; the provider is still asked for what the client sent.
assert transcription_calls[-1]["model"] == "/home/ana/models/whisper-v3"
+
+
+_TIMED = {
+ "text": "Hi there. Bye.",
+ "language": "English",
+ "duration": 2.5,
+ "model": "audio-cpp/audio.cpp-gguf/VibeVoice-ASR-GGUF",
+ "segments": [
+ {"start": 0.0, "end": 1.0, "text": "Hi there.", "speaker": "0"},
+ {"start": 1.2, "end": 2.5, "text": "Bye.", "speaker": "1"},
+ ],
+ "words": [
+ {"start": 0.0, "end": 0.4, "word": "Hi"},
+ {"start": 0.4, "end": 1.0, "word": "there."},
+ ],
+ "speakers": [{"id": "0", "label": "Speaker 1"}, {"id": "1", "label": "Speaker 2"}],
+}
+
+
+def _audio_cpp(
+ monkeypatch,
+ tmp_path,
+ result = _TIMED,
+ **caps,
+):
+ """An audio.cpp ASR model; the upload's prepared copy is a file the route must remove."""
+ from core.inference import stt_capabilities
+
+ async def _result(raw):
+ return dict(result)
+
+ cli, calls = _make_client(monkeypatch, transcribe = _result)
+ monkeypatch.setattr(routes_module, "_stt_engine_for_model", lambda model: "audiocpp")
+ monkeypatch.setattr(routes_module, "_resolve_serving_stt_engine", lambda engine: engine)
+ caps = {"family": "vibevoice_asr", "timestamps": "always", "speakers": True, **caps}
+ monkeypatch.setattr(stt_capabilities, "capabilities_for", lambda model, engine: caps)
+ prepared = []
+
+ def _prepare(raw, rate):
+ path = tmp_path / f"upload.{rate}.wav"
+ path.write_bytes(raw)
+ prepared.append(path)
+ return path
+
+ monkeypatch.setattr(routes_module, "_prepared_upload", _prepare)
+ return cli, calls, prepared
+
+
+def test_audio_cpp_verbose_json_has_openai_segments_and_words(monkeypatch, tmp_path):
+ openai_types = pytest.importorskip("openai.types.audio.transcription_verbose")
+ cli, calls, prepared = _audio_cpp(monkeypatch, tmp_path)
+ data = {"response_format": "verbose_json", "timestamp_granularities[]": ["segment", "word"]}
+ resp = _post(cli, data = data)
+ assert resp.status_code == 200, resp.text
+ body = resp.json()
+ openai_types.TranscriptionVerbose.model_validate(body)
+ assert body["language"] == "English"
+ assert [(s["id"], s["start"], s["end"], s["text"]) for s in body["segments"]] == [
+ (0, 0.0, 1.0, "Hi there."),
+ (1, 1.2, 2.5, "Bye."),
+ ]
+ assert body["words"] == [
+ {"word": "Hi", "start": 0.0, "end": 0.4},
+ {"word": "there.", "start": 0.4, "end": 1.0},
+ ]
+ # VibeVoice times its spans at 24 kHz, so the upload is prepared at that rate and removed.
+ (path,) = prepared
+ assert path.name == "upload.24000.wav" and not path.exists()
+ assert calls[0]["source_path"] == path and calls[0]["timestamps"] is True
+
+
+def test_audio_cpp_diarized_json_names_speakers(monkeypatch, tmp_path):
+ cli, calls, prepared = _audio_cpp(monkeypatch, tmp_path)
+ resp = _post(cli, data = {"response_format": "diarized_json"})
+ assert resp.status_code == 200, resp.text
+ body = resp.json()
+ assert (body["task"], body["duration"], body["text"]) == ("transcribe", 2.5, "Hi there. Bye.")
+ assert body["segments"] == [
+ {
+ "type": "transcript.text.segment",
+ "id": "seg_0",
+ "start": 0.0,
+ "end": 1.0,
+ "text": "Hi there.",
+ "speaker": "Speaker 1",
+ },
+ {
+ "type": "transcript.text.segment",
+ "id": "seg_1",
+ "start": 1.2,
+ "end": 2.5,
+ "text": "Bye.",
+ "speaker": "Speaker 2",
+ },
+ ]
+ assert calls[0]["timestamps"] is False
+
+
+ "caps, data, detail",
+ [
+ ({"speakers": False}, {"response_format": "diarized_json"}, "cannot tell speakers apart"),
+ (
+ {"timestamps": "unsupported"},
+ {"response_format": "verbose_json", "timestamp_granularities[]": "word"},
+ "cannot add timestamps",
+ ),
+ ],
+)
+def test_audio_cpp_refuses_what_the_model_cannot_add(monkeypatch, tmp_path, caps, data, detail):
+ cli, calls, prepared = _audio_cpp(monkeypatch, tmp_path, **caps)
+ resp = _post(cli, data = data)
+ assert resp.status_code == 422
+ assert detail in resp.json()["error"]["message"]
+ assert calls == [] and prepared == []
+
+
[email protected]("fmt", ["json", "text", "diarized_json"])
+def test_audio_cpp_granularities_need_verbose_json(monkeypatch, tmp_path, fmt):
+ cli, calls, prepared = _audio_cpp(monkeypatch, tmp_path)
+ resp = _post(cli, data = {"response_format": fmt, "timestamp_granularities[]": "word"})
+ assert (resp.status_code, resp.json()["error"]["param"]) == (400, "timestamp_granularities")
+ assert calls == [] and prepared == []
+
+
+def test_diarized_json_on_whisper_is_refused_before_any_work(monkeypatch):
+ cli, calls = _make_client(monkeypatch)
+ resp = _post(cli, data = {"response_format": "diarized_json"})
+ assert resp.status_code == 422
+ assert calls == []
+
+
[email protected]("reported, status", [("English", 200), (None, 501)])
+def test_audio_cpp_verbose_json_needs_no_language_when_the_model_reports_one(
+ monkeypatch, tmp_path, reported, status
+):
+ cli, calls, prepared = _audio_cpp(
+ monkeypatch, tmp_path, result = {**_TIMED, "language": reported}
+ )
+ resp = _post(cli, data = {"response_format": "verbose_json"})
+ assert resp.status_code == status, resp.text
+ if status == 200:
+ assert resp.json()["language"] == "English"
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 019666615af..0a16be9fde1 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -11821,6 +11821,52 @@ def test_speech_switch_admission(monkeypatch, audio_type, context, text, instruc
assert len(recorder.calls) == 1
+ "workflow, speech_type, workflows, admitted",
+ [
+ ("separate", None, ["separate"], True),
+ ("separate", None, [], False),
+ ("clone", "audiocpp_tts", ["speak"], False),
+ ("clone", "audiocpp_tts", ["speak", "clone"], True),
+ ],
+)
+def test_audio_workflow_switch_admission(monkeypatch, workflow, speech_type, workflows, admitted):
+ backend, recorder = _speech_case(monkeypatch, speech_type, gguf = False)
+ seen = []
+ monkeypatch.setattr(
+ inference_route, "_target_audio_workflows", lambda *a: seen.append(a) or workflows
+ )
+ call = inference_route._maybe_auto_switch_model(
+ "org/B-GGUF",
+ object(),
+ "tester",
+ require_speech = speech_type is not None,
+ require_audio_workflow = workflow,
+ )
+ if admitted:
+ asyncio.run(call)
+ assert len(recorder.calls) == 1
+ else:
+ with pytest.raises(HTTPException) as error:
+ asyncio.run(call)
+ assert error.value.status_code == 400
+ assert error.value.detail["error"]["param"] == "model"
+ # Refused before the resident model is evicted.
+ assert recorder.calls == [] and backend.model_identifier == "org/A-GGUF"
+ assert seen == [("/local/B", speech_type)]
+
+
+def test_target_audio_workflows_reads_audio_cpp_targets_from_the_cache(monkeypatch):
+ from core.inference import audio_cpp_models
+
+ sep = SimpleNamespace(workflows = {"separate": None})
+ monkeypatch.setattr(audio_cpp_models, "resolve", lambda target, network: sep)
+ workflows = inference_route._target_audio_workflows
+ assert workflows("audio-cpp/audio.cpp-gguf/HTDemucs-6stems-GGUF", None) == ["separate"]
+ assert workflows("/local/csm", "csm") == ["speak"]
+ assert workflows("/local/chat", None) == []
+
+
@pytest.mark.parametrize("audio_type", ["snac", "bicodec", "dac", "higgs_tts2"])
@pytest.mark.parametrize("token", [None, "caller-token"])
def test_speech_switch_preserves_staged_assets_and_identity(monkeypatch, audio_type, token):
diff --git a/tests/test_runtime_text_encoding.py b/tests/test_runtime_text_encoding.py
index 5bd40b5dcef..32c764b846c 100644
--- a/tests/test_runtime_text_encoding.py
+++ b/tests/test_runtime_text_encoding.py
@@ -67,7 +67,7 @@ REVIEWED_VENDORED_OFFENDERS = {
# keyed on path + expression, not line, so unrelated edits above it do not break the scan.
REVIEWED_NON_FILE_OPEN = (
"studio/backend/core/inference/audio_inputs.py",
- "stream.codec_context.open()",
+ "stream.codec_context.open(strict=True)",
)
GUARDED_METHODS = {"read_text", "write_text"}
# path classes, so an unbound `Path.open(p)` shifts every argument one right.
@@ -431,12 +431,12 @@ def test_skips_foreign_openers_and_readers():
def test_skips_only_the_reviewed_pyav_codec_open():
- pyav = "\n" * 263 + "stream.codec_context.open()\n"
+ pyav = "\n" * 263 + "stream.codec_context.open(strict = True)\n"
path = "studio/backend/core/inference/audio_inputs.py"
assert not _offenders_in(pyav, path)
assert not _offenders_in("\n" + pyav, path)
assert _offenders_in(pyav, "studio/backend/core/inference/other.py")
- assert _offenders_in("\n" * 263 + "config.codec_context.open()\n", path)
+ assert _offenders_in("\n" * 263 + "config.codec_context.open(strict = True)\n", path)
def test_test_trees_are_out_of_scope():