Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let Orpheus TTS pick one of its built-in voices (#13853)

Daniel Han848d72302ff8
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index d49bb1b783a..4b5611632a8 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -580,6 +580,40 @@ def build_dac_tts_prompt(text: str) -> str:
return f"<|im_start|>\n<|text_start|>{text}<|text_end|>\n<|audio_start|>\n"
+# canopylabs/orpheus-3b-0.1-ft speakers, in the README's order. The prompt reads "{voice}: {text}".
+ORPHEUS_VOICES = ("tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe")
+
+
+def tts_voice_options(audio_type) -> Optional[list]:
+ """The voice picker a codec TTS model offers, in the audio.cpp ``audio_options`` shape, or None.
+ No default, so an unset voice keeps the unprefixed prompt fine-tunes were trained on."""
+ if audio_type != "snac":
+ return None
+ return [
+ {
+ "name": "voice",
+ "type": "enum",
+ "values": list(ORPHEUS_VOICES),
+ "default": None,
+ "description": "Orpheus speaker. Leave unset for your own fine-tunes.",
+ }
+ ]
+
+
+def tts_prompt_with_voice(text: str, audio_type, audio_options: Optional[dict]) -> str:
+ """*text* with the chosen speaker prefix, when the codec takes one and a voice was picked."""
+ voice = (audio_options or {}).get("voice")
+ if audio_type != "snac" or not voice:
+ return text
+ if voice not in ORPHEUS_VOICES:
+ from core.inference.audio_errors import AudioRuntimeError
+ raise AudioRuntimeError(
+ f"Unknown Orpheus voice. Choose one of: {', '.join(ORPHEUS_VOICES)}.",
+ status = 400,
+ )
+ return f"{voice}: {text}"
+
+
def _neutralize_leaves(
value,
rewrite,
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index a1497e8e69d..c27a1e28590 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -51,6 +51,8 @@ from core.inference.chat_template_helpers import (
neutralize_tts_prompt_text,
prompt_opens_reasoning_channel,
trailing_assistant_text,
+ tts_prompt_with_voice,
+ tts_voice_options,
)
from core.inference.presence_penalty import _make_presence_penalty_processor
from core.inference.generation_timing import (
@@ -1097,6 +1099,9 @@ class InferenceBackend:
FastLanguageModel.for_inference(model)
self.models[model_name]["model"] = model
self.models[model_name]["tokenizer"] = tokenizer
+ voice_options = tts_voice_options(audio_type)
+ if voice_options:
+ self.models[model_name]["audio_options"] = voice_options
# Load external codec for TTS audio types
# (Whisper is ASR, audio_vlm is audio input — neither needs one)
@@ -3027,6 +3032,7 @@ class InferenceBackend:
instructions: Optional[str] = None,
language: Optional[str] = None,
seed: Optional[int] = None,
+ audio_options: Optional[dict] = None,
) -> Tuple[bytes, int]:
"""Generate audio from text for TTS models. Returns (wav_bytes, sample_rate). Blocking."""
# Reserved for native audio architectures; codec-backed TTS models do
@@ -3046,7 +3052,9 @@ class InferenceBackend:
top_k = self._normalize_top_k(top_k)
# Every codec below concatenates its prompt instead of templating it, so this
# is the one choke point for all four (#7066).
- text = neutralize_tts_prompt_text(text, audio_type)
+ text = tts_prompt_with_voice(
+ neutralize_tts_prompt_text(text, audio_type), audio_type, audio_options
+ )
if cancel_event is not None and cancel_event.is_set():
raise RuntimeError("Audio generation cancelled")
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 519ffc43278..edc2aeca49a 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -43537,6 +43537,7 @@ class LlamaCppBackend:
repetition_penalty: float = 1.1,
cancel_event: Optional[threading.Event] = None,
stats_holder: Optional[dict] = None,
+ audio_options: Optional[dict] = None,
) -> tuple:
"""
Generate TTS audio via llama-server /completion + codec decode.
@@ -43551,10 +43552,16 @@ class LlamaCppBackend:
tpl, stop, need_ids = self._TTS_PROMPTS[audio_type]
# Raw prompt, not messages: codec delimiters are the only structure to break (#7066).
- from core.inference.chat_template_helpers import neutralize_tts_prompt_text
+ from core.inference.chat_template_helpers import (
+ neutralize_tts_prompt_text,
+ tts_prompt_with_voice,
+ )
+ prompt_text = tts_prompt_with_voice(
+ neutralize_tts_prompt_text(text, audio_type), audio_type, audio_options
+ )
payload: dict = {
- "prompt": tpl.format(text = neutralize_tts_prompt_text(text, audio_type)),
+ "prompt": tpl.format(text = prompt_text),
"stream": False,
"n_predict": max_new_tokens,
"temperature": temperature,
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 89da9328f47..0efc5b0c8d9 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -1730,7 +1730,7 @@ def _handle_generate_audio(backend, cmd: dict, resp_queue: Any, cancel_event) ->
)
logger.info("Finished audio separation for request_id=%s", request_id)
return
- # Only audio.cpp models take per-model options; other backends never see the keyword.
+ # Only audio.cpp and Orpheus (voice) models take per-model options; others never see the keyword.
extra = {"audio_options": cmd["audio_options"]} if cmd.get("audio_options") else {}
# Run fields reach only a backend that takes them: a Speak run on a native TTS model carries a
# workflow its backend has no keyword for.
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index ba45e82f849..e5ee4443579 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -8595,6 +8595,8 @@ def _unsloth_serving_fields(model_info: dict) -> dict:
def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
"""Runtime state shared by load, dedupe, and status; duplicates echo active settings."""
+ from core.inference.chat_template_helpers import tts_voice_options
+
fields = {
name: getattr(llama_backend, name, getattr(llama_backend, f"_{name}", None))
for name in _InferenceRuntimeFields.model_fields
@@ -8624,9 +8626,13 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
context_length_enforced = True,
context_length_fitted = None,
context_unbounded_when_batched = False,
- # llama-server never serves an audio GGUF; those load through the audio worker.
+ # audio.cpp GGUFs load through the audio worker; llama-server serves only codec TTS.
audio_family = None,
- audio_options = None,
+ audio_options = (
+ tts_voice_options(getattr(llama_backend, "_audio_type", None))
+ if getattr(llama_backend, "_is_audio", False)
+ else None
+ ),
audio_workflows = None,
audio_reference_text = None,
audio_required_inputs = None,
@@ -22992,6 +22998,14 @@ def _audio_convert_problem(model_info: dict, label: str, run_inputs: dict) -> Op
return None
+def _apply_builtin_voice(payload, voice: Optional[str], options) -> None:
+ """Send a requested built-in speaker as the ``voice`` option when the loaded model lists it
+ and the request set none."""
+ speakers = next((o.get("values") for o in options or () if o.get("name") == "voice"), None)
+ if voice in (speakers or ()) and "voice" not in (payload.audio_options or {}):
+ payload.audio_options = {**(payload.audio_options or {}), "voice": voice}
+
+
async def _generate_tts_wav(
text: str,
payload: ChatCompletionRequest,
@@ -23058,6 +23072,9 @@ async def _generate_tts_wav(
audio_type = getattr(llama_backend, "_audio_type", None)
supported_audio_types = _GGUF_TTS_AUDIO_TYPES
_audio_model_id = getattr(llama_backend, "model_identifier", None) or model_name
+ from core.inference.chat_template_helpers import tts_voice_options
+
+ _apply_builtin_voice(payload, voice, tts_voice_options(audio_type))
gen = lambda: llama_backend.generate_audio_response(
text = text,
audio_type = llama_backend._audio_type,
@@ -23074,6 +23091,7 @@ async def _generate_tts_wav(
repetition_penalty = payload.repetition_penalty,
cancel_event = _audio_cancel,
stats_holder = stats_holder,
+ audio_options = payload.audio_options,
)
else:
backend = await asyncio.to_thread(get_inference_backend)
@@ -23086,16 +23104,7 @@ async def _generate_tts_wav(
audio_type = model_info.get("audio_type")
audio_family = model_info.get("audio_family")
supported_audio_types = _TRANSFORMERS_TTS_AUDIO_TYPES
- speakers = next(
- (
- o.get("values")
- for o in model_info.get("audio_options") or ()
- if o.get("name") == "voice"
- ),
- None,
- )
- if voice in (speakers or ()) and "voice" not in (payload.audio_options or {}):
- payload.audio_options = {**(payload.audio_options or {}), "voice": voice}
+ _apply_builtin_voice(payload, voice, model_info.get("audio_options"))
_audio_model_id = getattr(backend, "active_model_name", None) or model_name
gen = lambda: backend.generate_audio_response(
text = text,
diff --git a/studio/backend/tests/test_orpheus_voice.py b/studio/backend/tests/test_orpheus_voice.py
new file mode 100644
index 00000000000..fd9f9162642
--- /dev/null
+++ b/studio/backend/tests/test_orpheus_voice.py
@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Orpheus built-in voices (#5829): the "{voice}: {text}" prefix the -ft checkpoints were trained on."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from core.inference.audio_errors import AudioRuntimeError
+from core.inference.chat_template_helpers import (
+ ORPHEUS_VOICES,
+ tts_prompt_with_voice,
+ tts_voice_options,
+)
+
+
+def test_only_orpheus_offers_voices():
+ (option,) = tts_voice_options("snac")
+ assert option["name"] == "voice" and option["type"] == "enum"
+ assert option["values"] == list(ORPHEUS_VOICES)
+ # No default: an unset voice must keep today's unprefixed prompt.
+ assert option["default"] is None
+ for audio_type in ("csm", "bicodec", "dac", "whisper", None):
+ assert tts_voice_options(audio_type) is None
+
+
[email protected]("options", [None, {}, {"voice": None}, {"voice": ""}])
+def test_no_voice_leaves_the_prompt_unchanged(options):
+ assert tts_prompt_with_voice("Hello there.", "snac", options) == "Hello there."
+
+
+def test_a_voice_prefixes_the_prompt():
+ assert (
+ tts_prompt_with_voice("Hello <laugh>.", "snac", {"voice": "leo"}) == "leo: Hello <laugh>."
+ )
+
+
+def test_other_codecs_ignore_a_voice():
+ assert tts_prompt_with_voice("Hello.", "bicodec", {"voice": "tara"}) == "Hello."
+
+
+def test_an_unknown_voice_is_a_client_error():
+ with pytest.raises(AudioRuntimeError) as err:
+ tts_prompt_with_voice("Hello.", "snac", {"voice": "alloy"})
+ assert err.value.status == 400
+
+
+def test_transformers_orpheus_prompt_carries_the_voice(monkeypatch):
+ pytest.importorskip("peft")
+ from core.inference.inference import InferenceBackend
+
+ backend = InferenceBackend.__new__(InferenceBackend)
+ import threading
+
+ backend._generation_lock = threading.Lock()
+ backend.active_model_name = "unsloth/orpheus-3b-0.1-ft"
+ backend.models = {
+ backend.active_model_name: {"audio_type": "snac", "model": object(), "tokenizer": None}
+ }
+ seen = []
+
+ def fake_snac(_model, _tokenizer, text, *_args, **_kwargs):
+ seen.append(text)
+ return b"wav", 24000
+
+ monkeypatch.setattr(backend, "_generate_snac", fake_snac, raising = False)
+ monkeypatch.setattr(backend, "_cancel_stopping_criteria", lambda _event: None, raising = False)
+ backend.generate_audio_response("Hi.", audio_options = {"voice": "tara"})
+ backend.generate_audio_response("Hi.")
+ assert seen == ["tara: Hi.", "Hi."]
+
+
+class _Captured(Exception):
+ pass
+
+
+def test_gguf_orpheus_prompt_carries_the_voice(monkeypatch):
+ from core.inference import llama_cpp
+
+ payloads = []
+
+ class FakeClient:
+ def __init__(self, *_args, **_kwargs):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_exc):
+ return False
+
+ def post(self, _url, json):
+ payloads.append(json)
+ raise _Captured
+
+ monkeypatch.setattr(llama_cpp.httpx, "Client", FakeClient)
+ backend = llama_cpp.LlamaCppBackend.__new__(llama_cpp.LlamaCppBackend)
+ monkeypatch.setattr(llama_cpp.LlamaCppBackend, "base_url", "http://127.0.0.1:1", raising = False)
+ monkeypatch.setattr(llama_cpp.LlamaCppBackend, "_auth_headers", {}, raising = False)
+ for options in ({"voice": "zoe"}, None):
+ with pytest.raises(_Captured):
+ backend.generate_audio_response("Hi.", "snac", audio_options = options)
+ assert [p["prompt"] for p in payloads] == [
+ "<custom_token_3>zoe: Hi.<|eot_id|><custom_token_4>",
+ "<custom_token_3>Hi.<|eot_id|><custom_token_4>",
+ ]
+
+
+def test_gguf_status_lists_voices_only_for_orpheus():
+ from core.inference.llama_cpp import LlamaCppBackend
+ from routes.inference import _llama_runtime_fields
+
+ def status(**attrs):
+ backend = LlamaCppBackend()
+ for name, value in attrs.items():
+ setattr(backend, name, value)
+ return _llama_runtime_fields(backend)["audio_options"]
+
+ assert status(_is_audio = True, _audio_type = "snac") == tts_voice_options("snac")
+ assert status(_is_audio = True, _audio_type = "bicodec") is None
+ assert status(_is_audio = False, _audio_type = None) is None
+
+
+def test_speech_api_voice_maps_to_a_listed_speaker():
+ from routes.inference import _apply_builtin_voice
+ options = tts_voice_options("snac")
+ for voice, sent, expected in (
+ ("tara", None, {"voice": "tara"}),
+ ("alloy", None, None),
+ ("tara", {"voice": "leo"}, {"voice": "leo"}),
+ ):
+ payload = SimpleNamespace(audio_options = sent)
+ _apply_builtin_voice(payload, voice, options)
+ assert payload.audio_options == expected