Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: say so when the Audio page stops speech at Max tokens (#12583)
Nilaydccfb699358b
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex f88709fe1ec..34007175d17 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -2549,6 +2549,7 @@ class InferenceBackend:if use_adapter is not None:self._apply_adapter_state(use_adapter)stopping_criteria = self._cancel_stopping_criteria(cancel_event)+ self.last_generation_stats = Noneif audio_type == "snac":result = self._generate_snac(@@ -2638,6 +2639,12 @@ class InferenceBackend:)if cancel_event is not None and cancel_event.is_set():raise RuntimeError("Audio generation cancelled")+ self._record_generation_stats(+ prompt_tokens = input_ids.shape[1],+ completion_tokens = self._generated_token_count(model, generated, input_ids.shape[1]),+ max_new_tokens = max_new_tokens,+ ended_on_stop_token = self._ended_on_stop_token(generated, 128258),+ )return self._audio_codec_manager.decode_snac(generated, str(device))def _generate_csm(@@ -2692,7 +2699,14 @@ class InferenceBackend:)if cancel_event is not None and cancel_event.is_set():raise RuntimeError("Audio generation cancelled")- new_tokens = generated[:, inputs.input_ids.shape[1] :]+ prompt_len = inputs.input_ids.shape[1]+ self._record_generation_stats(+ prompt_tokens = prompt_len,+ completion_tokens = self._generated_token_count(model, generated, prompt_len),+ max_new_tokens = max_new_tokens,+ ended_on_stop_token = self._ended_on_stop_token(generated, tokenizer.eos_token_id),+ )+ new_tokens = generated[:, prompt_len:]decoded_text = tokenizer.batch_decode(new_tokens, skip_special_tokens = False)[0]return self._audio_codec_manager.decode_bicodec(decoded_text, str(model.device))@@ -2758,6 +2772,16 @@ class InferenceBackend:)if cancel_event is not None and cancel_event.is_set():raise RuntimeError("Audio generation cancelled")+ self._record_generation_stats(+ prompt_tokens = inputs.input_ids.shape[1],+ completion_tokens = self._generated_token_count(+ model, generated, inputs.input_ids.shape[1]+ ),+ max_new_tokens = max_new_tokens,+ ended_on_stop_token = self._ended_on_stop_token(+ generated, self._generation_stop_token_ids(model, {})+ ),+ )decoded_text = tokenizer.batch_decode(generated, skip_special_tokens = False)[0]return self._audio_codec_manager.decode_dac(decoded_text, str(model.device))diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex f7d0183458e..1f1f6051996 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -40850,6 +40850,7 @@ class LlamaCppBackend:max_new_tokens: int = 2048,repetition_penalty: float = 1.1,cancel_event: Optional[threading.Event] = None,+ stats_holder: Optional[dict] = None,) -> tuple:"""Generate TTS audio via llama-server /completion + codec decode.@@ -40934,6 +40935,8 @@ class LlamaCppBackend:raise RuntimeError("Audio generation cancelled")data = resp.json()+ if stats_holder is not None:+ stats_holder["stats"] = {"truncated": data.get("stop_type") == "limit"}token_ids = ([p["id"] for p in data.get("completion_probabilities", []) if "id" in p]if need_idsdiff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 4d682b054b6..4585e8b14be 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -3339,6 +3339,7 @@ class InferenceOrchestrator:language: Optional[str] = None,seed: Optional[int] = None,audio_options: Optional[dict] = None,+ stats_holder: Optional[dict] = None,) -> Tuple[bytes, int]:"""Generate TTS audio. Returns (wav_bytes, sample_rate). Blocking: sends the command andwaits for the full audio response."""@@ -3471,6 +3472,8 @@ class InferenceOrchestrator:raise AudioGenerationCancelledError("Audio generation cancelled")wav_bytes = base64.b64decode(resp["wav_base64"])sample_rate = resp["sample_rate"]+ if stats_holder is not None:+ stats_holder["stats"] = resp.get("stats")return wav_bytes, sample_rateif rtype == "audio_error":diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex b78e265586b..d286f57d273 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -1617,6 +1617,7 @@ def _handle_generate_audio(backend, cmd: dict, resp_queue: Any, cancel_event) ->"request_id": request_id,"wav_base64": base64.b64encode(wav_bytes).decode("ascii"),"sample_rate": sample_rate,+ "stats": getattr(backend, "last_generation_stats", None),},)logger.info("Finished audio generation for request_id=%s", request_id)diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 5dd1cc7992b..7b0deb444d1 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -21288,6 +21288,7 @@ async def _generate_tts_wav(*,speech_api_default_max_tokens: bool = False,requested_model: str = _RELOAD_ONLY_MODEL,+ stats_holder: Optional[dict] = None,) -> tuple[bytes, int, str, Optional[str]]:"""Shared core of /audio/generate and /audio/speech. Returns(wav_bytes, sample_rate, model_name, audio_type)."""@@ -21349,6 +21350,7 @@ async def _generate_tts_wav(),repetition_penalty = payload.repetition_penalty,cancel_event = _audio_cancel,+ stats_holder = stats_holder,)else:backend = await asyncio.to_thread(get_inference_backend)@@ -21381,6 +21383,7 @@ async def _generate_tts_wav(language = payload.audio_language,seed = payload.seed,**({"audio_options": payload.audio_options} if payload.audio_options else {}),+ stats_holder = stats_holder,)if audio_type not in supported_audio_types:@@ -21558,6 +21561,7 @@ async def generate_audio(raise HTTPException(status_code = 400, detail = "No user message found.")text = last_user_msg["content"]+ tts_stats: dict = {}wav_bytes, sample_rate, model_name, audio_type = await _generate_tts_wav(text,payload,@@ -21567,7 +21571,9 @@ async def generate_audio(# request model, and without this it stops the hook at its falsey check before# the idle-stash restore, failing a request the sibling route serves.requested_model = _switch_model_for_payload(payload) or _RELOAD_ONLY_MODEL,+ stats_holder = tts_stats,)+ truncated = bool((tts_stats.get("stats") or {}).get("truncated"))persisted_clip = await asyncio.to_thread(_persist_tts_clip, wav_bytes, sample_rate, text, model_name, audio_type)@@ -21587,7 +21593,7 @@ async def generate_audio("role": "assistant","content": text,},- "finish_reason": "stop",+ "finish_reason": "length" if truncated else "stop",}],}diff --git a/studio/backend/tests/test_audio_tts_cancellation.py b/studio/backend/tests/test_audio_tts_cancellation.pyindex 177b6715133..1a1e307bd52 100644--- a/studio/backend/tests/test_audio_tts_cancellation.py+++ b/studio/backend/tests/test_audio_tts_cancellation.py@@ -4,6 +4,7 @@import asyncioimport base64import importlib+import jsonimport queueimport sysimport threading@@ -765,3 +766,170 @@ def test_backend_tts_generation_uses_cancel_stopping_criteria(monkeypatch):assert backend.generate_audio_response("hello", cancel_event = cancel) == (b"RIFFfake", 24000)assert captured["stopping_criteria"] is criteria++[email protected](("stop_type", "finish_reason"), (("limit", "length"), ("eos", "stop")))+def test_gguf_speech_cut_at_max_tokens_is_reported(monkeypatch, stop_type, finish_reason):+ import core.inference.llama_cpp as llama_cpp++ sent = {}++ class _Resp:+ status_code = 200++ def json(self):+ return {"content": "<custom_token_10>", "stop_type": stop_type}++ class _Client:+ def __init__(self, *_args, **_kwargs):+ pass++ def __enter__(self):+ return self++ def __exit__(self, *_exc):+ return False++ def post(+ self,+ url,+ json = None,+ **_kwargs,+ ):+ sent.update(json)+ return _Resp()++ class _Llama(llama_cpp.LlamaCppBackend):+ is_loaded = True+ _is_audio = True+ _audio_type = "snac"+ model_identifier = "unsloth/orpheus-3b-0.1-ft-GGUF"+ base_url = "http://127.0.0.1:8080"+ _auth_headers: dict = {}++ async def _noop_switch(*_args, **_kwargs):+ return None++ monkeypatch.setattr(llama_cpp.httpx, "Client", _Client)+ monkeypatch.setattr(+ llama_cpp.LlamaCppBackend,+ "_codec_mgr",+ types.SimpleNamespace(decode = lambda *_args, **_kwargs: (b"RIFFfake", 24000)),+ )+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: _Llama.__new__(_Llama))+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _noop_switch)+ monkeypatch.setattr(inference_route, "_persist_tts_clip", lambda *_args: None)+ payload = ChatCompletionRequest(+ messages = [{"role": "user", "content": "A long paragraph."}],+ max_tokens = 2048,+ )++ response = asyncio.run(+ inference_route.generate_audio(payload, request = None, current_subject = "t")+ )++ assert sent["n_predict"] == 2048+ assert json.loads(response.body)["choices"][0]["finish_reason"] == finish_reason++[email protected](("last_token", "truncated"), ((4242, True), (128258, False)))+def test_snac_speech_cut_at_max_tokens_is_reported(monkeypatch, last_token, truncated):+ pytest.importorskip("peft")+ import torch+ from core.inference.inference import InferenceBackend++ class _Tokenizer:+ def __call__(self, text, return_tensors):+ return types.SimpleNamespace(input_ids = torch.tensor([[1, 2, 3]]))++ class _Model:+ device = torch.device("cpu")++ def generate(self, input_ids, max_new_tokens, **_kwargs):+ codes = [4242] * (max_new_tokens - 1) + [last_token]+ return torch.cat([input_ids, torch.tensor([codes])], dim = 1)++ backend = InferenceBackend.__new__(InferenceBackend)+ backend.active_model_name = "tts"+ backend._generation_lock = threading.Lock()+ backend.models = {"tts": {"audio_type": "snac", "model": _Model(), "tokenizer": _Tokenizer()}}+ backend._audio_codec_manager = types.SimpleNamespace(+ decode_snac = lambda *_args: (b"RIFFfake", 24000)+ )++ worker_responses = queue.Queue()+ _handle_generate_audio(+ backend,+ {"request_id": "audio-1", "text": "A long paragraph.", "max_new_tokens": 14},+ worker_responses,+ threading.Event(),+ )+ done = worker_responses.get_nowait()++ orchestrator = _bare_orchestrator()+ monkeypatch.setattr(orchestrator, "_ensure_subprocess_alive", lambda: True)+ monkeypatch.setattr(orchestrator, "_send_cmd", lambda cmd: None)++ def direct_reader(request_id, cancel_event = None):+ return (+ lambda *, timeout: {**done, "request_id": request_id},+ lambda **_kwargs: None,+ lambda: None,+ )++ monkeypatch.setattr(orchestrator, "_direct_reader", direct_reader)+ holder = {}+ orchestrator.generate_audio_response("A long paragraph.", stats_holder = holder)++ assert holder["stats"]["truncated"] is truncated++[email protected]("audio_type", ("bicodec", "dac"))[email protected](("last_token", "truncated"), ((4242, True), (7, False)))+def test_token_codec_speech_cut_at_max_tokens_is_reported(audio_type, last_token, truncated):+ pytest.importorskip("peft")+ import torch+ from core.inference.inference import InferenceBackend++ class _Inputs(dict):+ def __init__(self):+ super().__init__(input_ids = torch.tensor([[1, 2, 3]]))+ self.input_ids = self["input_ids"]++ def to(self, _device):+ return self++ class _Tokenizer:+ eos_token_id = 7+ pad_token_id = 0++ def __call__(self, texts, return_tensors):+ return _Inputs()++ def batch_decode(self, tokens, skip_special_tokens):+ return [""]++ class _Model:+ device = torch.device("cpu")+ dtype = torch.float32+ generation_config = types.SimpleNamespace(eos_token_id = 7)++ def generate(self, input_ids, max_new_tokens, **_kwargs):+ codes = [4242] * (max_new_tokens - 1) + [last_token]+ return torch.cat([input_ids, torch.tensor([codes])], dim = 1)++ backend = InferenceBackend.__new__(InferenceBackend)+ backend.active_model_name = "tts"+ backend._generation_lock = threading.Lock()+ backend.models = {+ "tts": {"audio_type": audio_type, "model": _Model(), "tokenizer": _Tokenizer()}+ }+ backend._audio_codec_manager = types.SimpleNamespace(+ decode_bicodec = lambda *_args: (b"RIFFfake", 16000),+ decode_dac = lambda *_args: (b"RIFFfake", 24000),+ )+ backend._patch_repetition_penalty_processor = lambda: None++ backend.generate_audio_response("A long paragraph.", max_new_tokens = 14)++ assert backend.last_generation_stats["truncated"] is truncateddiff --git a/studio/frontend/src/features/audio/api.ts b/studio/frontend/src/features/audio/api.tsindex 5064df6ab1e..59ca8119d8a 100644--- a/studio/frontend/src/features/audio/api.ts+++ b/studio/frontend/src/features/audio/api.ts@@ -21,6 +21,7 @@ export interface GenerateAudioResponse {model: string;audio: GeneratedAudio;clip_id?: string | null;+ choices: { finish_reason: string }[];}export interface GenerateAudioOptions {diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsxindex 78df6616c96..72b78769349 100644--- a/studio/frontend/src/features/audio/audio-page.tsx+++ b/studio/frontend/src/features/audio/audio-page.tsx@@ -2179,6 +2179,12 @@ export function AudioPage({: {}),signal: controller.signal,});+ if (generated.choices[0]?.finish_reason === "length")+ toast.warning(+ maxTokens < TTS_MAX_TOKENS+ ? "Speech stopped at the Max tokens limit before the end of the text. Raise Max tokens under Advanced to hear the rest."+ : "Speech stopped at the Max tokens limit before the end of the text. Split the text into shorter parts to hear the rest.",+ );updateGenerationPhase("finishing");const refreshed = await refreshGallery();const generatedClip = persistedClipForGeneration(