Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: say so when the Audio page stops speech at Max tokens (#12583)

Nilaydccfb699358b
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index f88709fe1ec..34007175d17 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -2549,6 +2549,7 @@ class InferenceBackend:
if use_adapter is not None:
self._apply_adapter_state(use_adapter)
stopping_criteria = self._cancel_stopping_criteria(cancel_event)
+ self.last_generation_stats = None
if audio_type == "snac":
result = self._generate_snac(
@@ -2638,6 +2639,12 @@ class InferenceBackend:
)
if cancel_event is not None and cancel_event.is_set():
raise RuntimeError("Audio generation cancelled")
+ self._record_generation_stats(
+ prompt_tokens = input_ids.shape[1],
+ completion_tokens = self._generated_token_count(model, generated, input_ids.shape[1]),
+ max_new_tokens = max_new_tokens,
+ ended_on_stop_token = self._ended_on_stop_token(generated, 128258),
+ )
return self._audio_codec_manager.decode_snac(generated, str(device))
def _generate_csm(
@@ -2692,7 +2699,14 @@ class InferenceBackend:
)
if cancel_event is not None and cancel_event.is_set():
raise RuntimeError("Audio generation cancelled")
- new_tokens = generated[:, inputs.input_ids.shape[1] :]
+ prompt_len = inputs.input_ids.shape[1]
+ self._record_generation_stats(
+ prompt_tokens = prompt_len,
+ completion_tokens = self._generated_token_count(model, generated, prompt_len),
+ max_new_tokens = max_new_tokens,
+ ended_on_stop_token = self._ended_on_stop_token(generated, tokenizer.eos_token_id),
+ )
+ new_tokens = generated[:, prompt_len:]
decoded_text = tokenizer.batch_decode(new_tokens, skip_special_tokens = False)[0]
return self._audio_codec_manager.decode_bicodec(decoded_text, str(model.device))
@@ -2758,6 +2772,16 @@ class InferenceBackend:
)
if cancel_event is not None and cancel_event.is_set():
raise RuntimeError("Audio generation cancelled")
+ self._record_generation_stats(
+ prompt_tokens = inputs.input_ids.shape[1],
+ completion_tokens = self._generated_token_count(
+ model, generated, inputs.input_ids.shape[1]
+ ),
+ max_new_tokens = max_new_tokens,
+ ended_on_stop_token = self._ended_on_stop_token(
+ generated, self._generation_stop_token_ids(model, {})
+ ),
+ )
decoded_text = tokenizer.batch_decode(generated, skip_special_tokens = False)[0]
return self._audio_codec_manager.decode_dac(decoded_text, str(model.device))
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index f7d0183458e..1f1f6051996 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -40850,6 +40850,7 @@ class LlamaCppBackend:
max_new_tokens: int = 2048,
repetition_penalty: float = 1.1,
cancel_event: Optional[threading.Event] = None,
+ stats_holder: Optional[dict] = None,
) -> tuple:
"""
Generate TTS audio via llama-server /completion + codec decode.
@@ -40934,6 +40935,8 @@ class LlamaCppBackend:
raise RuntimeError("Audio generation cancelled")
data = resp.json()
+ if stats_holder is not None:
+ stats_holder["stats"] = {"truncated": data.get("stop_type") == "limit"}
token_ids = (
[p["id"] for p in data.get("completion_probabilities", []) if "id" in p]
if need_ids
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 4d682b054b6..4585e8b14be 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -3339,6 +3339,7 @@ class InferenceOrchestrator:
language: Optional[str] = None,
seed: Optional[int] = None,
audio_options: Optional[dict] = None,
+ stats_holder: Optional[dict] = None,
) -> Tuple[bytes, int]:
"""Generate TTS audio. Returns (wav_bytes, sample_rate). Blocking: sends the command and
waits for the full audio response."""
@@ -3471,6 +3472,8 @@ class InferenceOrchestrator:
raise AudioGenerationCancelledError("Audio generation cancelled")
wav_bytes = base64.b64decode(resp["wav_base64"])
sample_rate = resp["sample_rate"]
+ if stats_holder is not None:
+ stats_holder["stats"] = resp.get("stats")
return wav_bytes, sample_rate
if rtype == "audio_error":
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index b78e265586b..d286f57d273 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -1617,6 +1617,7 @@ def _handle_generate_audio(backend, cmd: dict, resp_queue: Any, cancel_event) ->
"request_id": request_id,
"wav_base64": base64.b64encode(wav_bytes).decode("ascii"),
"sample_rate": sample_rate,
+ "stats": getattr(backend, "last_generation_stats", None),
},
)
logger.info("Finished audio generation for request_id=%s", request_id)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 5dd1cc7992b..7b0deb444d1 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -21288,6 +21288,7 @@ async def _generate_tts_wav(
*,
speech_api_default_max_tokens: bool = False,
requested_model: str = _RELOAD_ONLY_MODEL,
+ stats_holder: Optional[dict] = None,
) -> tuple[bytes, int, str, Optional[str]]:
"""Shared core of /audio/generate and /audio/speech. Returns
(wav_bytes, sample_rate, model_name, audio_type)."""
@@ -21349,6 +21350,7 @@ async def _generate_tts_wav(
),
repetition_penalty = payload.repetition_penalty,
cancel_event = _audio_cancel,
+ stats_holder = stats_holder,
)
else:
backend = await asyncio.to_thread(get_inference_backend)
@@ -21381,6 +21383,7 @@ async def _generate_tts_wav(
language = payload.audio_language,
seed = payload.seed,
**({"audio_options": payload.audio_options} if payload.audio_options else {}),
+ stats_holder = stats_holder,
)
if audio_type not in supported_audio_types:
@@ -21558,6 +21561,7 @@ async def generate_audio(
raise HTTPException(status_code = 400, detail = "No user message found.")
text = last_user_msg["content"]
+ tts_stats: dict = {}
wav_bytes, sample_rate, model_name, audio_type = await _generate_tts_wav(
text,
payload,
@@ -21567,7 +21571,9 @@ async def generate_audio(
# request model, and without this it stops the hook at its falsey check before
# the idle-stash restore, failing a request the sibling route serves.
requested_model = _switch_model_for_payload(payload) or _RELOAD_ONLY_MODEL,
+ stats_holder = tts_stats,
)
+ truncated = bool((tts_stats.get("stats") or {}).get("truncated"))
persisted_clip = await asyncio.to_thread(
_persist_tts_clip, wav_bytes, sample_rate, text, model_name, audio_type
)
@@ -21587,7 +21593,7 @@ async def generate_audio(
"role": "assistant",
"content": text,
},
- "finish_reason": "stop",
+ "finish_reason": "length" if truncated else "stop",
}
],
}
diff --git a/studio/backend/tests/test_audio_tts_cancellation.py b/studio/backend/tests/test_audio_tts_cancellation.py
index 177b6715133..1a1e307bd52 100644
--- a/studio/backend/tests/test_audio_tts_cancellation.py
+++ b/studio/backend/tests/test_audio_tts_cancellation.py
@@ -4,6 +4,7 @@
import asyncio
import base64
import importlib
+import json
import queue
import sys
import threading
@@ -765,3 +766,170 @@ def test_backend_tts_generation_uses_cancel_stopping_criteria(monkeypatch):
assert backend.generate_audio_response("hello", cancel_event = cancel) == (b"RIFFfake", 24000)
assert captured["stopping_criteria"] is criteria
+
+
[email protected](("stop_type", "finish_reason"), (("limit", "length"), ("eos", "stop")))
+def test_gguf_speech_cut_at_max_tokens_is_reported(monkeypatch, stop_type, finish_reason):
+ import core.inference.llama_cpp as llama_cpp
+
+ sent = {}
+
+ class _Resp:
+ status_code = 200
+
+ def json(self):
+ return {"content": "<custom_token_10>", "stop_type": stop_type}
+
+ class _Client:
+ def __init__(self, *_args, **_kwargs):
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_exc):
+ return False
+
+ def post(
+ self,
+ url,
+ json = None,
+ **_kwargs,
+ ):
+ sent.update(json)
+ return _Resp()
+
+ class _Llama(llama_cpp.LlamaCppBackend):
+ is_loaded = True
+ _is_audio = True
+ _audio_type = "snac"
+ model_identifier = "unsloth/orpheus-3b-0.1-ft-GGUF"
+ base_url = "http://127.0.0.1:8080"
+ _auth_headers: dict = {}
+
+ async def _noop_switch(*_args, **_kwargs):
+ return None
+
+ monkeypatch.setattr(llama_cpp.httpx, "Client", _Client)
+ monkeypatch.setattr(
+ llama_cpp.LlamaCppBackend,
+ "_codec_mgr",
+ types.SimpleNamespace(decode = lambda *_args, **_kwargs: (b"RIFFfake", 24000)),
+ )
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: _Llama.__new__(_Llama))
+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _noop_switch)
+ monkeypatch.setattr(inference_route, "_persist_tts_clip", lambda *_args: None)
+ payload = ChatCompletionRequest(
+ messages = [{"role": "user", "content": "A long paragraph."}],
+ max_tokens = 2048,
+ )
+
+ response = asyncio.run(
+ inference_route.generate_audio(payload, request = None, current_subject = "t")
+ )
+
+ assert sent["n_predict"] == 2048
+ assert json.loads(response.body)["choices"][0]["finish_reason"] == finish_reason
+
+
[email protected](("last_token", "truncated"), ((4242, True), (128258, False)))
+def test_snac_speech_cut_at_max_tokens_is_reported(monkeypatch, last_token, truncated):
+ pytest.importorskip("peft")
+ import torch
+ from core.inference.inference import InferenceBackend
+
+ class _Tokenizer:
+ def __call__(self, text, return_tensors):
+ return types.SimpleNamespace(input_ids = torch.tensor([[1, 2, 3]]))
+
+ class _Model:
+ device = torch.device("cpu")
+
+ def generate(self, input_ids, max_new_tokens, **_kwargs):
+ codes = [4242] * (max_new_tokens - 1) + [last_token]
+ return torch.cat([input_ids, torch.tensor([codes])], dim = 1)
+
+ backend = InferenceBackend.__new__(InferenceBackend)
+ backend.active_model_name = "tts"
+ backend._generation_lock = threading.Lock()
+ backend.models = {"tts": {"audio_type": "snac", "model": _Model(), "tokenizer": _Tokenizer()}}
+ backend._audio_codec_manager = types.SimpleNamespace(
+ decode_snac = lambda *_args: (b"RIFFfake", 24000)
+ )
+
+ worker_responses = queue.Queue()
+ _handle_generate_audio(
+ backend,
+ {"request_id": "audio-1", "text": "A long paragraph.", "max_new_tokens": 14},
+ worker_responses,
+ threading.Event(),
+ )
+ done = worker_responses.get_nowait()
+
+ orchestrator = _bare_orchestrator()
+ monkeypatch.setattr(orchestrator, "_ensure_subprocess_alive", lambda: True)
+ monkeypatch.setattr(orchestrator, "_send_cmd", lambda cmd: None)
+
+ def direct_reader(request_id, cancel_event = None):
+ return (
+ lambda *, timeout: {**done, "request_id": request_id},
+ lambda **_kwargs: None,
+ lambda: None,
+ )
+
+ monkeypatch.setattr(orchestrator, "_direct_reader", direct_reader)
+ holder = {}
+ orchestrator.generate_audio_response("A long paragraph.", stats_holder = holder)
+
+ assert holder["stats"]["truncated"] is truncated
+
+
[email protected]("audio_type", ("bicodec", "dac"))
[email protected](("last_token", "truncated"), ((4242, True), (7, False)))
+def test_token_codec_speech_cut_at_max_tokens_is_reported(audio_type, last_token, truncated):
+ pytest.importorskip("peft")
+ import torch
+ from core.inference.inference import InferenceBackend
+
+ class _Inputs(dict):
+ def __init__(self):
+ super().__init__(input_ids = torch.tensor([[1, 2, 3]]))
+ self.input_ids = self["input_ids"]
+
+ def to(self, _device):
+ return self
+
+ class _Tokenizer:
+ eos_token_id = 7
+ pad_token_id = 0
+
+ def __call__(self, texts, return_tensors):
+ return _Inputs()
+
+ def batch_decode(self, tokens, skip_special_tokens):
+ return [""]
+
+ class _Model:
+ device = torch.device("cpu")
+ dtype = torch.float32
+ generation_config = types.SimpleNamespace(eos_token_id = 7)
+
+ def generate(self, input_ids, max_new_tokens, **_kwargs):
+ codes = [4242] * (max_new_tokens - 1) + [last_token]
+ return torch.cat([input_ids, torch.tensor([codes])], dim = 1)
+
+ backend = InferenceBackend.__new__(InferenceBackend)
+ backend.active_model_name = "tts"
+ backend._generation_lock = threading.Lock()
+ backend.models = {
+ "tts": {"audio_type": audio_type, "model": _Model(), "tokenizer": _Tokenizer()}
+ }
+ backend._audio_codec_manager = types.SimpleNamespace(
+ decode_bicodec = lambda *_args: (b"RIFFfake", 16000),
+ decode_dac = lambda *_args: (b"RIFFfake", 24000),
+ )
+ backend._patch_repetition_penalty_processor = lambda: None
+
+ backend.generate_audio_response("A long paragraph.", max_new_tokens = 14)
+
+ assert backend.last_generation_stats["truncated"] is truncated
diff --git a/studio/frontend/src/features/audio/api.ts b/studio/frontend/src/features/audio/api.ts
index 5064df6ab1e..59ca8119d8a 100644
--- a/studio/frontend/src/features/audio/api.ts
+++ b/studio/frontend/src/features/audio/api.ts
@@ -21,6 +21,7 @@ export interface GenerateAudioResponse {
model: string;
audio: GeneratedAudio;
clip_id?: string | null;
+ choices: { finish_reason: string }[];
}
export interface GenerateAudioOptions {
diff --git a/studio/frontend/src/features/audio/audio-page.tsx b/studio/frontend/src/features/audio/audio-page.tsx
index 78df6616c96..72b78769349 100644
--- a/studio/frontend/src/features/audio/audio-page.tsx
+++ b/studio/frontend/src/features/audio/audio-page.tsx
@@ -2179,6 +2179,12 @@ export function AudioPage({
: {}),
signal: controller.signal,
});
+ if (generated.choices[0]?.finish_reason === "length")
+ toast.warning(
+ maxTokens < TTS_MAX_TOKENS
+ ? "Speech stopped at the Max tokens limit before the end of the text. Raise Max tokens under Advanced to hear the rest."
+ : "Speech stopped at the Max tokens limit before the end of the text. Split the text into shorter parts to hear the rest.",
+ );
updateGenerationPhase("finishing");
const refreshed = await refreshGallery();
const generatedClip = persistedClipForGeneration(