Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio: LFM3 spoken replies and read aloud on the transformers backend (#16972)
Leo Borcherding605a6f167997
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 8dbc7733a2d..fc81f07b920 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -34,6 +34,7 @@ from utils.hardware import (from core.inference.audio_codecs import AudioCodecManagerfrom core.inference.context_refusal import ContextBudgetExceededfrom core.inference.runtime_context import (+ UNSET_GENERATION_BUDGET,generation_budget_within_context,runtime_context_length,)@@ -801,6 +802,72 @@ def _audio_chat_auto_model(config):return getattr(transformers, "AutoModelForMultimodalLM", None)+# Audio sampling stays with each checkpoint's generation_config: LFM3-Audio and -Omni differ.+_SPEECH_REPLY_SYSTEM_PROMPT = "Respond with interleaved text and audio."+_TTS_LOCALES = {+ "ar": "Arabic",+ "da": "Danish",+ "de": "German",+ "el": "Greek",+ "en": "American English",+ "en-us": "American English",+ "en-gb": "British English",+ "fi": "Finnish",+ "fr": "French",+ "he": "Hebrew",+ "it": "Italian",+ "ja": "Japanese",+ "ko": "Korean",+ "nl": "Dutch",+ "pl": "Polish",+ "pt": "Portuguese",+ "es": "Spanish",+ "sv": "Swedish",+ "tr": "Turkish",+ "zh": "Chinese",+}+++def _speech_codec(model, processor):+ """The processor's bundled codec when the model speaks (LFM3-Audio / -Omni), else None."""+ codec = getattr(processor, "audio_tokenizer", None)+ if codec is None or not isinstance(+ getattr(getattr(model, "config", None), "interleaved_n_audio", None), int+ ):+ return None+ return codec+++def _tts_system_prompt(language: Optional[str], audio_options: Optional[dict]) -> str:+ """The model card's TTS prompt; an unknown language falls back to American English."""+ lang = str(language or "").strip()+ locale = _TTS_LOCALES.get(lang.lower().replace("_", "-")) or next(+ (name for name in _TTS_LOCALES.values() if name.lower() == lang.lower()),+ "American English",+ )+ voice = str((audio_options or {}).get("voice") or "").strip().lower()+ return f"Perform TTS in {locale} using a {'male' if voice == 'male' else 'female'} voice."+++class _SpeechStopped(Exception):+ pass+++def _speech_stop_streamer(cancel_event):+ """LFM3's generate refuses stopping_criteria; a streamer raising on put ends it."""+ from transformers.generation.streamers import BaseStreamer++ class _StopOnCancel(BaseStreamer):+ def put(self, value):+ if cancel_event.is_set():+ raise _SpeechStopped++ def end(self):+ pass++ return _StopOnCancel()++def _processor_takes_text_first(processor) -> bool:"""LFM3-Omni's processor is (text, audio, ..., images); the usual VLM order is (images, text)."""# unsloth_zoo wraps __call__ as (*args, **kwargs); read the first class that names its arguments.@@ -1489,6 +1556,15 @@ class InferenceBackend:else:_refuse_without_generate(model, model_name)+ if self.models[model_name].get("audio_type") == "audio_vlm":+ self.models[model_name]["speaks"] = (+ _speech_codec(+ self.models[model_name].get("model"),+ self.models[model_name].get("processor"),+ )+ is not None+ )+raise_if_offloaded(self.models[model_name]["model"], device_map, "Inference")self.models[model_name]["context_length"] = runtime_context_length(self.models[model_name].get("model"),@@ -2605,16 +2681,26 @@ class InferenceBackend:cancel_event = None,extra_audio_arrays: Optional[list] = None,tools: Optional[list] = None,+ speak: bool = False,) -> Generator[str, None, None]:"""Audio goes into the chat messages via processor.apply_chat_template (Gemma 3n pattern).``tools`` (a list, empty for tool history alone) renders the catalogue and keeps- ``messages`` as sent."""+ ``messages`` as sent. ``speak`` answers in the model's own voice, as the MLX backend+ does: the reply text, then one ``{"speech": <base64 wav>, "sample_rate"}`` item."""import threadingimport numpy as np# Reset so a failed or uncountable run cannot surface stale stats.self.last_generation_stats = None+ if speak:+ if extra_audio_arrays:+ raise RuntimeError("A spoken reply answers one audio clip.")+ yield from self._speak_audio_reply(+ messages, system_prompt, audio_array, max_new_tokens, use_adapter, cancel_event+ )+ return+model_info = self.models[self.active_model_name]model = model_info["model"]processor = model_info.get("processor") or model_info.get("tokenizer")@@ -2761,6 +2847,118 @@ class InferenceBackend:logger.error(f"Audio input generation error: {e}")raise+ def _speak_audio_reply(+ self, messages, system_prompt, audio_array, max_new_tokens, use_adapter, cancel_event+ ):+ import base64++ system = "\n\n".join(p for p in (_SPEECH_REPLY_SYSTEM_PROMPT, system_prompt) if p)+ conversation = messages_with_attached_image(+ alternating_turns(messages),+ system_prompt = system,+ structured_content = True,+ image = 0,+ audio = audio_array,+ )+ result = self._generate_speech(+ conversation, "<|mixed_start|>", max_new_tokens, use_adapter, cancel_event+ )+ if result is None:+ return+ text, wav, sample_rate = result+ if text:+ yield text+ if wav:+ yield {"speech": base64.b64encode(wav).decode("ascii"), "sample_rate": sample_rate}++ def _speak_text(self, text, max_new_tokens, use_adapter, cancel_event, language, audio_options):+ """The loaded LFM3 model reads ``text`` aloud in its TTS mode: ``(wav_bytes, sample_rate)``."""+ from core.inference.audio_errors import AudioGenerationCancelledError++ conversation = [+ {"role": "system", "content": _tts_system_prompt(language, audio_options)},+ {"role": "user", "content": text},+ ]+ result = self._generate_speech(+ conversation, "<|audio_start|>", max_new_tokens, use_adapter, cancel_event+ )+ if result is None:+ raise AudioGenerationCancelledError("Audio generation cancelled")+ _, wav, sample_rate = result+ if not wav:+ raise RuntimeError("The model returned no speech.")+ return wav, sample_rate++ def _generate_speech(self, conversation, prefill, max_new_tokens, use_adapter, cancel_event):+ """Render ``conversation`` with the assistant turn opened by ``prefill`` and generate.+ Returns ``(text, wav_bytes or None, sample_rate)``, or None once stopped."""+ from core.inference.audio_codecs import _numpy_to_wav_bytes+ from core.inference.chat_template_helpers import neutralize_for_render++ model_info = self.models[self.active_model_name]+ model = model_info["model"]+ processor = model_info.get("processor") or model_info.get("tokenizer")+ codec = _speech_codec(model, processor)+ if codec is None:+ raise RuntimeError("The loaded model cannot answer in speech.")+ # Neutralize before the prefill, which is itself control markup (#7066).+ conversation, _, _ = neutralize_for_render(processor, conversation, None)+ conversation = [*conversation, {"role": "assistant", "content": prefill}]+ inputs = processor.apply_chat_template(+ conversation,+ continue_final_message = True,+ tokenize = True,+ return_dict = True,+ return_tensors = "pt",+ truncation = False,+ ).to(model.device)+ prompt_len = int(inputs["input_ids"].shape[1])+ budget = generation_budget_within_context(model, prompt_len, max_new_tokens)+ # Unset: the free window, capped as the MLX path caps it.+ max_new_tokens = budget if max_new_tokens else min(budget, UNSET_GENERATION_BUDGET)+ kwargs = dict(+ **inputs,+ max_new_tokens = max_new_tokens,+ do_sample = False,+ return_dict_in_generate = True,+ )+ if cancel_event is not None:+ stopper = _speech_stop_streamer(cancel_event)+ kwargs.update(streamer = stopper, audio_streamer = stopper)+ timer = GenerationTimer()+ with self._generation_lock:+ if cancel_event is not None and cancel_event.is_set():+ return None+ self._apply_adapter_state(use_adapter)+ timer.start()+ try:+ out = model.generate(**kwargs)+ except _SpeechStopped:+ return None+ finally:+ timer.finish()+ lengths = getattr(out, "audio_lengths", None)+ frames = int(lengths[0]) if lengths is not None else 0+ wav, sample_rate = None, int(getattr(codec.config, "sampling_rate", None) or 24000)+ if frames > 0:+ device = next(model.parameters()).device+ if next(codec.parameters()).device != device:+ codec.to(device = device)+ with torch.inference_mode():+ waveform = codec.decode(out.audio_codes[:1, :, :frames]).audio_values[0]+ wav = _numpy_to_wav_bytes(waveform.float().cpu().numpy(), sample_rate)+ if cancel_event is not None and cancel_event.is_set():+ return None+ raw_tokenizer = getattr(processor, "tokenizer", processor)+ text = raw_tokenizer.decode(out.sequences[0, prompt_len:], skip_special_tokens = True)+ self._record_generation_stats(+ prompt_tokens = prompt_len,+ completion_tokens = int(out.sequences.shape[1]) - prompt_len,+ max_new_tokens = max_new_tokens,+ timer = timer,+ )+ return text.strip(), wav, sample_rate+def generate_whisper_response(self,audio_array,@@ -3204,8 +3402,9 @@ class InferenceBackend:seed: Optional[int] = None,audio_options: Optional[dict] = None,) -> Tuple[bytes, int]:- """Args unused: reserved for native audio models; codec TTS has no instructions, language or seed."""- del instructions, language, seed+ """Args unused: reserved for native audio models; codec TTS has no instructions, language or seed.+ A chat model that speaks (LFM3) reads the text aloud in its TTS mode, in ``language``."""+ del instructions, seedif not self.active_model_name:raise RuntimeError("No active model")@@ -3216,6 +3415,10 @@ class InferenceBackend:if not audio_type:raise RuntimeError(f"Model {self.active_model_name} is not an audio model")+ if audio_type == "audio_vlm" and model_info.get("speaks"):+ return self._speak_text(+ text, max_new_tokens, use_adapter, cancel_event, language, audio_options+ )top_k = self._normalize_top_k(top_k)# Every codec below concatenates its prompt instead of templating it, so thisdiff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 8b81b13dc37..d5804535a1d 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -734,6 +734,12 @@ def _has_openai_tool_history(messages) -> bool:return False+def _after_newest_user_turn(messages) -> list:+ messages = list(messages or [])+ roles = [m.get("role") if isinstance(m, dict) else getattr(m, "role", None) for m in messages]+ return messages[len(roles) - roles[::-1].index("user") :] if "user" in roles else messages++def _raise_unsupported_openai_parameter(param: str, message: str) -> None:raise HTTPException(status_code = 400,@@ -23015,13 +23021,27 @@ async def _generate_tts_wav(if not backend.active_model_name:raise HTTPException(status_code = 400, detail = "No model loaded.")model_info = backend.models.get(backend.active_model_name, {})- if not model_info.get("is_audio"):+ # A chat model that speaks (LFM3) reads aloud too; the MLX backend has no TTS mode for it.+ chat_speaks = (+ model_info.get("audio_type") == "audio_vlm"+ and bool(model_info.get("speaks"))+ and not model_info.get("is_mlx")+ )+ if not model_info.get("is_audio") and not chat_speaks:raise HTTPException(status_code = 400, detail = "Active model is not an audio model.")model_name = _orchestrator_public_model_id(backend)audio_type = model_info.get("audio_type")audio_family = model_info.get("audio_family")- supported_audio_types = _TRANSFORMERS_TTS_AUDIO_TYPES- _apply_builtin_voice(payload, voice, model_info.get("audio_options"))+ supported_audio_types = _TRANSFORMERS_TTS_AUDIO_TYPES | (+ {"audio_vlm"} if chat_speaks else set()+ )+ _apply_builtin_voice(+ payload,+ voice,+ [{"name": "voice", "values": ["female", "male"]}]+ if chat_speaks+ else model_info.get("audio_options"),+ )_audio_model_id = getattr(backend, "active_model_name", None) or model_namegen = lambda: backend.generate_audio_response(text = text,@@ -31415,6 +31435,13 @@ async def produce_openai_chat_completions(_speak_detail = "The loaded model cannot answer in speech."elif _speak and not payload.audio_base64:_speak_detail = "A spoken reply answers a spoken question: send it as audio_base64."+ elif _speak and payload.extra_audio_base64:+ _speak_detail = "A spoken reply answers one audio clip: drop extra_audio_base64."+ elif _speak and _has_openai_tool_history(_after_newest_user_turn(payload.messages)):+ # The spoken prompt ends on the newest user turn, so later tool results would be lost.+ _speak_detail = (+ "A spoken reply answers the newest user turn: drop the tool turns after it."+ )elif _speak and _normalize_stop_sequences(payload.stop):# The speech cannot be cut at a text marker, so the stop would not hold._speak_detail = "Stop sequences do not apply to a spoken reply."diff --git a/studio/backend/tests/test_chat_input_audio_part.py b/studio/backend/tests/test_chat_input_audio_part.pyindex 7cd6a15160d..3686cc82161 100644--- a/studio/backend/tests/test_chat_input_audio_part.py+++ b/studio/backend/tests/test_chat_input_audio_part.py@@ -849,3 +849,73 @@ def test_a_spoken_question_can_call_a_client_tool(monkeypatch, stream, cut, choicalls = [t["function"] for t in chunks[0]["message"].get("tool_calls") or []]promoted = [{"name": "lookup", "arguments": '{"q": "Paris"}'}] if choice != "none" else []assert (calls, chunks[-1]["finish_reason"] == "tool_calls") == (promoted, bool(promoted))+++def test_a_spoken_reply_with_extra_clips_is_a_client_error(monkeypatch):+ from types import SimpleNamespace++ seen = {}++ def _generate(**kwargs):+ seen.update(kwargs)+ yield "unreached"++ backend = SimpleNamespace(active_model_name = "m", generate_audio_input_response = _generate)+ backend.models = {"m": {"is_mlx": False, "has_audio_input": True, "speaks": True}}+ idle = SimpleNamespace(is_loaded = False)+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: idle)+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend)+ monkeypatch.setattr(inference_route, "_decode_audio_clips", lambda clips: [[0.0]] * len(clips))+ body = {+ "model": "m",+ "messages": [{"role": "user", "content": ""}],+ "audio_base64": AUDIO_B64,+ "extra_audio_base64": [AUDIO_B64],+ "modalities": ["text", "audio"],+ }+ with _route_client() as client:+ response = client.post("/chat/completions", json = body)++ assert response.status_code == 400 and not seen+ assert "one audio clip" in response.text+++_CALL = {"id": "c1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}++[email protected]("trailing_tool", [True, False])+def test_a_spoken_reply_refuses_tool_turns_after_the_question(monkeypatch, trailing_tool):+ from types import SimpleNamespace++ seen = {}++ def _generate(**kwargs):+ seen.update(kwargs)+ yield "ok"++ backend = SimpleNamespace(active_model_name = "m", generate_audio_input_response = _generate)+ backend.models = {"m": {"is_mlx": False, "has_audio_input": True, "speaks": True}}+ idle = SimpleNamespace(is_loaded = False)+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: idle)+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend)+ monkeypatch.setattr(inference_route, "_decode_audio_clips", lambda clips: [[0.0]] * len(clips))+ tool_turns = [+ {"role": "assistant", "content": "", "tool_calls": [_CALL]},+ {"role": "tool", "tool_call_id": "c1", "content": "Paris"},+ ]+ question = [{"role": "user", "content": ""}]+ messages = question + tool_turns if trailing_tool else tool_turns + question+ body = {+ "model": "m",+ "messages": messages,+ "audio_base64": AUDIO_B64,+ "modalities": ["text", "audio"],+ }+ with _route_client() as client:+ response = client.post("/chat/completions", json = body)++ if trailing_tool:+ assert response.status_code == 400 and not seen+ assert "tool turns after it" in response.text+ else:+ assert seen.get("speak") is Truediff --git a/studio/backend/tests/test_lfm3_spoken_reply.py b/studio/backend/tests/test_lfm3_spoken_reply.pynew file mode 100644index 00000000000..6cd206abf57--- /dev/null+++ b/studio/backend/tests/test_lfm3_spoken_reply.py@@ -0,0 +1,346 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""LFM3-Audio / -Omni on the transformers backend: spoken replies (the MLX contract from #15889)+and reading text aloud through the chat model's TTS mode."""++import asyncio+import base64+import importlib+import io+import sys+import threading+import types+import wave+from pathlib import Path+from unittest.mock import MagicMock++import numpy as np+import pytest+import torch+from fastapi import HTTPException++_STUBBED: list[str] = []+++def _stub_if_missing(name, attrs):+ """Stubs a dep the backend pytest job lacks, so collection succeeds; real installs are left alone."""+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001 - any import failure means "not usable here", so stub it+ pass+ _STUBBED.append(name)+ mod = types.ModuleType(name)+ mod.__spec__ = None+ for attr in attrs:+ setattr(mod, attr, MagicMock())+ sys.modules[name] = mod+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, mod)+++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))++import core.inference.inference as inf # noqa: E402++# Drop the stubs once inf is bound: leftover stubs make _shared_policy return None.+for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)++sys.path.insert(0, str(Path(__file__).resolve().parent))+from test_audio_chat_load import _config, backend, multimodal_auto_class # noqa: E402, F401 - fixtures+++class _Codec(torch.nn.Module):+ config = types.SimpleNamespace(sampling_rate = 24000)++ def __init__(self):+ super().__init__()+ self.w = torch.nn.Parameter(torch.zeros(1))+ self.decoded = None++ def decode(self, codes):+ self.decoded = codes+ samples = codes.shape[-1] * 1920+ return types.SimpleNamespace(audio_values = torch.full((1, 1, samples), 0.25))+++class _Tokenizer:+ def decode(+ self,+ ids,+ skip_special_tokens = False,+ ):+ assert skip_special_tokens+ return " Paris is the capital. "+++class _Inputs(dict):+ def to(self, device):+ return self+++class _Processor:+ feature_extractor = object()++ def __init__(self, codec = None):+ self.tokenizer = _Tokenizer()+ self.audio_tokenizer = codec+ self.rendered = None++ def apply_chat_template(self, conversation, **kwargs):+ self.rendered = (conversation, kwargs)+ return _Inputs(input_ids = torch.zeros((1, 7), dtype = torch.long))+++class _Model(torch.nn.Module):+ config = types.SimpleNamespace(interleaved_n_audio = 12)+ device = "cpu"++ def __init__(+ self,+ frames = 3,+ cancel_during = None,+ ):+ super().__init__()+ self.p = torch.nn.Parameter(torch.zeros(1))+ self.frames, self.cancel_during, self.kwargs = frames, cancel_during, None++ def generate(self, **kwargs):+ self.kwargs = kwargs+ streamers = (kwargs.get("streamer"), kwargs.get("audio_streamer"))+ for streamer in {id(s): s for s in streamers if s}.values():+ if self.cancel_during is not None:+ self.cancel_during.set()+ streamer.put(torch.zeros(1))+ codes = torch.full((1, 12, max(self.frames, 1)), -1)+ return types.SimpleNamespace(+ sequences = torch.zeros((1, 7 + 5), dtype = torch.long),+ audio_codes = codes,+ audio_lengths = torch.tensor([self.frames]),+ )+++def _backend(+ model,+ processor,+ audio_type = "audio_vlm",+):+ b = inf.InferenceBackend.__new__(inf.InferenceBackend)+ b.active_model_name = "LiquidAI/LFM3-Audio-2B"+ b.models = {+ b.active_model_name: {+ "model": model,+ "processor": processor,+ "tokenizer": processor,+ "audio_type": audio_type,+ "speaks": inf._speech_codec(model, processor) is not None,+ }+ }+ b._generation_lock = threading.Lock()+ b._apply_adapter_state = lambda *_a, **_k: None+ b.last_generation_stats = None+ return b+++def _spoken(+ b,+ cancel_event = None,+ **kwargs,+):+ return list(+ inf.InferenceBackend.generate_audio_input_response(+ b,+ messages = [{"role": "user", "content": "Answer this."}],+ system_prompt = "Be brief.",+ audio_array = np.zeros(1600, dtype = np.float32),+ temperature = 0.7,+ top_p = 0.9,+ top_k = 40,+ min_p = 0.0,+ max_new_tokens = 256,+ repetition_penalty = 1.0,+ cancel_event = cancel_event,+ **kwargs,+ )+ )+++def _wav(b64):+ with wave.open(io.BytesIO(base64.b64decode(b64))) as w:+ return w.getframerate(), w.getnframes()+++def test_speech_codec_needs_the_bundled_codec_and_speech_frames():+ assert inf._speech_codec(_Model(), _Processor(_Codec())) is not None+ assert inf._speech_codec(_Model(), _Processor(None)) is None # Gemma 3n, Qwen-Omni text-only+ plain = _Model()+ plain.config = types.SimpleNamespace()+ assert inf._speech_codec(plain, _Processor(_Codec())) is None++[email protected]("codec, speaks", [(True, True), (False, False)])+def test_load_reports_speaks(backend, codec, speaks): # noqa: F811+ b, _ = backend+ model, processor = _Model(), _Processor(_Codec() if codec else None)+ # The backend fixture put a per-test unsloth module in sys.modules; load_model imports from it.+ sys.modules["unsloth"].FastModel = types.SimpleNamespace(+ from_pretrained = lambda **_k: (model, processor), for_inference = lambda *_a: None+ )+ assert inf.InferenceBackend.load_model(b, _config(), trust_remote_code = True) is True+ assert b.models["LiquidAI/LFM3-Audio-2B"]["speaks"] is speaks+++def test_spoken_reply_yields_text_then_one_wav():+ codec = _Codec()+ model, processor = _Model(frames = 3), _Processor(codec)+ b = _backend(model, processor)++ text, speech = _spoken(b, speak = True)++ assert text == "Paris is the capital."+ rate, frames = _wav(speech["speech"])+ assert (rate, frames, speech["sample_rate"]) == (24000, 3 * 1920, 24000)+ assert codec.decoded.shape == (1, 12, 3)+ conversation, kwargs = processor.rendered+ assert conversation[0]["role"] == "system"+ assert "Respond with interleaved text and audio." in str(conversation[0]["content"])+ assert "Be brief." in str(conversation[0]["content"])+ assert conversation[-1] == {"role": "assistant", "content": "<|mixed_start|>"}+ assert kwargs["continue_final_message"] is True+ assert model.kwargs["return_dict_in_generate"] is True+ # Each checkpoint's generation_config carries its own audio sampling (Audio and Omni differ).+ assert not [k for k in model.kwargs if k.startswith("audio_")]+ assert b.last_generation_stats["usage"]["completion_tokens"] == 5+++def test_spoken_reply_without_speech_frames_returns_text_alone():+ b = _backend(_Model(frames = 0), _Processor(_Codec()))+ assert _spoken(b, speak = True) == ["Paris is the capital."]+++def test_stopping_a_spoken_reply_yields_nothing():+ cancel = threading.Event()+ b = _backend(_Model(cancel_during = cancel), _Processor(_Codec()))+ assert _spoken(b, cancel_event = cancel, speak = True) == []+++def test_spoken_reply_refuses_a_model_that_cannot_speak():+ b = _backend(_Model(), _Processor(None))+ with pytest.raises(RuntimeError, match = "cannot answer in speech"):+ _spoken(b, speak = True)+++def test_read_aloud_uses_the_chat_models_tts_mode():+ model, processor = _Model(frames = 2), _Processor(_Codec())+ b = _backend(model, processor)++ wav, rate = inf.InferenceBackend.generate_audio_response(+ b,+ text = "Unsloth makes fine tuning fast.",+ language = "en-GB",+ audio_options = {"voice": "male"},+ )++ assert rate == 24000 and wav[:4] == b"RIFF"+ conversation, _ = processor.rendered+ assert conversation[0]["content"] == "Perform TTS in British English using a male voice."+ assert conversation[1]["content"] == "Unsloth makes fine tuning fast."+ assert conversation[-1] == {"role": "assistant", "content": "<|audio_start|>"}+++def test_read_aloud_with_no_speech_is_an_error_and_a_stop_is_a_cancel():+ from core.inference.audio_errors import AudioGenerationCancelledError++ with pytest.raises(RuntimeError, match = "no speech"):+ inf.InferenceBackend.generate_audio_response(+ _backend(_Model(frames = 0), _Processor(_Codec())), text = "hi"+ )+ cancel = threading.Event()+ with pytest.raises(AudioGenerationCancelledError):+ inf.InferenceBackend.generate_audio_response(+ _backend(_Model(cancel_during = cancel), _Processor(_Codec())),+ text = "hi",+ cancel_event = cancel,+ )+++ "language, voice, expected",+ [+ (None, None, "Perform TTS in American English using a female voice."),+ ("fr", "female", "Perform TTS in French using a female voice."),+ ("Japanese", "male", "Perform TTS in Japanese using a male voice."),+ ("xx", None, "Perform TTS in American English using a female voice."),+ ],+)+def test_tts_system_prompt(language, voice, expected):+ assert inf._tts_system_prompt(language, {"voice": voice} if voice else None) == expected+++def test_tts_route_serves_a_speaking_chat_model_and_refuses_a_silent_one(monkeypatch):+ import routes.inference as route+ from models.inference import ChatCompletionRequest++ class _Llama:+ is_loaded = False+ _is_audio = False++ seen = []++ def backend_for(speaks, is_mlx = False):+ class _Orchestrator:+ active_model_name = "LiquidAI/LFM3-Audio-2B"+ models = {+ active_model_name: {+ "is_audio": False,+ "audio_type": "audio_vlm",+ "speaks": speaks,+ "is_mlx": is_mlx,+ }+ }++ def generate_audio_response(self, **kwargs):+ seen.append(kwargs)+ return b"RIFFwav", 24000++ return _Orchestrator()++ async def _no_switch(*_a, **_k):+ return None++ monkeypatch.setattr(route, "get_llama_cpp_backend", lambda: _Llama())+ monkeypatch.setattr(route, "_maybe_auto_switch_model", _no_switch)+ payload = ChatCompletionRequest(messages = [{"role": "user", "content": "hello"}])++ monkeypatch.setattr(route, "get_inference_backend", lambda: backend_for(True))+ wav, rate, _, audio_type = asyncio.run(route._generate_tts_wav("hello", payload, None, "t"))+ assert (wav, rate, audio_type) == (b"RIFFwav", 24000, "audio_vlm")+ asyncio.run(route._generate_tts_wav("hello", payload, None, "t", voice = "male"))+ assert seen[-1]["audio_options"] == {"voice": "male"}++ for silent in (backend_for(False), backend_for(True, is_mlx = True)):+ monkeypatch.setattr(route, "get_inference_backend", lambda silent = silent: silent)+ with pytest.raises(HTTPException) as exc:+ asyncio.run(route._generate_tts_wav("hello", payload, None, "t"))+ assert exc.value.detail == "Active model is not an audio model."++[email protected]("window, expected", [(20, 13), (100_000, 2048), (None, 2048)])+def test_unset_spoken_budget_fits_the_served_window(monkeypatch, window, expected):+ import core.inference.runtime_context as rc++ monkeypatch.setattr(rc, "runtime_context_length", lambda _m, *_a: window)+ model = _Model()+ b = _backend(model, _Processor(_Codec()))+ list(+ inf.InferenceBackend._speak_audio_reply(+ b, [{"role": "user", "content": "Hi"}], "", np.zeros(1600, np.float32), None, None, None+ )+ )+ assert model.kwargs["max_new_tokens"] == expected