Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio: LFM3 spoken replies and read aloud on the transformers backend (#16972)

Leo Borcherding605a6f167997
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 8dbc7733a2d..fc81f07b920 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -34,6 +34,7 @@ from utils.hardware import (
from core.inference.audio_codecs import AudioCodecManager
from core.inference.context_refusal import ContextBudgetExceeded
from core.inference.runtime_context import (
+ UNSET_GENERATION_BUDGET,
generation_budget_within_context,
runtime_context_length,
)
@@ -801,6 +802,72 @@ def _audio_chat_auto_model(config):
return getattr(transformers, "AutoModelForMultimodalLM", None)
+# Audio sampling stays with each checkpoint's generation_config: LFM3-Audio and -Omni differ.
+_SPEECH_REPLY_SYSTEM_PROMPT = "Respond with interleaved text and audio."
+_TTS_LOCALES = {
+ "ar": "Arabic",
+ "da": "Danish",
+ "de": "German",
+ "el": "Greek",
+ "en": "American English",
+ "en-us": "American English",
+ "en-gb": "British English",
+ "fi": "Finnish",
+ "fr": "French",
+ "he": "Hebrew",
+ "it": "Italian",
+ "ja": "Japanese",
+ "ko": "Korean",
+ "nl": "Dutch",
+ "pl": "Polish",
+ "pt": "Portuguese",
+ "es": "Spanish",
+ "sv": "Swedish",
+ "tr": "Turkish",
+ "zh": "Chinese",
+}
+
+
+def _speech_codec(model, processor):
+ """The processor's bundled codec when the model speaks (LFM3-Audio / -Omni), else None."""
+ codec = getattr(processor, "audio_tokenizer", None)
+ if codec is None or not isinstance(
+ getattr(getattr(model, "config", None), "interleaved_n_audio", None), int
+ ):
+ return None
+ return codec
+
+
+def _tts_system_prompt(language: Optional[str], audio_options: Optional[dict]) -> str:
+ """The model card's TTS prompt; an unknown language falls back to American English."""
+ lang = str(language or "").strip()
+ locale = _TTS_LOCALES.get(lang.lower().replace("_", "-")) or next(
+ (name for name in _TTS_LOCALES.values() if name.lower() == lang.lower()),
+ "American English",
+ )
+ voice = str((audio_options or {}).get("voice") or "").strip().lower()
+ return f"Perform TTS in {locale} using a {'male' if voice == 'male' else 'female'} voice."
+
+
+class _SpeechStopped(Exception):
+ pass
+
+
+def _speech_stop_streamer(cancel_event):
+ """LFM3's generate refuses stopping_criteria; a streamer raising on put ends it."""
+ from transformers.generation.streamers import BaseStreamer
+
+ class _StopOnCancel(BaseStreamer):
+ def put(self, value):
+ if cancel_event.is_set():
+ raise _SpeechStopped
+
+ def end(self):
+ pass
+
+ return _StopOnCancel()
+
+
def _processor_takes_text_first(processor) -> bool:
"""LFM3-Omni's processor is (text, audio, ..., images); the usual VLM order is (images, text)."""
# unsloth_zoo wraps __call__ as (*args, **kwargs); read the first class that names its arguments.
@@ -1489,6 +1556,15 @@ class InferenceBackend:
else:
_refuse_without_generate(model, model_name)
+ if self.models[model_name].get("audio_type") == "audio_vlm":
+ self.models[model_name]["speaks"] = (
+ _speech_codec(
+ self.models[model_name].get("model"),
+ self.models[model_name].get("processor"),
+ )
+ is not None
+ )
+
raise_if_offloaded(self.models[model_name]["model"], device_map, "Inference")
self.models[model_name]["context_length"] = runtime_context_length(
self.models[model_name].get("model"),
@@ -2605,16 +2681,26 @@ class InferenceBackend:
cancel_event = None,
extra_audio_arrays: Optional[list] = None,
tools: Optional[list] = None,
+ speak: bool = False,
) -> Generator[str, None, None]:
"""Audio goes into the chat messages via processor.apply_chat_template (Gemma 3n pattern).
``tools`` (a list, empty for tool history alone) renders the catalogue and keeps
- ``messages`` as sent."""
+ ``messages`` as sent. ``speak`` answers in the model's own voice, as the MLX backend
+ does: the reply text, then one ``{"speech": <base64 wav>, "sample_rate"}`` item."""
import threading
import numpy as np
# Reset so a failed or uncountable run cannot surface stale stats.
self.last_generation_stats = None
+ if speak:
+ if extra_audio_arrays:
+ raise RuntimeError("A spoken reply answers one audio clip.")
+ yield from self._speak_audio_reply(
+ messages, system_prompt, audio_array, max_new_tokens, use_adapter, cancel_event
+ )
+ return
+
model_info = self.models[self.active_model_name]
model = model_info["model"]
processor = model_info.get("processor") or model_info.get("tokenizer")
@@ -2761,6 +2847,118 @@ class InferenceBackend:
logger.error(f"Audio input generation error: {e}")
raise
+ def _speak_audio_reply(
+ self, messages, system_prompt, audio_array, max_new_tokens, use_adapter, cancel_event
+ ):
+ import base64
+
+ system = "\n\n".join(p for p in (_SPEECH_REPLY_SYSTEM_PROMPT, system_prompt) if p)
+ conversation = messages_with_attached_image(
+ alternating_turns(messages),
+ system_prompt = system,
+ structured_content = True,
+ image = 0,
+ audio = audio_array,
+ )
+ result = self._generate_speech(
+ conversation, "<|mixed_start|>", max_new_tokens, use_adapter, cancel_event
+ )
+ if result is None:
+ return
+ text, wav, sample_rate = result
+ if text:
+ yield text
+ if wav:
+ yield {"speech": base64.b64encode(wav).decode("ascii"), "sample_rate": sample_rate}
+
+ def _speak_text(self, text, max_new_tokens, use_adapter, cancel_event, language, audio_options):
+ """The loaded LFM3 model reads ``text`` aloud in its TTS mode: ``(wav_bytes, sample_rate)``."""
+ from core.inference.audio_errors import AudioGenerationCancelledError
+
+ conversation = [
+ {"role": "system", "content": _tts_system_prompt(language, audio_options)},
+ {"role": "user", "content": text},
+ ]
+ result = self._generate_speech(
+ conversation, "<|audio_start|>", max_new_tokens, use_adapter, cancel_event
+ )
+ if result is None:
+ raise AudioGenerationCancelledError("Audio generation cancelled")
+ _, wav, sample_rate = result
+ if not wav:
+ raise RuntimeError("The model returned no speech.")
+ return wav, sample_rate
+
+ def _generate_speech(self, conversation, prefill, max_new_tokens, use_adapter, cancel_event):
+ """Render ``conversation`` with the assistant turn opened by ``prefill`` and generate.
+ Returns ``(text, wav_bytes or None, sample_rate)``, or None once stopped."""
+ from core.inference.audio_codecs import _numpy_to_wav_bytes
+ from core.inference.chat_template_helpers import neutralize_for_render
+
+ model_info = self.models[self.active_model_name]
+ model = model_info["model"]
+ processor = model_info.get("processor") or model_info.get("tokenizer")
+ codec = _speech_codec(model, processor)
+ if codec is None:
+ raise RuntimeError("The loaded model cannot answer in speech.")
+ # Neutralize before the prefill, which is itself control markup (#7066).
+ conversation, _, _ = neutralize_for_render(processor, conversation, None)
+ conversation = [*conversation, {"role": "assistant", "content": prefill}]
+ inputs = processor.apply_chat_template(
+ conversation,
+ continue_final_message = True,
+ tokenize = True,
+ return_dict = True,
+ return_tensors = "pt",
+ truncation = False,
+ ).to(model.device)
+ prompt_len = int(inputs["input_ids"].shape[1])
+ budget = generation_budget_within_context(model, prompt_len, max_new_tokens)
+ # Unset: the free window, capped as the MLX path caps it.
+ max_new_tokens = budget if max_new_tokens else min(budget, UNSET_GENERATION_BUDGET)
+ kwargs = dict(
+ **inputs,
+ max_new_tokens = max_new_tokens,
+ do_sample = False,
+ return_dict_in_generate = True,
+ )
+ if cancel_event is not None:
+ stopper = _speech_stop_streamer(cancel_event)
+ kwargs.update(streamer = stopper, audio_streamer = stopper)
+ timer = GenerationTimer()
+ with self._generation_lock:
+ if cancel_event is not None and cancel_event.is_set():
+ return None
+ self._apply_adapter_state(use_adapter)
+ timer.start()
+ try:
+ out = model.generate(**kwargs)
+ except _SpeechStopped:
+ return None
+ finally:
+ timer.finish()
+ lengths = getattr(out, "audio_lengths", None)
+ frames = int(lengths[0]) if lengths is not None else 0
+ wav, sample_rate = None, int(getattr(codec.config, "sampling_rate", None) or 24000)
+ if frames > 0:
+ device = next(model.parameters()).device
+ if next(codec.parameters()).device != device:
+ codec.to(device = device)
+ with torch.inference_mode():
+ waveform = codec.decode(out.audio_codes[:1, :, :frames]).audio_values[0]
+ wav = _numpy_to_wav_bytes(waveform.float().cpu().numpy(), sample_rate)
+ if cancel_event is not None and cancel_event.is_set():
+ return None
+ raw_tokenizer = getattr(processor, "tokenizer", processor)
+ text = raw_tokenizer.decode(out.sequences[0, prompt_len:], skip_special_tokens = True)
+ self._record_generation_stats(
+ prompt_tokens = prompt_len,
+ completion_tokens = int(out.sequences.shape[1]) - prompt_len,
+ max_new_tokens = max_new_tokens,
+ timer = timer,
+ )
+ return text.strip(), wav, sample_rate
+
def generate_whisper_response(
self,
audio_array,
@@ -3204,8 +3402,9 @@ class InferenceBackend:
seed: Optional[int] = None,
audio_options: Optional[dict] = None,
) -> Tuple[bytes, int]:
- """Args unused: reserved for native audio models; codec TTS has no instructions, language or seed."""
- del instructions, language, seed
+ """Args unused: reserved for native audio models; codec TTS has no instructions, language or seed.
+ A chat model that speaks (LFM3) reads the text aloud in its TTS mode, in ``language``."""
+ del instructions, seed
if not self.active_model_name:
raise RuntimeError("No active model")
@@ -3216,6 +3415,10 @@ class InferenceBackend:
if not audio_type:
raise RuntimeError(f"Model {self.active_model_name} is not an audio model")
+ if audio_type == "audio_vlm" and model_info.get("speaks"):
+ return self._speak_text(
+ text, max_new_tokens, use_adapter, cancel_event, language, audio_options
+ )
top_k = self._normalize_top_k(top_k)
# Every codec below concatenates its prompt instead of templating it, so this
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 8b81b13dc37..d5804535a1d 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -734,6 +734,12 @@ def _has_openai_tool_history(messages) -> bool:
return False
+def _after_newest_user_turn(messages) -> list:
+ messages = list(messages or [])
+ roles = [m.get("role") if isinstance(m, dict) else getattr(m, "role", None) for m in messages]
+ return messages[len(roles) - roles[::-1].index("user") :] if "user" in roles else messages
+
+
def _raise_unsupported_openai_parameter(param: str, message: str) -> None:
raise HTTPException(
status_code = 400,
@@ -23015,13 +23021,27 @@ async def _generate_tts_wav(
if not backend.active_model_name:
raise HTTPException(status_code = 400, detail = "No model loaded.")
model_info = backend.models.get(backend.active_model_name, {})
- if not model_info.get("is_audio"):
+ # A chat model that speaks (LFM3) reads aloud too; the MLX backend has no TTS mode for it.
+ chat_speaks = (
+ model_info.get("audio_type") == "audio_vlm"
+ and bool(model_info.get("speaks"))
+ and not model_info.get("is_mlx")
+ )
+ if not model_info.get("is_audio") and not chat_speaks:
raise HTTPException(status_code = 400, detail = "Active model is not an audio model.")
model_name = _orchestrator_public_model_id(backend)
audio_type = model_info.get("audio_type")
audio_family = model_info.get("audio_family")
- supported_audio_types = _TRANSFORMERS_TTS_AUDIO_TYPES
- _apply_builtin_voice(payload, voice, model_info.get("audio_options"))
+ supported_audio_types = _TRANSFORMERS_TTS_AUDIO_TYPES | (
+ {"audio_vlm"} if chat_speaks else set()
+ )
+ _apply_builtin_voice(
+ payload,
+ voice,
+ [{"name": "voice", "values": ["female", "male"]}]
+ if chat_speaks
+ else model_info.get("audio_options"),
+ )
_audio_model_id = getattr(backend, "active_model_name", None) or model_name
gen = lambda: backend.generate_audio_response(
text = text,
@@ -31415,6 +31435,13 @@ async def produce_openai_chat_completions(
_speak_detail = "The loaded model cannot answer in speech."
elif _speak and not payload.audio_base64:
_speak_detail = "A spoken reply answers a spoken question: send it as audio_base64."
+ elif _speak and payload.extra_audio_base64:
+ _speak_detail = "A spoken reply answers one audio clip: drop extra_audio_base64."
+ elif _speak and _has_openai_tool_history(_after_newest_user_turn(payload.messages)):
+ # The spoken prompt ends on the newest user turn, so later tool results would be lost.
+ _speak_detail = (
+ "A spoken reply answers the newest user turn: drop the tool turns after it."
+ )
elif _speak and _normalize_stop_sequences(payload.stop):
# The speech cannot be cut at a text marker, so the stop would not hold.
_speak_detail = "Stop sequences do not apply to a spoken reply."
diff --git a/studio/backend/tests/test_chat_input_audio_part.py b/studio/backend/tests/test_chat_input_audio_part.py
index 7cd6a15160d..3686cc82161 100644
--- a/studio/backend/tests/test_chat_input_audio_part.py
+++ b/studio/backend/tests/test_chat_input_audio_part.py
@@ -849,3 +849,73 @@ def test_a_spoken_question_can_call_a_client_tool(monkeypatch, stream, cut, choi
calls = [t["function"] for t in chunks[0]["message"].get("tool_calls") or []]
promoted = [{"name": "lookup", "arguments": '{"q": "Paris"}'}] if choice != "none" else []
assert (calls, chunks[-1]["finish_reason"] == "tool_calls") == (promoted, bool(promoted))
+
+
+def test_a_spoken_reply_with_extra_clips_is_a_client_error(monkeypatch):
+ from types import SimpleNamespace
+
+ seen = {}
+
+ def _generate(**kwargs):
+ seen.update(kwargs)
+ yield "unreached"
+
+ backend = SimpleNamespace(active_model_name = "m", generate_audio_input_response = _generate)
+ backend.models = {"m": {"is_mlx": False, "has_audio_input": True, "speaks": True}}
+ idle = SimpleNamespace(is_loaded = False)
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: idle)
+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend)
+ monkeypatch.setattr(inference_route, "_decode_audio_clips", lambda clips: [[0.0]] * len(clips))
+ body = {
+ "model": "m",
+ "messages": [{"role": "user", "content": ""}],
+ "audio_base64": AUDIO_B64,
+ "extra_audio_base64": [AUDIO_B64],
+ "modalities": ["text", "audio"],
+ }
+ with _route_client() as client:
+ response = client.post("/chat/completions", json = body)
+
+ assert response.status_code == 400 and not seen
+ assert "one audio clip" in response.text
+
+
+_CALL = {"id": "c1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}
+
+
[email protected]("trailing_tool", [True, False])
+def test_a_spoken_reply_refuses_tool_turns_after_the_question(monkeypatch, trailing_tool):
+ from types import SimpleNamespace
+
+ seen = {}
+
+ def _generate(**kwargs):
+ seen.update(kwargs)
+ yield "ok"
+
+ backend = SimpleNamespace(active_model_name = "m", generate_audio_input_response = _generate)
+ backend.models = {"m": {"is_mlx": False, "has_audio_input": True, "speaks": True}}
+ idle = SimpleNamespace(is_loaded = False)
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: idle)
+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: backend)
+ monkeypatch.setattr(inference_route, "_decode_audio_clips", lambda clips: [[0.0]] * len(clips))
+ tool_turns = [
+ {"role": "assistant", "content": "", "tool_calls": [_CALL]},
+ {"role": "tool", "tool_call_id": "c1", "content": "Paris"},
+ ]
+ question = [{"role": "user", "content": ""}]
+ messages = question + tool_turns if trailing_tool else tool_turns + question
+ body = {
+ "model": "m",
+ "messages": messages,
+ "audio_base64": AUDIO_B64,
+ "modalities": ["text", "audio"],
+ }
+ with _route_client() as client:
+ response = client.post("/chat/completions", json = body)
+
+ if trailing_tool:
+ assert response.status_code == 400 and not seen
+ assert "tool turns after it" in response.text
+ else:
+ assert seen.get("speak") is True
diff --git a/studio/backend/tests/test_lfm3_spoken_reply.py b/studio/backend/tests/test_lfm3_spoken_reply.py
new file mode 100644
index 00000000000..6cd206abf57
--- /dev/null
+++ b/studio/backend/tests/test_lfm3_spoken_reply.py
@@ -0,0 +1,346 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""LFM3-Audio / -Omni on the transformers backend: spoken replies (the MLX contract from #15889)
+and reading text aloud through the chat model's TTS mode."""
+
+import asyncio
+import base64
+import importlib
+import io
+import sys
+import threading
+import types
+import wave
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import numpy as np
+import pytest
+import torch
+from fastapi import HTTPException
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(name, attrs):
+ """Stubs a dep the backend pytest job lacks, so collection succeeds; real installs are left alone."""
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001 - any import failure means "not usable here", so stub it
+ pass
+ _STUBBED.append(name)
+ mod = types.ModuleType(name)
+ mod.__spec__ = None
+ for attr in attrs:
+ setattr(mod, attr, MagicMock())
+ sys.modules[name] = mod
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, mod)
+
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+
+import core.inference.inference as inf # noqa: E402
+
+# Drop the stubs once inf is bound: leftover stubs make _shared_policy return None.
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_audio_chat_load import _config, backend, multimodal_auto_class # noqa: E402, F401 - fixtures
+
+
+class _Codec(torch.nn.Module):
+ config = types.SimpleNamespace(sampling_rate = 24000)
+
+ def __init__(self):
+ super().__init__()
+ self.w = torch.nn.Parameter(torch.zeros(1))
+ self.decoded = None
+
+ def decode(self, codes):
+ self.decoded = codes
+ samples = codes.shape[-1] * 1920
+ return types.SimpleNamespace(audio_values = torch.full((1, 1, samples), 0.25))
+
+
+class _Tokenizer:
+ def decode(
+ self,
+ ids,
+ skip_special_tokens = False,
+ ):
+ assert skip_special_tokens
+ return " Paris is the capital. "
+
+
+class _Inputs(dict):
+ def to(self, device):
+ return self
+
+
+class _Processor:
+ feature_extractor = object()
+
+ def __init__(self, codec = None):
+ self.tokenizer = _Tokenizer()
+ self.audio_tokenizer = codec
+ self.rendered = None
+
+ def apply_chat_template(self, conversation, **kwargs):
+ self.rendered = (conversation, kwargs)
+ return _Inputs(input_ids = torch.zeros((1, 7), dtype = torch.long))
+
+
+class _Model(torch.nn.Module):
+ config = types.SimpleNamespace(interleaved_n_audio = 12)
+ device = "cpu"
+
+ def __init__(
+ self,
+ frames = 3,
+ cancel_during = None,
+ ):
+ super().__init__()
+ self.p = torch.nn.Parameter(torch.zeros(1))
+ self.frames, self.cancel_during, self.kwargs = frames, cancel_during, None
+
+ def generate(self, **kwargs):
+ self.kwargs = kwargs
+ streamers = (kwargs.get("streamer"), kwargs.get("audio_streamer"))
+ for streamer in {id(s): s for s in streamers if s}.values():
+ if self.cancel_during is not None:
+ self.cancel_during.set()
+ streamer.put(torch.zeros(1))
+ codes = torch.full((1, 12, max(self.frames, 1)), -1)
+ return types.SimpleNamespace(
+ sequences = torch.zeros((1, 7 + 5), dtype = torch.long),
+ audio_codes = codes,
+ audio_lengths = torch.tensor([self.frames]),
+ )
+
+
+def _backend(
+ model,
+ processor,
+ audio_type = "audio_vlm",
+):
+ b = inf.InferenceBackend.__new__(inf.InferenceBackend)
+ b.active_model_name = "LiquidAI/LFM3-Audio-2B"
+ b.models = {
+ b.active_model_name: {
+ "model": model,
+ "processor": processor,
+ "tokenizer": processor,
+ "audio_type": audio_type,
+ "speaks": inf._speech_codec(model, processor) is not None,
+ }
+ }
+ b._generation_lock = threading.Lock()
+ b._apply_adapter_state = lambda *_a, **_k: None
+ b.last_generation_stats = None
+ return b
+
+
+def _spoken(
+ b,
+ cancel_event = None,
+ **kwargs,
+):
+ return list(
+ inf.InferenceBackend.generate_audio_input_response(
+ b,
+ messages = [{"role": "user", "content": "Answer this."}],
+ system_prompt = "Be brief.",
+ audio_array = np.zeros(1600, dtype = np.float32),
+ temperature = 0.7,
+ top_p = 0.9,
+ top_k = 40,
+ min_p = 0.0,
+ max_new_tokens = 256,
+ repetition_penalty = 1.0,
+ cancel_event = cancel_event,
+ **kwargs,
+ )
+ )
+
+
+def _wav(b64):
+ with wave.open(io.BytesIO(base64.b64decode(b64))) as w:
+ return w.getframerate(), w.getnframes()
+
+
+def test_speech_codec_needs_the_bundled_codec_and_speech_frames():
+ assert inf._speech_codec(_Model(), _Processor(_Codec())) is not None
+ assert inf._speech_codec(_Model(), _Processor(None)) is None # Gemma 3n, Qwen-Omni text-only
+ plain = _Model()
+ plain.config = types.SimpleNamespace()
+ assert inf._speech_codec(plain, _Processor(_Codec())) is None
+
+
[email protected]("codec, speaks", [(True, True), (False, False)])
+def test_load_reports_speaks(backend, codec, speaks): # noqa: F811
+ b, _ = backend
+ model, processor = _Model(), _Processor(_Codec() if codec else None)
+ # The backend fixture put a per-test unsloth module in sys.modules; load_model imports from it.
+ sys.modules["unsloth"].FastModel = types.SimpleNamespace(
+ from_pretrained = lambda **_k: (model, processor), for_inference = lambda *_a: None
+ )
+ assert inf.InferenceBackend.load_model(b, _config(), trust_remote_code = True) is True
+ assert b.models["LiquidAI/LFM3-Audio-2B"]["speaks"] is speaks
+
+
+def test_spoken_reply_yields_text_then_one_wav():
+ codec = _Codec()
+ model, processor = _Model(frames = 3), _Processor(codec)
+ b = _backend(model, processor)
+
+ text, speech = _spoken(b, speak = True)
+
+ assert text == "Paris is the capital."
+ rate, frames = _wav(speech["speech"])
+ assert (rate, frames, speech["sample_rate"]) == (24000, 3 * 1920, 24000)
+ assert codec.decoded.shape == (1, 12, 3)
+ conversation, kwargs = processor.rendered
+ assert conversation[0]["role"] == "system"
+ assert "Respond with interleaved text and audio." in str(conversation[0]["content"])
+ assert "Be brief." in str(conversation[0]["content"])
+ assert conversation[-1] == {"role": "assistant", "content": "<|mixed_start|>"}
+ assert kwargs["continue_final_message"] is True
+ assert model.kwargs["return_dict_in_generate"] is True
+ # Each checkpoint's generation_config carries its own audio sampling (Audio and Omni differ).
+ assert not [k for k in model.kwargs if k.startswith("audio_")]
+ assert b.last_generation_stats["usage"]["completion_tokens"] == 5
+
+
+def test_spoken_reply_without_speech_frames_returns_text_alone():
+ b = _backend(_Model(frames = 0), _Processor(_Codec()))
+ assert _spoken(b, speak = True) == ["Paris is the capital."]
+
+
+def test_stopping_a_spoken_reply_yields_nothing():
+ cancel = threading.Event()
+ b = _backend(_Model(cancel_during = cancel), _Processor(_Codec()))
+ assert _spoken(b, cancel_event = cancel, speak = True) == []
+
+
+def test_spoken_reply_refuses_a_model_that_cannot_speak():
+ b = _backend(_Model(), _Processor(None))
+ with pytest.raises(RuntimeError, match = "cannot answer in speech"):
+ _spoken(b, speak = True)
+
+
+def test_read_aloud_uses_the_chat_models_tts_mode():
+ model, processor = _Model(frames = 2), _Processor(_Codec())
+ b = _backend(model, processor)
+
+ wav, rate = inf.InferenceBackend.generate_audio_response(
+ b,
+ text = "Unsloth makes fine tuning fast.",
+ language = "en-GB",
+ audio_options = {"voice": "male"},
+ )
+
+ assert rate == 24000 and wav[:4] == b"RIFF"
+ conversation, _ = processor.rendered
+ assert conversation[0]["content"] == "Perform TTS in British English using a male voice."
+ assert conversation[1]["content"] == "Unsloth makes fine tuning fast."
+ assert conversation[-1] == {"role": "assistant", "content": "<|audio_start|>"}
+
+
+def test_read_aloud_with_no_speech_is_an_error_and_a_stop_is_a_cancel():
+ from core.inference.audio_errors import AudioGenerationCancelledError
+
+ with pytest.raises(RuntimeError, match = "no speech"):
+ inf.InferenceBackend.generate_audio_response(
+ _backend(_Model(frames = 0), _Processor(_Codec())), text = "hi"
+ )
+ cancel = threading.Event()
+ with pytest.raises(AudioGenerationCancelledError):
+ inf.InferenceBackend.generate_audio_response(
+ _backend(_Model(cancel_during = cancel), _Processor(_Codec())),
+ text = "hi",
+ cancel_event = cancel,
+ )
+
+
+ "language, voice, expected",
+ [
+ (None, None, "Perform TTS in American English using a female voice."),
+ ("fr", "female", "Perform TTS in French using a female voice."),
+ ("Japanese", "male", "Perform TTS in Japanese using a male voice."),
+ ("xx", None, "Perform TTS in American English using a female voice."),
+ ],
+)
+def test_tts_system_prompt(language, voice, expected):
+ assert inf._tts_system_prompt(language, {"voice": voice} if voice else None) == expected
+
+
+def test_tts_route_serves_a_speaking_chat_model_and_refuses_a_silent_one(monkeypatch):
+ import routes.inference as route
+ from models.inference import ChatCompletionRequest
+
+ class _Llama:
+ is_loaded = False
+ _is_audio = False
+
+ seen = []
+
+ def backend_for(speaks, is_mlx = False):
+ class _Orchestrator:
+ active_model_name = "LiquidAI/LFM3-Audio-2B"
+ models = {
+ active_model_name: {
+ "is_audio": False,
+ "audio_type": "audio_vlm",
+ "speaks": speaks,
+ "is_mlx": is_mlx,
+ }
+ }
+
+ def generate_audio_response(self, **kwargs):
+ seen.append(kwargs)
+ return b"RIFFwav", 24000
+
+ return _Orchestrator()
+
+ async def _no_switch(*_a, **_k):
+ return None
+
+ monkeypatch.setattr(route, "get_llama_cpp_backend", lambda: _Llama())
+ monkeypatch.setattr(route, "_maybe_auto_switch_model", _no_switch)
+ payload = ChatCompletionRequest(messages = [{"role": "user", "content": "hello"}])
+
+ monkeypatch.setattr(route, "get_inference_backend", lambda: backend_for(True))
+ wav, rate, _, audio_type = asyncio.run(route._generate_tts_wav("hello", payload, None, "t"))
+ assert (wav, rate, audio_type) == (b"RIFFwav", 24000, "audio_vlm")
+ asyncio.run(route._generate_tts_wav("hello", payload, None, "t", voice = "male"))
+ assert seen[-1]["audio_options"] == {"voice": "male"}
+
+ for silent in (backend_for(False), backend_for(True, is_mlx = True)):
+ monkeypatch.setattr(route, "get_inference_backend", lambda silent = silent: silent)
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(route._generate_tts_wav("hello", payload, None, "t"))
+ assert exc.value.detail == "Active model is not an audio model."
+
+
[email protected]("window, expected", [(20, 13), (100_000, 2048), (None, 2048)])
+def test_unset_spoken_budget_fits_the_served_window(monkeypatch, window, expected):
+ import core.inference.runtime_context as rc
+
+ monkeypatch.setattr(rc, "runtime_context_length", lambda _m, *_a: window)
+ model = _Model()
+ b = _backend(model, _Processor(_Codec()))
+ list(
+ inf.InferenceBackend._speak_audio_reply(
+ b, [{"role": "user", "content": "Hi"}], "", np.zeros(1600, np.float32), None, None, None
+ )
+ )
+ assert model.kwargs["max_new_tokens"] == expected