Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: chat with LFM3-Audio-2B (text and speech in, text out) (#16243)
Leo Borcherdingd533017ea860
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 702c130a0da..1e51b84b0b6 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -756,7 +756,8 @@ def _supported_generate_kwargs(model, generation_kwargs: dict) -> dict:if not isinstance(getattr(model, "_generation_fields_unsupported", None), dict):return generation_kwargskept = {k: v for k, v in generation_kwargs.items() if k not in _CUSTOM_GENERATE_REJECTED_KWARGS}- if getattr(getattr(model, "config", None), "model_type", None) == "lfm3_omni":+ model_type = getattr(getattr(model, "config", None), "model_type", None)+ if model_type in {"lfm3_omni", "lfm3_audio"}:kept["generation_mode"] = "text"events = [c.ev@@ -769,8 +770,9 @@ def _supported_generate_kwargs(model, generation_kwargs: dict) -> dict:def _generate_supported(model, generation_kwargs: dict):+ kwargs = _supported_generate_kwargs(model, generation_kwargs)try:- return model.generate(**_supported_generate_kwargs(model, generation_kwargs))+ return model.generate(**kwargs)except _GenerationHalted:streamer = generation_kwargs.get("streamer")matched = getattr(streamer, "matched", None)@@ -788,6 +790,16 @@ def _refuse_without_generate(model, model_name: str) -> None:)+def _audio_chat_auto_model(config):+ if not getattr(config, "has_audio_input", False) or getattr(config, "is_vision", False):+ return None+ if getattr(config, "audio_type", None) != "audio_vlm":+ return None+ import transformers++ return getattr(transformers, "AutoModelForMultimodalLM", None)++def _processor_takes_text_first(processor) -> bool:"""LFM3-Omni's processor is (text, audio, ..., images); the usual VLM order is (images, text)."""# unsloth_zoo wraps __call__ as (*args, **kwargs); read the first class that names its arguments.@@ -1419,6 +1431,36 @@ class InferenceBackend:self.models[model_name]["tokenizer"] = processorself.models[model_name]["processor"] = processor+ elif _audio_chat_auto_model(config) is not None:+ # Its text load builds the bare backbone; the multimodal class carries generate().+ from unsloth import FastModel++ model, processor = _load_with_offload_fallback(+ FastModel.from_pretrained,+ model_name = load_path,+ auto_model = _audio_chat_auto_model(config),+ max_seq_length = max_seq_length,+ dtype = dtype,+ **_load_in_4bit_kwargs(load_in_4bit),+ device_map = device_map,+ token = _hf_token_for_loader(hf_token),+ trust_remote_code = trust_remote_code,+ use_exact_model_name = use_exact_model_name,+ **offload_kwargs,+ )+ FastModel.for_inference(model)+ if not hasattr(processor, "feature_extractor"):+ from transformers import AutoProcessor+ processor = AutoProcessor.from_pretrained(+ config.base_model if config.is_lora else load_path,+ token = _hf_token_for_loader(hf_token),+ trust_remote_code = trust_remote_code,+ )+ _refuse_without_generate(model, model_name)+ self.models[model_name]["model"] = model+ self.models[model_name]["tokenizer"] = processor+ self.models[model_name]["processor"] = processor+else:model, tokenizer = _load_with_offload_fallback(FastLanguageModel.from_pretrained,diff --git a/studio/backend/tests/test_audio_chat_load.py b/studio/backend/tests/test_audio_chat_load.pynew file mode 100644index 00000000000..c749c77990b--- /dev/null+++ b/studio/backend/tests/test_audio_chat_load.py@@ -0,0 +1,153 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+++import importlib+import sys+import types+from unittest.mock import MagicMock++import pytest++_STUBBED: list[str] = []+++def _stub_if_missing(name, attrs):+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception:+ pass+ _STUBBED.append(name)+ mod = types.ModuleType(name)+ mod.__spec__ = None+ for attr in attrs:+ setattr(mod, attr, MagicMock())+ sys.modules[name] = mod+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, mod)+++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))++import core.inference.inference as inf # noqa: E402++# Leaving these installed changes _shared_policy in later tests.+for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)+++class _Lfm3AudioModel:+ device = "cpu"+++class _Lfm3AudioForConditionalGeneration(_Lfm3AudioModel):+ def generate(self, **kwargs):+ return None+++class _Processor:+ feature_extractor = object()+ tokenizer = object()+++def _config(**overrides):+ fields = dict(+ identifier = "LiquidAI/LFM3-Audio-2B",+ path = "LiquidAI/LFM3-Audio-2B",+ is_vision = False,+ is_lora = False,+ is_audio = False,+ audio_type = "audio_vlm",+ has_audio_input = True,+ is_local = False,+ base_model = None,+ gguf_variant = None,+ )+ fields.update(overrides)+ return types.SimpleNamespace(**fields)+++def multimodal_auto_class(monkeypatch):+ import transformers++ auto_class = type("AutoModelForMultimodalLM", (), {})+ monkeypatch.setattr(transformers, "AutoModelForMultimodalLM", auto_class, raising = False)+ return auto_class+++def backend(monkeypatch, multimodal_auto_class):+ import utils.models.diffusion_lm as diffusion_lm++ unsloth = types.ModuleType("unsloth")+ monkeypatch.setitem(sys.modules, "unsloth", unsloth)++ calls = {}++ def fast_model(**kwargs):+ calls["fast_model"] = kwargs+ return _Lfm3AudioForConditionalGeneration(), _Processor()++ def fast_language_model(**kwargs):+ calls["fast_language_model"] = kwargs+ return _Lfm3AudioModel(), object()++ noop = lambda *a, **k: None+ unsloth.FastModel = types.SimpleNamespace(from_pretrained = fast_model, for_inference = noop)+ monkeypatch.setattr(+ inf,+ "FastLanguageModel",+ types.SimpleNamespace(from_pretrained = fast_language_model, for_inference = noop),+ )+ monkeypatch.setattr(diffusion_lm, "diffusion_lm_profile_for_model", lambda *a, **k: None)+ monkeypatch.setattr(diffusion_lm, "loaded_diffusion_profile", lambda *a, **k: None)+ monkeypatch.setattr(inf, "get_device_map", lambda *a, **k: "sequential")+ monkeypatch.setattr(inf, "get_visible_gpu_count", lambda: 1)+ monkeypatch.setattr(inf, "log_gpu_memory", noop)+ monkeypatch.setattr(inf, "_exact_model_name_for_load", lambda *a, **k: None)+ monkeypatch.setattr(inf, "_offload_kwargs_for_fit", lambda *a, **k: {})+ monkeypatch.setattr(inf, "raise_if_offloaded", noop)+ monkeypatch.setattr(inf, "runtime_context_length", lambda *a, **k: 4096)++ b = inf.InferenceBackend.__new__(inf.InferenceBackend)+ b.models, b.loading_models, b.active_model_name = {}, set(), None+ b._resolve_chat_eos = noop+ b._load_chat_template_info = noop+ return b, calls+++def test_audio_input_chat_loads_through_the_multimodal_class(backend, multimodal_auto_class):+ b, calls = backend+ assert inf.InferenceBackend.load_model(b, _config(), trust_remote_code = True) is True+ assert calls["fast_model"]["auto_model"] is multimodal_auto_class+ assert "fast_language_model" not in calls+ loaded = b.models["LiquidAI/LFM3-Audio-2B"]+ assert isinstance(loaded["processor"], _Processor)+ assert loaded["has_audio_input"] is True+++def test_a_plain_text_model_still_takes_the_text_load(backend):+ b, calls = backend+ config = _config(audio_type = None, has_audio_input = False)+ with pytest.raises(Exception, match = "cannot generate text"):+ inf.InferenceBackend.load_model(b, config)+ assert "fast_model" not in calls and "fast_language_model" in calls+++def test_audio_chat_auto_model_needs_audio_vlm_without_vision(multimodal_auto_class):+ assert inf._audio_chat_auto_model(_config()) is multimodal_auto_class+ assert inf._audio_chat_auto_model(_config(is_vision = True)) is None+ assert inf._audio_chat_auto_model(_config(audio_type = "whisper")) is None+ assert inf._audio_chat_auto_model(_config(has_audio_input = False, audio_type = None)) is None+++def test_an_unavailable_multimodal_api_keeps_the_fallback(monkeypatch):+ import transformers+ monkeypatch.setattr(transformers, "AutoModelForMultimodalLM", None, raising = False)+ assert inf._audio_chat_auto_model(_config()) is Nonediff --git a/studio/backend/tests/test_audio_token_detection.py b/studio/backend/tests/test_audio_token_detection.pyindex ec4d50368c4..00f699917ca 100644--- a/studio/backend/tests/test_audio_token_detection.py+++ b/studio/backend/tests/test_audio_token_detection.py@@ -661,12 +661,18 @@ def test_lfm3_omni_on_the_hub_is_audio_vlm(monkeypatch):assert (audio_type, definitive) == ("audio_vlm", True)-def test_a_speech_only_processor_is_not_audio_vlm():+def test_an_audio_feature_extractor_processor_is_audio_vlm():from utils.audio_tokens import classify_audio_tokens- assert classify_audio_tokens(_LFM3_AUDIO_PROCESSOR_CONFIG) is None+ assert classify_audio_tokens(_LFM3_AUDIO_PROCESSOR_CONFIG) == "audio_vlm"-def test_lfm3_audio_on_the_hub_is_not_routed_as_audio_vlm(monkeypatch):+def test_an_audio_token_with_no_input_processor_is_not_audio_vlm():+ from utils.audio_tokens import classify_audio_tokens+ bare = {k: v for k, v in _LFM3_AUDIO_PROCESSOR_CONFIG.items() if k != "feature_extractor"}+ assert classify_audio_tokens(bare) is None+++def test_lfm3_audio_on_the_hub_is_audio_vlm(monkeypatch):audio_type, definitive = _detect_checked(monkeypatch,[@@ -676,4 +682,4 @@ def test_lfm3_audio_on_the_hub_is_not_routed_as_audio_vlm(monkeypatch):],model = "LiquidAI/LFM3-Audio-2B",)- assert audio_type is None+ assert (audio_type, definitive) == ("audio_vlm", True)diff --git a/studio/backend/tests/test_custom_generate_kwargs.py b/studio/backend/tests/test_custom_generate_kwargs.pyindex 6ff7a1567ea..3cecb030641 100644--- a/studio/backend/tests/test_custom_generate_kwargs.py+++ b/studio/backend/tests/test_custom_generate_kwargs.py@@ -1,14 +1,13 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""LFM3-Omni's custom generate raises on logits_processor and stopping_criteria (#16054)."""-import astimport inspectimport typesfrom pathlib import Pathimport pytest+import torch# Loaded from source: importing core.inference.inference needs unsloth, which the backend matrix lacks._SRC = Path(__file__).resolve().parents[1] / "core" / "inference" / "inference.py"@@ -20,8 +19,9 @@ _WANTED = {"_HaltingStreamer","_generate_supported","_refuse_without_generate",+ "_think_tag_ids",}-_NS = {"inspect": inspect}+_NS = {"inspect": inspect, "torch": torch}exec(compile(ast.Module(@@ -57,9 +57,10 @@ def test_regular_model_keeps_all_kwargs():assert _supported_generate_kwargs(types.SimpleNamespace(), _KWARGS) is _KWARGS-def test_omni_chat_requests_text_instead_of_unstreamed_speech():[email protected]("model_type", ["lfm3_omni", "lfm3_audio"])+def test_audio_chat_requests_text_instead_of_unstreamed_speech(model_type):model = types.SimpleNamespace(- config = types.SimpleNamespace(model_type = "lfm3_omni"),+ config = types.SimpleNamespace(model_type = model_type),_generation_fields_unsupported = {},)assert _supported_generate_kwargs(model, _KWARGS)["generation_mode"] == "text"@@ -113,8 +114,6 @@ class _Recorder:class _CustomGenerate:- """LFM3-style: rejects stopping_criteria, streams each token, ends its streamers on any exit."""-_generation_fields_unsupported = {"cache_implementation": None}def __init__(self, on_step = None):@@ -245,3 +244,96 @@ def test_a_text_load_without_generate_is_refused_by_name():with pytest.raises(ValueError, match = "Lfm3AudioModel, which cannot generate text"):_refuse_without_generate(Lfm3AudioModel(), "LiquidAI/LFM3-Audio-2B")_refuse_without_generate(types.SimpleNamespace(generate = lambda **kw: None), "any/causal-lm")+++_OPEN, _CLOSE, _EOS = 124901, 124902, 124900+_THINK_TOKENIZER = types.SimpleNamespace(+ unk_token_id = 0,+ convert_tokens_to_ids = lambda t: {"<think>": _OPEN, "</think>": _CLOSE}.get(t, 0),+)+++class _ScriptedGenerate:+ _generation_fields_unsupported = {"cache_implementation": None}++ def __init__(+ self,+ reply,+ model_type = "lfm3_audio",+ ):+ self.reply, self.steps = reply, 0+ self.config = types.SimpleNamespace(model_type = model_type)++ def generate(self, *, input_ids, streamer, **kwargs):+ try:+ streamer.put(input_ids)+ for token in self.reply:+ self.steps += 1+ streamer.put(torch.tensor([[token]]))+ finally:+ streamer.end()+ return torch.cat((input_ids, torch.tensor([self.reply])), 1)+++def _scripted(+ reply,+ model_type = "lfm3_audio",+ **extra,+):+ model, streamer = _ScriptedGenerate(reply, model_type), _Recorder()+ kwargs = {+ "input_ids": torch.tensor([[1, 2]]),+ "streamer": streamer,+ "eos_token_id": [_EOS],+ **extra,+ }+ if "tokenizer" in inspect.signature(_generate_supported).parameters:+ out = _generate_supported(model, kwargs, _THINK_TOKENIZER)+ else:+ out = _generate_supported(model, kwargs)+ return model, [int(t) for t in streamer.tokens[1:]], out++[email protected]("model_type", ["lfm3_audio", "lfm3_omni", "other_custom_model"])+def test_literal_think_close_in_an_answer_is_not_a_stop(model_type):+ import threading++ # LiquidAI/LFM3-Audio-2B: "<think>I should explain the tag.</think>The literal closing tag </think> ends a thinking block."+ reply = [+ _OPEN,+ 49,+ 1252,+ 6986,+ 278,+ 10194,+ 22,+ _CLOSE,+ 597,+ 30020,+ 23189,+ 10194,+ 229,+ _CLOSE,+ 11676,+ 267,+ 6416,+ 3908,+ 22,+ _EOS,+ ]+ model, streamed, out = _scripted(+ reply,+ model_type = model_type,+ stopping_criteria = _criteria(threading.Event()),+ )+ assert streamed == reply and out.tolist() == [[1, 2, *reply]]+ assert model.steps == len(reply)+++def test_cancel_still_returns_no_sequences_with_a_tensor_stream():+ import threading++ cancel = threading.Event()+ cancel.set()+ _, streamed, out = _scripted([_OPEN, 10, _CLOSE, 11, _EOS], stopping_criteria = _criteria(cancel))+ assert out is None and streamed == []diff --git a/studio/backend/utils/audio_tokens.py b/studio/backend/utils/audio_tokens.pyindex 00cb6211893..79eb6327e8a 100644--- a/studio/backend/utils/audio_tokens.py+++ b/studio/backend/utils/audio_tokens.py@@ -99,9 +99,10 @@ def classify_audio_tokens(tok_config: dict) -> Optional[str]:"""The audio_type a parsed tokenizer_config fingerprints, or None."""added = tok_config.get("added_tokens_decoder") or {}token_contents = [value.get("content", "") for value in added.values()]- # Only beside an image processor (LFM3-Omni): audio-input chat loads as a VLM, and- # speech-only LFM3-Audio has no such load.- if isinstance(tok_config.get("audio_token"), str) and tok_config.get("image_processor"):+ # Only beside a processor that takes input: LFM3-Omni's image processor, LFM3-Audio's feature extractor.+ if isinstance(tok_config.get("audio_token"), str) and (+ tok_config.get("image_processor") or tok_config.get("feature_extractor")+ ):token_contents.append(tok_config["audio_token"])if not token_contents:return None