Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: chat with LFM3-Audio-2B (text and speech in, text out) (#16243)

Leo Borcherdingd533017ea860
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 702c130a0da..1e51b84b0b6 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -756,7 +756,8 @@ def _supported_generate_kwargs(model, generation_kwargs: dict) -> dict:
if not isinstance(getattr(model, "_generation_fields_unsupported", None), dict):
return generation_kwargs
kept = {k: v for k, v in generation_kwargs.items() if k not in _CUSTOM_GENERATE_REJECTED_KWARGS}
- if getattr(getattr(model, "config", None), "model_type", None) == "lfm3_omni":
+ model_type = getattr(getattr(model, "config", None), "model_type", None)
+ if model_type in {"lfm3_omni", "lfm3_audio"}:
kept["generation_mode"] = "text"
events = [
c.ev
@@ -769,8 +770,9 @@ def _supported_generate_kwargs(model, generation_kwargs: dict) -> dict:
def _generate_supported(model, generation_kwargs: dict):
+ kwargs = _supported_generate_kwargs(model, generation_kwargs)
try:
- return model.generate(**_supported_generate_kwargs(model, generation_kwargs))
+ return model.generate(**kwargs)
except _GenerationHalted:
streamer = generation_kwargs.get("streamer")
matched = getattr(streamer, "matched", None)
@@ -788,6 +790,16 @@ def _refuse_without_generate(model, model_name: str) -> None:
)
+def _audio_chat_auto_model(config):
+ if not getattr(config, "has_audio_input", False) or getattr(config, "is_vision", False):
+ return None
+ if getattr(config, "audio_type", None) != "audio_vlm":
+ return None
+ import transformers
+
+ return getattr(transformers, "AutoModelForMultimodalLM", None)
+
+
def _processor_takes_text_first(processor) -> bool:
"""LFM3-Omni's processor is (text, audio, ..., images); the usual VLM order is (images, text)."""
# unsloth_zoo wraps __call__ as (*args, **kwargs); read the first class that names its arguments.
@@ -1419,6 +1431,36 @@ class InferenceBackend:
self.models[model_name]["tokenizer"] = processor
self.models[model_name]["processor"] = processor
+ elif _audio_chat_auto_model(config) is not None:
+ # Its text load builds the bare backbone; the multimodal class carries generate().
+ from unsloth import FastModel
+
+ model, processor = _load_with_offload_fallback(
+ FastModel.from_pretrained,
+ model_name = load_path,
+ auto_model = _audio_chat_auto_model(config),
+ max_seq_length = max_seq_length,
+ dtype = dtype,
+ **_load_in_4bit_kwargs(load_in_4bit),
+ device_map = device_map,
+ token = _hf_token_for_loader(hf_token),
+ trust_remote_code = trust_remote_code,
+ use_exact_model_name = use_exact_model_name,
+ **offload_kwargs,
+ )
+ FastModel.for_inference(model)
+ if not hasattr(processor, "feature_extractor"):
+ from transformers import AutoProcessor
+ processor = AutoProcessor.from_pretrained(
+ config.base_model if config.is_lora else load_path,
+ token = _hf_token_for_loader(hf_token),
+ trust_remote_code = trust_remote_code,
+ )
+ _refuse_without_generate(model, model_name)
+ self.models[model_name]["model"] = model
+ self.models[model_name]["tokenizer"] = processor
+ self.models[model_name]["processor"] = processor
+
else:
model, tokenizer = _load_with_offload_fallback(
FastLanguageModel.from_pretrained,
diff --git a/studio/backend/tests/test_audio_chat_load.py b/studio/backend/tests/test_audio_chat_load.py
new file mode 100644
index 00000000000..c749c77990b
--- /dev/null
+++ b/studio/backend/tests/test_audio_chat_load.py
@@ -0,0 +1,153 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+
+import importlib
+import sys
+import types
+from unittest.mock import MagicMock
+
+import pytest
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(name, attrs):
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception:
+ pass
+ _STUBBED.append(name)
+ mod = types.ModuleType(name)
+ mod.__spec__ = None
+ for attr in attrs:
+ setattr(mod, attr, MagicMock())
+ sys.modules[name] = mod
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, mod)
+
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+
+import core.inference.inference as inf # noqa: E402
+
+# Leaving these installed changes _shared_policy in later tests.
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+
+class _Lfm3AudioModel:
+ device = "cpu"
+
+
+class _Lfm3AudioForConditionalGeneration(_Lfm3AudioModel):
+ def generate(self, **kwargs):
+ return None
+
+
+class _Processor:
+ feature_extractor = object()
+ tokenizer = object()
+
+
+def _config(**overrides):
+ fields = dict(
+ identifier = "LiquidAI/LFM3-Audio-2B",
+ path = "LiquidAI/LFM3-Audio-2B",
+ is_vision = False,
+ is_lora = False,
+ is_audio = False,
+ audio_type = "audio_vlm",
+ has_audio_input = True,
+ is_local = False,
+ base_model = None,
+ gguf_variant = None,
+ )
+ fields.update(overrides)
+ return types.SimpleNamespace(**fields)
+
+
+def multimodal_auto_class(monkeypatch):
+ import transformers
+
+ auto_class = type("AutoModelForMultimodalLM", (), {})
+ monkeypatch.setattr(transformers, "AutoModelForMultimodalLM", auto_class, raising = False)
+ return auto_class
+
+
+def backend(monkeypatch, multimodal_auto_class):
+ import utils.models.diffusion_lm as diffusion_lm
+
+ unsloth = types.ModuleType("unsloth")
+ monkeypatch.setitem(sys.modules, "unsloth", unsloth)
+
+ calls = {}
+
+ def fast_model(**kwargs):
+ calls["fast_model"] = kwargs
+ return _Lfm3AudioForConditionalGeneration(), _Processor()
+
+ def fast_language_model(**kwargs):
+ calls["fast_language_model"] = kwargs
+ return _Lfm3AudioModel(), object()
+
+ noop = lambda *a, **k: None
+ unsloth.FastModel = types.SimpleNamespace(from_pretrained = fast_model, for_inference = noop)
+ monkeypatch.setattr(
+ inf,
+ "FastLanguageModel",
+ types.SimpleNamespace(from_pretrained = fast_language_model, for_inference = noop),
+ )
+ monkeypatch.setattr(diffusion_lm, "diffusion_lm_profile_for_model", lambda *a, **k: None)
+ monkeypatch.setattr(diffusion_lm, "loaded_diffusion_profile", lambda *a, **k: None)
+ monkeypatch.setattr(inf, "get_device_map", lambda *a, **k: "sequential")
+ monkeypatch.setattr(inf, "get_visible_gpu_count", lambda: 1)
+ monkeypatch.setattr(inf, "log_gpu_memory", noop)
+ monkeypatch.setattr(inf, "_exact_model_name_for_load", lambda *a, **k: None)
+ monkeypatch.setattr(inf, "_offload_kwargs_for_fit", lambda *a, **k: {})
+ monkeypatch.setattr(inf, "raise_if_offloaded", noop)
+ monkeypatch.setattr(inf, "runtime_context_length", lambda *a, **k: 4096)
+
+ b = inf.InferenceBackend.__new__(inf.InferenceBackend)
+ b.models, b.loading_models, b.active_model_name = {}, set(), None
+ b._resolve_chat_eos = noop
+ b._load_chat_template_info = noop
+ return b, calls
+
+
+def test_audio_input_chat_loads_through_the_multimodal_class(backend, multimodal_auto_class):
+ b, calls = backend
+ assert inf.InferenceBackend.load_model(b, _config(), trust_remote_code = True) is True
+ assert calls["fast_model"]["auto_model"] is multimodal_auto_class
+ assert "fast_language_model" not in calls
+ loaded = b.models["LiquidAI/LFM3-Audio-2B"]
+ assert isinstance(loaded["processor"], _Processor)
+ assert loaded["has_audio_input"] is True
+
+
+def test_a_plain_text_model_still_takes_the_text_load(backend):
+ b, calls = backend
+ config = _config(audio_type = None, has_audio_input = False)
+ with pytest.raises(Exception, match = "cannot generate text"):
+ inf.InferenceBackend.load_model(b, config)
+ assert "fast_model" not in calls and "fast_language_model" in calls
+
+
+def test_audio_chat_auto_model_needs_audio_vlm_without_vision(multimodal_auto_class):
+ assert inf._audio_chat_auto_model(_config()) is multimodal_auto_class
+ assert inf._audio_chat_auto_model(_config(is_vision = True)) is None
+ assert inf._audio_chat_auto_model(_config(audio_type = "whisper")) is None
+ assert inf._audio_chat_auto_model(_config(has_audio_input = False, audio_type = None)) is None
+
+
+def test_an_unavailable_multimodal_api_keeps_the_fallback(monkeypatch):
+ import transformers
+ monkeypatch.setattr(transformers, "AutoModelForMultimodalLM", None, raising = False)
+ assert inf._audio_chat_auto_model(_config()) is None
diff --git a/studio/backend/tests/test_audio_token_detection.py b/studio/backend/tests/test_audio_token_detection.py
index ec4d50368c4..00f699917ca 100644
--- a/studio/backend/tests/test_audio_token_detection.py
+++ b/studio/backend/tests/test_audio_token_detection.py
@@ -661,12 +661,18 @@ def test_lfm3_omni_on_the_hub_is_audio_vlm(monkeypatch):
assert (audio_type, definitive) == ("audio_vlm", True)
-def test_a_speech_only_processor_is_not_audio_vlm():
+def test_an_audio_feature_extractor_processor_is_audio_vlm():
from utils.audio_tokens import classify_audio_tokens
- assert classify_audio_tokens(_LFM3_AUDIO_PROCESSOR_CONFIG) is None
+ assert classify_audio_tokens(_LFM3_AUDIO_PROCESSOR_CONFIG) == "audio_vlm"
-def test_lfm3_audio_on_the_hub_is_not_routed_as_audio_vlm(monkeypatch):
+def test_an_audio_token_with_no_input_processor_is_not_audio_vlm():
+ from utils.audio_tokens import classify_audio_tokens
+ bare = {k: v for k, v in _LFM3_AUDIO_PROCESSOR_CONFIG.items() if k != "feature_extractor"}
+ assert classify_audio_tokens(bare) is None
+
+
+def test_lfm3_audio_on_the_hub_is_audio_vlm(monkeypatch):
audio_type, definitive = _detect_checked(
monkeypatch,
[
@@ -676,4 +682,4 @@ def test_lfm3_audio_on_the_hub_is_not_routed_as_audio_vlm(monkeypatch):
],
model = "LiquidAI/LFM3-Audio-2B",
)
- assert audio_type is None
+ assert (audio_type, definitive) == ("audio_vlm", True)
diff --git a/studio/backend/tests/test_custom_generate_kwargs.py b/studio/backend/tests/test_custom_generate_kwargs.py
index 6ff7a1567ea..3cecb030641 100644
--- a/studio/backend/tests/test_custom_generate_kwargs.py
+++ b/studio/backend/tests/test_custom_generate_kwargs.py
@@ -1,14 +1,13 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""LFM3-Omni's custom generate raises on logits_processor and stopping_criteria (#16054)."""
-
import ast
import inspect
import types
from pathlib import Path
import pytest
+import torch
# Loaded from source: importing core.inference.inference needs unsloth, which the backend matrix lacks.
_SRC = Path(__file__).resolve().parents[1] / "core" / "inference" / "inference.py"
@@ -20,8 +19,9 @@ _WANTED = {
"_HaltingStreamer",
"_generate_supported",
"_refuse_without_generate",
+ "_think_tag_ids",
}
-_NS = {"inspect": inspect}
+_NS = {"inspect": inspect, "torch": torch}
exec(
compile(
ast.Module(
@@ -57,9 +57,10 @@ def test_regular_model_keeps_all_kwargs():
assert _supported_generate_kwargs(types.SimpleNamespace(), _KWARGS) is _KWARGS
-def test_omni_chat_requests_text_instead_of_unstreamed_speech():
[email protected]("model_type", ["lfm3_omni", "lfm3_audio"])
+def test_audio_chat_requests_text_instead_of_unstreamed_speech(model_type):
model = types.SimpleNamespace(
- config = types.SimpleNamespace(model_type = "lfm3_omni"),
+ config = types.SimpleNamespace(model_type = model_type),
_generation_fields_unsupported = {},
)
assert _supported_generate_kwargs(model, _KWARGS)["generation_mode"] == "text"
@@ -113,8 +114,6 @@ class _Recorder:
class _CustomGenerate:
- """LFM3-style: rejects stopping_criteria, streams each token, ends its streamers on any exit."""
-
_generation_fields_unsupported = {"cache_implementation": None}
def __init__(self, on_step = None):
@@ -245,3 +244,96 @@ def test_a_text_load_without_generate_is_refused_by_name():
with pytest.raises(ValueError, match = "Lfm3AudioModel, which cannot generate text"):
_refuse_without_generate(Lfm3AudioModel(), "LiquidAI/LFM3-Audio-2B")
_refuse_without_generate(types.SimpleNamespace(generate = lambda **kw: None), "any/causal-lm")
+
+
+_OPEN, _CLOSE, _EOS = 124901, 124902, 124900
+_THINK_TOKENIZER = types.SimpleNamespace(
+ unk_token_id = 0,
+ convert_tokens_to_ids = lambda t: {"<think>": _OPEN, "</think>": _CLOSE}.get(t, 0),
+)
+
+
+class _ScriptedGenerate:
+ _generation_fields_unsupported = {"cache_implementation": None}
+
+ def __init__(
+ self,
+ reply,
+ model_type = "lfm3_audio",
+ ):
+ self.reply, self.steps = reply, 0
+ self.config = types.SimpleNamespace(model_type = model_type)
+
+ def generate(self, *, input_ids, streamer, **kwargs):
+ try:
+ streamer.put(input_ids)
+ for token in self.reply:
+ self.steps += 1
+ streamer.put(torch.tensor([[token]]))
+ finally:
+ streamer.end()
+ return torch.cat((input_ids, torch.tensor([self.reply])), 1)
+
+
+def _scripted(
+ reply,
+ model_type = "lfm3_audio",
+ **extra,
+):
+ model, streamer = _ScriptedGenerate(reply, model_type), _Recorder()
+ kwargs = {
+ "input_ids": torch.tensor([[1, 2]]),
+ "streamer": streamer,
+ "eos_token_id": [_EOS],
+ **extra,
+ }
+ if "tokenizer" in inspect.signature(_generate_supported).parameters:
+ out = _generate_supported(model, kwargs, _THINK_TOKENIZER)
+ else:
+ out = _generate_supported(model, kwargs)
+ return model, [int(t) for t in streamer.tokens[1:]], out
+
+
[email protected]("model_type", ["lfm3_audio", "lfm3_omni", "other_custom_model"])
+def test_literal_think_close_in_an_answer_is_not_a_stop(model_type):
+ import threading
+
+ # LiquidAI/LFM3-Audio-2B: "<think>I should explain the tag.</think>The literal closing tag </think> ends a thinking block."
+ reply = [
+ _OPEN,
+ 49,
+ 1252,
+ 6986,
+ 278,
+ 10194,
+ 22,
+ _CLOSE,
+ 597,
+ 30020,
+ 23189,
+ 10194,
+ 229,
+ _CLOSE,
+ 11676,
+ 267,
+ 6416,
+ 3908,
+ 22,
+ _EOS,
+ ]
+ model, streamed, out = _scripted(
+ reply,
+ model_type = model_type,
+ stopping_criteria = _criteria(threading.Event()),
+ )
+ assert streamed == reply and out.tolist() == [[1, 2, *reply]]
+ assert model.steps == len(reply)
+
+
+def test_cancel_still_returns_no_sequences_with_a_tensor_stream():
+ import threading
+
+ cancel = threading.Event()
+ cancel.set()
+ _, streamed, out = _scripted([_OPEN, 10, _CLOSE, 11, _EOS], stopping_criteria = _criteria(cancel))
+ assert out is None and streamed == []
diff --git a/studio/backend/utils/audio_tokens.py b/studio/backend/utils/audio_tokens.py
index 00cb6211893..79eb6327e8a 100644
--- a/studio/backend/utils/audio_tokens.py
+++ b/studio/backend/utils/audio_tokens.py
@@ -99,9 +99,10 @@ def classify_audio_tokens(tok_config: dict) -> Optional[str]:
"""The audio_type a parsed tokenizer_config fingerprints, or None."""
added = tok_config.get("added_tokens_decoder") or {}
token_contents = [value.get("content", "") for value in added.values()]
- # Only beside an image processor (LFM3-Omni): audio-input chat loads as a VLM, and
- # speech-only LFM3-Audio has no such load.
- if isinstance(tok_config.get("audio_token"), str) and tok_config.get("image_processor"):
+ # Only beside a processor that takes input: LFM3-Omni's image processor, LFM3-Audio's feature extractor.
+ if isinstance(tok_config.get("audio_token"), str) and (
+ tok_config.get("image_processor") or tok_config.get("feature_extractor")
+ ):
token_contents.append(tok_config["audio_token"])
if not token_contents:
return None