Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Unsloth Studio: serve an MLX model through the library that loaded it (#14612)

Long Yixingf029025c85ec
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index aa1ec9f97a1..b4d8f782286 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -1335,6 +1335,17 @@ def _video_placeholder(processor):
return None
+def _mlx_lm_reads_images(model) -> bool:
+ """An image model unsloth-zoo loaded through mlx-lm, with an unsloth-zoo that prefills its images."""
+ if getattr(model, "_unsloth_supports_images", None) is not True:
+ return False
+ try:
+ from unsloth_zoo.mlx.generate import prefill_image_prompt # noqa: F401
+ except ImportError:
+ return False
+ return True
+
+
def _mlx_reads_video(processor) -> bool:
"""Processor carries a video component AND the template places a video marker: a render that
merely differs from the text one can still be prose, as in the image probe."""
@@ -3836,26 +3847,43 @@ class _TextBatchSession:
def admit(self, request, handle):
"""Take one reply in, answering with the text preceding its first token."""
backend = self.backend
+ image = request.get("image") if backend._reads_vision_input() else None
+ messages = request.get("messages") or []
+ row_kwargs = _row_kwargs(request)
plan = backend._plan_text_row(
- _messages_with_system_prompt(
- request.get("messages") or [],
- request.get("system_prompt", ""),
- ),
- **_row_kwargs(request),
- )
- prefix, remainder, cache, cache_key, all_tokens = backend._batch_prompt_row(
- plan.prompt,
- self._adapter_state,
+ _messages_with_system_prompt(messages, request.get("system_prompt", ""))
+ if image is None
+ else backend._conversation_to_render(messages, request.get("system_prompt", ""), image),
+ **row_kwargs,
)
+ began = time.perf_counter()
+ if image is None:
+ prefix, remainder, cache, cache_key, all_tokens = backend._batch_prompt_row(
+ plan.prompt,
+ self._adapter_state,
+ )
+ else:
+ from unsloth_zoo.mlx.generate import prefill_image_prompt
+
+ # mlx-lm batches token ids only: the image prefills here, and stays out of the token-keyed cache.
+ cache = backend._make_row_cache()
+ all_tokens = prefill_image_prompt(
+ backend._model, backend._tokenizer, plan.prompt, [image], cache
+ )
+ prefix, remainder, cache_key = all_tokens[:-1], all_tokens[-1:], None
+ if row_kwargs["max_new_tokens"] is None:
+ plan.max_tokens = backend._unset_generation_budget(plan.prompt, all_tokens)
row = _TextBatchRow(
handle = handle,
plan = plan,
cache_key = cache_key,
all_tokens = all_tokens,
- prompt_len = len(remainder),
- cached = len(prefix),
+ prompt_len = len(remainder) if image is None else len(all_tokens),
+ cached = len(prefix) if image is None else 0,
detokenizer = backend._tokenizer.detokenizer if plan.detokenizes else None,
)
+ if image is not None:
+ row.processing_at = began
uids = self.generator.insert(
[remainder],
max_tokens = [plan.max_tokens],
@@ -5015,13 +5043,12 @@ class MLXInferenceBackend:
reason,
)
model = tokenizer_or_processor = _eligibility = None
- if model is None:
- import gc
-
- # Outside the except: its traceback keeps the failed load's weights alive.
- gc.collect()
- _drain_generation_streams(mx)
- mx.clear_cache()
+ # unsloth-zoo builds an image checkpoint mlx-vlm lacks through mlx-lm: the text paths
+ # serve it, reading its images only through an unsloth-zoo that prefills them.
+ served_by_mlx_lm = use_vlm and getattr(model, "_is_vlm_model", None) is False
+ if served_by_mlx_lm and not _mlx_lm_reads_images(model):
+ is_vision = False
+ if model is None or served_by_mlx_lm:
if speculates:
speculates = False
spec_reason = (
@@ -5032,6 +5059,13 @@ class MLXInferenceBackend:
elif self._turboquant:
self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOAD
use_vlm = False
+ if model is None:
+ import gc
+
+ # Outside the except: its traceback keeps the failed load's weights alive.
+ gc.collect()
+ _drain_generation_streams(mx)
+ mx.clear_cache()
load_kwargs["text_only"] = True
model, tokenizer_or_processor = FastMLXModel.from_pretrained(
load_source,
@@ -5056,11 +5090,15 @@ class MLXInferenceBackend:
self._processor = None
self._is_vlm = False
+ _config_audio_type = getattr(config, "audio_type", None)
+ if served_by_mlx_lm and _config_audio_type == "audio_vlm":
+ # Omni audio input rides mlx-vlm's processor, which an mlx-lm load does not have.
+ _config_audio_type = None
_audio_type = _classify_mlx_audio_type(
model,
self._processor,
is_vision,
- config_audio_type = getattr(config, "audio_type", None),
+ config_audio_type = _config_audio_type,
)
_served_ctx, _native_ctx, _max_ctx = self._resolve_context_lengths(
self._model, max_seq_length
@@ -5330,7 +5368,10 @@ class MLXInferenceBackend:
)
if isinstance(_proc_tpl, (str, dict, list, tuple)) and _proc_tpl:
info["processor_template"] = _proc_tpl
- info["renders_image"] = _proc is not None and self._reads_vision_input()
+ # An image model served through mlx-lm marks images with its tokenizer's template.
+ info["renders_image"] = self._reads_vision_input() and (
+ _proc is not None or not getattr(self, "_is_vlm", False)
+ )
info["accepts_multiple_images"] = info["renders_image"] and bool(
getattr(self, "_multi_image_marker", None)
)
@@ -5453,7 +5494,7 @@ class MLXInferenceBackend:
def _conversation_to_render(self, messages, system_prompt, image):
"""The turns this request renders, whichever way its reply is decoded."""
- if self._is_vlm and image is not None:
+ if (self._is_vlm or self._reads_vision_input()) and image is not None:
from core.inference.chat_template_helpers import (
chat_render_target as _chat_render_target,
)
@@ -5637,6 +5678,7 @@ class MLXInferenceBackend:
max_new_tokens,
repetition_penalty,
cancel_event,
+ images = attached if self._reads_vision_input() else None,
tools = tools,
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
@@ -5761,6 +5803,7 @@ class MLXInferenceBackend:
repetition_penalty,
cancel_event,
*,
+ images = None,
tools = None,
enable_thinking = None,
reasoning_effort = None,
@@ -5916,13 +5959,23 @@ class MLXInferenceBackend:
self._int8_prefill_scope(),
self._generation_scopes(),
):
- (
- gen_prompt,
- prompt_cache,
- cache_key,
- prompt_tokens,
- cached_n,
- ) = self._prepare_prompt_cache(prompt, _adapter_state)
+ input_embeddings = None
+ if images:
+ from unsloth_zoo.mlx.generate import image_prompt_embeddings
+
+ # Any two images of one size expand to the same ids, so the token-keyed cache stays out.
+ prompt_tokens, input_embeddings = image_prompt_embeddings(
+ self._model, self._tokenizer, prompt, images
+ )
+ gen_prompt, prompt_cache, cache_key, cached_n = prompt_tokens, None, None, 0
+ else:
+ (
+ gen_prompt,
+ prompt_cache,
+ cache_key,
+ prompt_tokens,
+ cached_n,
+ ) = self._prepare_prompt_cache(prompt, _adapter_state)
max_new_tokens = self._generation_limit(prompt, max_new_tokens, prompt_tokens)
logger.info(
"Generating: prompt_len=%d, cached=%d, max_tokens=%d, model=%s, tokenizer=%s",
@@ -5947,6 +6000,8 @@ class MLXInferenceBackend:
gen_kwargs["prompt_cache"] = prompt_cache
else:
gen_kwargs.update(self._kv_quant_generate_kwargs())
+ if input_embeddings is not None:
+ gen_kwargs["input_embeddings"] = input_embeddings
if logits_processors is not None:
gen_kwargs["logits_processors"] = logits_processors
for response in stream_generate(
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 6df1c5af6fa..c86e18aa34e 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -903,6 +903,9 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
if k in _entry
}
)
+ # MLX serves a vision config mlx-lm built as text; the parent must stop accepting images.
+ if model_info["is_mlx"] and _entry.get("is_vision") is False:
+ model_info["is_vision"] = False
# Resolved MLX runtime knobs; only the backend knows what it honored.
model_info.update(
{
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index a1438b5dd58..a1ba059daca 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -2740,6 +2740,78 @@ def test_mlx_text_reuses_prompt_cache_on_the_next_turn(monkeypatch):
assert stats["usage"]["prompt_tokens"] == 7
+def _zoo_image_prefill(monkeypatch, **helpers):
+ zoo = types.ModuleType("unsloth_zoo.mlx.generate")
+ zoo.__dict__.update(helpers)
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.generate", zoo)
+
+
+def test_an_image_turn_through_mlx_lm_feeds_its_embeddings_outside_the_prompt_cache(monkeypatch):
+ """Two images of one size expand to the same ids, so an image turn neither reuses nor banks a cache."""
+ _install_fake_prompt_cache_api(monkeypatch)
+ captured, seen = [], []
+ token_map = {"P2": [1, 2, 3, 7, 8, 9, 10], "generated": [7, 8]}
+ backend = _install_fake_text_stack(monkeypatch, token_map, captured)
+ backend._reads_vision = True
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",
+ lambda _tok, messages, **_kw: messages[-1]["content"]
+ if isinstance(messages[-1]["content"], str)
+ else "P1 with an image",
+ )
+ _zoo_image_prefill(
+ monkeypatch,
+ image_prompt_embeddings = lambda _m, _t, prompt, images: (
+ seen.append((prompt, images)) or ([1, 2, 3], "embedded")
+ ),
+ )
+ image = object()
+ messages = [{"role": "user", "content": "P1"}]
+ list(backend.generate_chat_response(messages = messages, image = image, max_new_tokens = 4))
+ _run_turn(backend, "P2")
+ assert seen == [("P1 with an image", [image])]
+ assert captured[0]["prompt"] == [1, 2, 3] and captured[0]["input_embeddings"] == "embedded"
+ assert "prompt_cache" not in captured[0] and captured[1]["prompt"] == token_map["P2"]
+
+
+def test_an_image_row_through_mlx_lm_joins_the_batch_at_its_last_token(monkeypatch):
+ from core.inference import mlx_inference
+
+ inserted, rendered = [], []
+ generator = SimpleNamespace(
+ insert = lambda prompts, **kw: inserted.append((prompts, kw)) or [len(inserted)],
+ remove = lambda *_a, **_k: {},
+ close = lambda: None,
+ )
+ generate = types.ModuleType("mlx_lm.generate")
+ generate.BatchGenerator = lambda *_a, **_k: generator
+ _zoo_image_prefill(monkeypatch, prefill_image_prompt = lambda *_a: [1, 5, 5, 2])
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._model, backend._tokenizer = SimpleNamespace(), SimpleNamespace(eos_token_ids = [2])
+ backend._is_vlm, backend._reads_vision = False, True
+ backend._make_row_cache = lambda: ["cache"]
+ backend._batch_prompt_row = lambda prompt, _state: ([], [9], ["text cache"], "key", [9])
+ plan = SimpleNamespace(
+ prompt = "P", max_tokens = 4, sampler = None, processors = (), detokenizes = False, think_prefix = ""
+ )
+ backend._plan_text_row = lambda messages, **_kw: rendered.append(messages) or plan
+ with monkeypatch.context() as patch:
+ patch.setitem(sys.modules, "mlx_lm.generate", generate)
+ session = mlx_inference._TextBatchSession(backend, width = 2)
+ image, messages = object(), [{"role": "user", "content": "hi"}]
+ backend._unset_generation_budget = lambda prompt, tokens: 100 - len(tokens)
+ session.admit({"messages": messages}, 0)
+ session.admit({"messages": messages, "image": image, "max_new_tokens": None}, 1)
+ assert rendered[0] == messages and {"type": "image"} in rendered[1][-1]["content"]
+ assert [kw["max_tokens"] for _prompts, kw in inserted] == [[4], [96]]
+ assert [(prompts, kw["caches"], kw["all_tokens"]) for prompts, kw in inserted] == [
+ ([[9]], [["text cache"]], [[]]),
+ ([[2]], [["cache"]], [[1, 5, 5]]),
+ ]
+ assert session._rows[1].cache_key is None and session._rows[1].prompt_len == 4
+ session.close()
+
+
def test_mlx_text_without_lru_prompt_cache_prefills_the_full_prompt(monkeypatch):
from core.inference import mlx_inference
@@ -4329,6 +4401,68 @@ def test_a_text_model_mlx_vlm_cannot_load_is_served_without_turboquant(monkeypat
assert attempts == [(False, None)]
+def test_a_vision_checkpoint_served_through_mlx_lm_reads_images_only_where_zoo_prefills_them(
+ monkeypatch,
+):
+ """unsloth-zoo builds an image checkpoint mlx-vlm lacks (Muse Spark) through mlx-lm."""
+ pytest.importorskip("mlx.core")
+ from core.inference import mlx_inference
+
+ attempts, built_by_mlx_vlm, reads_images = [], [], []
+
+ class _Loader:
+ @staticmethod
+ def from_pretrained(name, **kwargs):
+ attempts.append(kwargs["text_only"])
+ model = SimpleNamespace(config = {}, _is_vlm_model = bool(built_by_mlx_vlm))
+ if reads_images:
+ model._unsloth_supports_images = True
+ return model, SimpleNamespace(tokenizer = "inner")
+
+ loader = types.ModuleType("unsloth_zoo.mlx.loader")
+ loader.FastMLXModel = _Loader
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)
+ monkeypatch.setattr(
+ mlx_inference, "_classify_mlx_audio_type", lambda *a, config_audio_type: config_audio_type
+ )
+ monkeypatch.setattr(
+ mlx_inference.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (2048, 2048, 2048)
+ )
+ vision = SimpleNamespace(
+ identifier = "org/vision", is_vision = True, is_lora = False, audio_type = "audio_vlm"
+ )
+
+ backend = mlx_inference.MLXInferenceBackend()
+ backend.load_model(vision, kv_quant = "tq-4")
+ assert attempts == [False]
+ entry = backend.models["org/vision"]
+ assert entry["audio_type"] is None and entry["has_audio_input"] is False
+ assert backend._is_vlm is False and backend._processor is None
+ assert backend._reads_vision_input() is False
+ assert backend.models["org/vision"]["is_vision"] is False
+ assert backend._turboquant_refusal == mlx_inference.MLX_TURBOQUANT_TEXT_LOAD
+
+ reads_images.append(True)
+ generate = types.ModuleType("unsloth_zoo.mlx.generate")
+ for zoo_prefills in (False, True):
+ if zoo_prefills:
+ generate.prefill_image_prompt = object()
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.generate", generate)
+ backend = mlx_inference.MLXInferenceBackend()
+ backend.load_model(vision, kv_quant = "tq-4")
+ assert backend._is_vlm is False and backend._reads_vision_input() is zoo_prefills
+ assert backend.models["org/vision"]["is_vision"] is zoo_prefills
+ assert backend.models["org/vision"]["audio_type"] is None
+ assert backend._turboquant_refusal == mlx_inference.MLX_TURBOQUANT_TEXT_LOAD
+
+ reads_images.clear()
+ built_by_mlx_vlm.append(True)
+ backend = mlx_inference.MLXInferenceBackend()
+ backend.load_model(vision)
+ assert backend._is_vlm is True and backend._reads_vision_input() is True
+ assert backend.models["org/vision"]["is_vision"] is True
+
+
def test_a_uniformly_quantized_text_load_goes_through_mlx_vlm_only_when_that_batches(monkeypatch):
pytest.importorskip("mlx.core")
from core.inference import mlx_inference
diff --git a/studio/backend/tests/test_pr7699_worker_audio_mirror.py b/studio/backend/tests/test_pr7699_worker_audio_mirror.py
index bc4e3bfbc75..1b8debb042d 100644
--- a/studio/backend/tests/test_pr7699_worker_audio_mirror.py
+++ b/studio/backend/tests/test_pr7699_worker_audio_mirror.py
@@ -161,3 +161,22 @@ def test_a_classified_load_failure_crosses_the_worker_and_the_parent(monkeypatch
loaded = _drive_handle_load(monkeypatch, mc, device = "cuda", load_model = refuse)
assert loaded["success"] is False and AudioCppFailure.from_payload(loaded["failure"]) == failure
assert failure_of(AudioCppFailureError(loaded["error"], failure)) is failure
+
+
+ "device, served_as_vision, published",
+ [("mlx", False, False), ("mlx", True, True), ("cuda", False, True)],
+)
+def test_an_mlx_vision_config_served_as_text_stops_accepting_images(
+ monkeypatch, device, served_as_vision, published
+):
+ """MLX serves a vision config mlx-lm built (Muse Spark) as text: the parent must not offer images."""
+ from core.inference.orchestrator import _mirrored_model_entry
+
+ mc = _mc(None, is_audio = False, has_audio_input = False)
+ mc.is_vision = True
+ entry = {"is_audio": False, "audio_type": None, "has_audio_input": False}
+ entry["is_vision"] = served_as_vision
+ info = _drive_handle_load(monkeypatch, mc, entry, device = device)
+ assert info["is_vision"] is published
+ assert _mirrored_model_entry(info, mc.identifier)["is_vision"] is published