Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio: serve an MLX model through the library that loaded it (#14612)
Long Yixingf029025c85ec
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex aa1ec9f97a1..b4d8f782286 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -1335,6 +1335,17 @@ def _video_placeholder(processor):return None+def _mlx_lm_reads_images(model) -> bool:+ """An image model unsloth-zoo loaded through mlx-lm, with an unsloth-zoo that prefills its images."""+ if getattr(model, "_unsloth_supports_images", None) is not True:+ return False+ try:+ from unsloth_zoo.mlx.generate import prefill_image_prompt # noqa: F401+ except ImportError:+ return False+ return True++def _mlx_reads_video(processor) -> bool:"""Processor carries a video component AND the template places a video marker: a render thatmerely differs from the text one can still be prose, as in the image probe."""@@ -3836,26 +3847,43 @@ class _TextBatchSession:def admit(self, request, handle):"""Take one reply in, answering with the text preceding its first token."""backend = self.backend+ image = request.get("image") if backend._reads_vision_input() else None+ messages = request.get("messages") or []+ row_kwargs = _row_kwargs(request)plan = backend._plan_text_row(- _messages_with_system_prompt(- request.get("messages") or [],- request.get("system_prompt", ""),- ),- **_row_kwargs(request),- )- prefix, remainder, cache, cache_key, all_tokens = backend._batch_prompt_row(- plan.prompt,- self._adapter_state,+ _messages_with_system_prompt(messages, request.get("system_prompt", ""))+ if image is None+ else backend._conversation_to_render(messages, request.get("system_prompt", ""), image),+ **row_kwargs,)+ began = time.perf_counter()+ if image is None:+ prefix, remainder, cache, cache_key, all_tokens = backend._batch_prompt_row(+ plan.prompt,+ self._adapter_state,+ )+ else:+ from unsloth_zoo.mlx.generate import prefill_image_prompt++ # mlx-lm batches token ids only: the image prefills here, and stays out of the token-keyed cache.+ cache = backend._make_row_cache()+ all_tokens = prefill_image_prompt(+ backend._model, backend._tokenizer, plan.prompt, [image], cache+ )+ prefix, remainder, cache_key = all_tokens[:-1], all_tokens[-1:], None+ if row_kwargs["max_new_tokens"] is None:+ plan.max_tokens = backend._unset_generation_budget(plan.prompt, all_tokens)row = _TextBatchRow(handle = handle,plan = plan,cache_key = cache_key,all_tokens = all_tokens,- prompt_len = len(remainder),- cached = len(prefix),+ prompt_len = len(remainder) if image is None else len(all_tokens),+ cached = len(prefix) if image is None else 0,detokenizer = backend._tokenizer.detokenizer if plan.detokenizes else None,)+ if image is not None:+ row.processing_at = beganuids = self.generator.insert([remainder],max_tokens = [plan.max_tokens],@@ -5015,13 +5043,12 @@ class MLXInferenceBackend:reason,)model = tokenizer_or_processor = _eligibility = None- if model is None:- import gc-- # Outside the except: its traceback keeps the failed load's weights alive.- gc.collect()- _drain_generation_streams(mx)- mx.clear_cache()+ # unsloth-zoo builds an image checkpoint mlx-vlm lacks through mlx-lm: the text paths+ # serve it, reading its images only through an unsloth-zoo that prefills them.+ served_by_mlx_lm = use_vlm and getattr(model, "_is_vlm_model", None) is False+ if served_by_mlx_lm and not _mlx_lm_reads_images(model):+ is_vision = False+ if model is None or served_by_mlx_lm:if speculates:speculates = Falsespec_reason = (@@ -5032,6 +5059,13 @@ class MLXInferenceBackend:elif self._turboquant:self._turboquant_refusal = MLX_TURBOQUANT_TEXT_LOADuse_vlm = False+ if model is None:+ import gc++ # Outside the except: its traceback keeps the failed load's weights alive.+ gc.collect()+ _drain_generation_streams(mx)+ mx.clear_cache()load_kwargs["text_only"] = Truemodel, tokenizer_or_processor = FastMLXModel.from_pretrained(load_source,@@ -5056,11 +5090,15 @@ class MLXInferenceBackend:self._processor = Noneself._is_vlm = False+ _config_audio_type = getattr(config, "audio_type", None)+ if served_by_mlx_lm and _config_audio_type == "audio_vlm":+ # Omni audio input rides mlx-vlm's processor, which an mlx-lm load does not have.+ _config_audio_type = None_audio_type = _classify_mlx_audio_type(model,self._processor,is_vision,- config_audio_type = getattr(config, "audio_type", None),+ config_audio_type = _config_audio_type,)_served_ctx, _native_ctx, _max_ctx = self._resolve_context_lengths(self._model, max_seq_length@@ -5330,7 +5368,10 @@ class MLXInferenceBackend:)if isinstance(_proc_tpl, (str, dict, list, tuple)) and _proc_tpl:info["processor_template"] = _proc_tpl- info["renders_image"] = _proc is not None and self._reads_vision_input()+ # An image model served through mlx-lm marks images with its tokenizer's template.+ info["renders_image"] = self._reads_vision_input() and (+ _proc is not None or not getattr(self, "_is_vlm", False)+ )info["accepts_multiple_images"] = info["renders_image"] and bool(getattr(self, "_multi_image_marker", None))@@ -5453,7 +5494,7 @@ class MLXInferenceBackend:def _conversation_to_render(self, messages, system_prompt, image):"""The turns this request renders, whichever way its reply is decoded."""- if self._is_vlm and image is not None:+ if (self._is_vlm or self._reads_vision_input()) and image is not None:from core.inference.chat_template_helpers import (chat_render_target as _chat_render_target,)@@ -5637,6 +5678,7 @@ class MLXInferenceBackend:max_new_tokens,repetition_penalty,cancel_event,+ images = attached if self._reads_vision_input() else None,tools = tools,enable_thinking = enable_thinking,reasoning_effort = reasoning_effort,@@ -5761,6 +5803,7 @@ class MLXInferenceBackend:repetition_penalty,cancel_event,*,+ images = None,tools = None,enable_thinking = None,reasoning_effort = None,@@ -5916,13 +5959,23 @@ class MLXInferenceBackend:self._int8_prefill_scope(),self._generation_scopes(),):- (- gen_prompt,- prompt_cache,- cache_key,- prompt_tokens,- cached_n,- ) = self._prepare_prompt_cache(prompt, _adapter_state)+ input_embeddings = None+ if images:+ from unsloth_zoo.mlx.generate import image_prompt_embeddings++ # Any two images of one size expand to the same ids, so the token-keyed cache stays out.+ prompt_tokens, input_embeddings = image_prompt_embeddings(+ self._model, self._tokenizer, prompt, images+ )+ gen_prompt, prompt_cache, cache_key, cached_n = prompt_tokens, None, None, 0+ else:+ (+ gen_prompt,+ prompt_cache,+ cache_key,+ prompt_tokens,+ cached_n,+ ) = self._prepare_prompt_cache(prompt, _adapter_state)max_new_tokens = self._generation_limit(prompt, max_new_tokens, prompt_tokens)logger.info("Generating: prompt_len=%d, cached=%d, max_tokens=%d, model=%s, tokenizer=%s",@@ -5947,6 +6000,8 @@ class MLXInferenceBackend:gen_kwargs["prompt_cache"] = prompt_cacheelse:gen_kwargs.update(self._kv_quant_generate_kwargs())+ if input_embeddings is not None:+ gen_kwargs["input_embeddings"] = input_embeddingsif logits_processors is not None:gen_kwargs["logits_processors"] = logits_processorsfor response in stream_generate(diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 6df1c5af6fa..c86e18aa34e 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -903,6 +903,9 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:if k in _entry})+ # MLX serves a vision config mlx-lm built as text; the parent must stop accepting images.+ if model_info["is_mlx"] and _entry.get("is_vision") is False:+ model_info["is_vision"] = False# Resolved MLX runtime knobs; only the backend knows what it honored.model_info.update({diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex a1438b5dd58..a1ba059daca 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -2740,6 +2740,78 @@ def test_mlx_text_reuses_prompt_cache_on_the_next_turn(monkeypatch):assert stats["usage"]["prompt_tokens"] == 7+def _zoo_image_prefill(monkeypatch, **helpers):+ zoo = types.ModuleType("unsloth_zoo.mlx.generate")+ zoo.__dict__.update(helpers)+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.generate", zoo)+++def test_an_image_turn_through_mlx_lm_feeds_its_embeddings_outside_the_prompt_cache(monkeypatch):+ """Two images of one size expand to the same ids, so an image turn neither reuses nor banks a cache."""+ _install_fake_prompt_cache_api(monkeypatch)+ captured, seen = [], []+ token_map = {"P2": [1, 2, 3, 7, 8, 9, 10], "generated": [7, 8]}+ backend = _install_fake_text_stack(monkeypatch, token_map, captured)+ backend._reads_vision = True+ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",+ lambda _tok, messages, **_kw: messages[-1]["content"]+ if isinstance(messages[-1]["content"], str)+ else "P1 with an image",+ )+ _zoo_image_prefill(+ monkeypatch,+ image_prompt_embeddings = lambda _m, _t, prompt, images: (+ seen.append((prompt, images)) or ([1, 2, 3], "embedded")+ ),+ )+ image = object()+ messages = [{"role": "user", "content": "P1"}]+ list(backend.generate_chat_response(messages = messages, image = image, max_new_tokens = 4))+ _run_turn(backend, "P2")+ assert seen == [("P1 with an image", [image])]+ assert captured[0]["prompt"] == [1, 2, 3] and captured[0]["input_embeddings"] == "embedded"+ assert "prompt_cache" not in captured[0] and captured[1]["prompt"] == token_map["P2"]+++def test_an_image_row_through_mlx_lm_joins_the_batch_at_its_last_token(monkeypatch):+ from core.inference import mlx_inference++ inserted, rendered = [], []+ generator = SimpleNamespace(+ insert = lambda prompts, **kw: inserted.append((prompts, kw)) or [len(inserted)],+ remove = lambda *_a, **_k: {},+ close = lambda: None,+ )+ generate = types.ModuleType("mlx_lm.generate")+ generate.BatchGenerator = lambda *_a, **_k: generator+ _zoo_image_prefill(monkeypatch, prefill_image_prompt = lambda *_a: [1, 5, 5, 2])+ backend = mlx_inference.MLXInferenceBackend()+ backend._model, backend._tokenizer = SimpleNamespace(), SimpleNamespace(eos_token_ids = [2])+ backend._is_vlm, backend._reads_vision = False, True+ backend._make_row_cache = lambda: ["cache"]+ backend._batch_prompt_row = lambda prompt, _state: ([], [9], ["text cache"], "key", [9])+ plan = SimpleNamespace(+ prompt = "P", max_tokens = 4, sampler = None, processors = (), detokenizes = False, think_prefix = ""+ )+ backend._plan_text_row = lambda messages, **_kw: rendered.append(messages) or plan+ with monkeypatch.context() as patch:+ patch.setitem(sys.modules, "mlx_lm.generate", generate)+ session = mlx_inference._TextBatchSession(backend, width = 2)+ image, messages = object(), [{"role": "user", "content": "hi"}]+ backend._unset_generation_budget = lambda prompt, tokens: 100 - len(tokens)+ session.admit({"messages": messages}, 0)+ session.admit({"messages": messages, "image": image, "max_new_tokens": None}, 1)+ assert rendered[0] == messages and {"type": "image"} in rendered[1][-1]["content"]+ assert [kw["max_tokens"] for _prompts, kw in inserted] == [[4], [96]]+ assert [(prompts, kw["caches"], kw["all_tokens"]) for prompts, kw in inserted] == [+ ([[9]], [["text cache"]], [[]]),+ ([[2]], [["cache"]], [[1, 5, 5]]),+ ]+ assert session._rows[1].cache_key is None and session._rows[1].prompt_len == 4+ session.close()++def test_mlx_text_without_lru_prompt_cache_prefills_the_full_prompt(monkeypatch):from core.inference import mlx_inference@@ -4329,6 +4401,68 @@ def test_a_text_model_mlx_vlm_cannot_load_is_served_without_turboquant(monkeypatassert attempts == [(False, None)]+def test_a_vision_checkpoint_served_through_mlx_lm_reads_images_only_where_zoo_prefills_them(+ monkeypatch,+):+ """unsloth-zoo builds an image checkpoint mlx-vlm lacks (Muse Spark) through mlx-lm."""+ pytest.importorskip("mlx.core")+ from core.inference import mlx_inference++ attempts, built_by_mlx_vlm, reads_images = [], [], []++ class _Loader:+ @staticmethod+ def from_pretrained(name, **kwargs):+ attempts.append(kwargs["text_only"])+ model = SimpleNamespace(config = {}, _is_vlm_model = bool(built_by_mlx_vlm))+ if reads_images:+ model._unsloth_supports_images = True+ return model, SimpleNamespace(tokenizer = "inner")++ loader = types.ModuleType("unsloth_zoo.mlx.loader")+ loader.FastMLXModel = _Loader+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)+ monkeypatch.setattr(+ mlx_inference, "_classify_mlx_audio_type", lambda *a, config_audio_type: config_audio_type+ )+ monkeypatch.setattr(+ mlx_inference.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (2048, 2048, 2048)+ )+ vision = SimpleNamespace(+ identifier = "org/vision", is_vision = True, is_lora = False, audio_type = "audio_vlm"+ )++ backend = mlx_inference.MLXInferenceBackend()+ backend.load_model(vision, kv_quant = "tq-4")+ assert attempts == [False]+ entry = backend.models["org/vision"]+ assert entry["audio_type"] is None and entry["has_audio_input"] is False+ assert backend._is_vlm is False and backend._processor is None+ assert backend._reads_vision_input() is False+ assert backend.models["org/vision"]["is_vision"] is False+ assert backend._turboquant_refusal == mlx_inference.MLX_TURBOQUANT_TEXT_LOAD++ reads_images.append(True)+ generate = types.ModuleType("unsloth_zoo.mlx.generate")+ for zoo_prefills in (False, True):+ if zoo_prefills:+ generate.prefill_image_prompt = object()+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.generate", generate)+ backend = mlx_inference.MLXInferenceBackend()+ backend.load_model(vision, kv_quant = "tq-4")+ assert backend._is_vlm is False and backend._reads_vision_input() is zoo_prefills+ assert backend.models["org/vision"]["is_vision"] is zoo_prefills+ assert backend.models["org/vision"]["audio_type"] is None+ assert backend._turboquant_refusal == mlx_inference.MLX_TURBOQUANT_TEXT_LOAD++ reads_images.clear()+ built_by_mlx_vlm.append(True)+ backend = mlx_inference.MLXInferenceBackend()+ backend.load_model(vision)+ assert backend._is_vlm is True and backend._reads_vision_input() is True+ assert backend.models["org/vision"]["is_vision"] is True++def test_a_uniformly_quantized_text_load_goes_through_mlx_vlm_only_when_that_batches(monkeypatch):pytest.importorskip("mlx.core")from core.inference import mlx_inferencediff --git a/studio/backend/tests/test_pr7699_worker_audio_mirror.py b/studio/backend/tests/test_pr7699_worker_audio_mirror.pyindex bc4e3bfbc75..1b8debb042d 100644--- a/studio/backend/tests/test_pr7699_worker_audio_mirror.py+++ b/studio/backend/tests/test_pr7699_worker_audio_mirror.py@@ -161,3 +161,22 @@ def test_a_classified_load_failure_crosses_the_worker_and_the_parent(monkeypatchloaded = _drive_handle_load(monkeypatch, mc, device = "cuda", load_model = refuse)assert loaded["success"] is False and AudioCppFailure.from_payload(loaded["failure"]) == failureassert failure_of(AudioCppFailureError(loaded["error"], failure)) is failure+++ "device, served_as_vision, published",+ [("mlx", False, False), ("mlx", True, True), ("cuda", False, True)],+)+def test_an_mlx_vision_config_served_as_text_stops_accepting_images(+ monkeypatch, device, served_as_vision, published+):+ """MLX serves a vision config mlx-lm built (Muse Spark) as text: the parent must not offer images."""+ from core.inference.orchestrator import _mirrored_model_entry++ mc = _mc(None, is_audio = False, has_audio_input = False)+ mc.is_vision = True+ entry = {"is_audio": False, "audio_type": None, "has_audio_input": False}+ entry["is_vision"] = served_as_vision+ info = _drive_handle_load(monkeypatch, mc, entry, device = device)+ assert info["is_vision"] is published+ assert _mirrored_model_entry(info, mc.identifier)["is_vision"] is published