Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: move MLX to mlx-vlm 0.7.12 and serve mlx-vlm-only text families (#16765)
Long Yixingbfdd8fa7d2d4
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex f36f34ef2ed..dddea6c68ea 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -2143,6 +2143,19 @@ def _zoo_declares_multimodal(model_name, hf_token = None) -> bool:return False+def _zoo_builds_text_on_mlx_vlm(model_name, hf_token = None) -> bool:+ """A text family only mlx-vlm ships (Raon-1 Mini): unsloth_zoo builds it there, so it is served there."""+ try:+ from utils.models.model_config import _read_raw_config_json+ from unsloth_zoo.mlx.loader import _mlx_vlm_only_text_model++ config = _read_raw_config_json(model_name, hf_token) or {}+ return bool(_mlx_vlm_only_text_model(config, config.get("model_type") or ""))+ except Exception as exc: # noqa: BLE001 -- detection must never fail a load+ logger.debug("unsloth_zoo text-route detection skipped for %s (%s)", model_name, exc)+ return False++def _classify_mlx_audio_type(model,processor,@@ -6064,8 +6077,13 @@ class MLXInferenceBackend:model_name = config.identifier if hasattr(config, "identifier") else str(config)is_vision = getattr(config, "is_vision", False)is_lora = getattr(config, "is_lora", False)- zoo_multimodal = not is_vision and _zoo_declares_multimodal(- getattr(config, "base_model", None) if is_lora else model_name, hf_token+ # Route by the config the loader reads.+ route_config = getattr(config, "config_dir", None) or (+ getattr(config, "base_model", None) if is_lora else model_name+ )+ zoo_multimodal = not is_vision and _zoo_declares_multimodal(route_config, hf_token)+ vlm_only_text = not (is_vision or zoo_multimodal) and _zoo_builds_text_on_mlx_vlm(+ route_config, hf_token)kv_bits, turboquant = parse_mlx_kv_quant(kv_quant)self._turboquant = turboquant@@ -6091,7 +6109,7 @@ class MLXInferenceBackend:if speculates:spec_reason = mlx_speculative.speculation_refusal(distributed = is_distributed,- # A text architecture's adapter loads only through mlx-lm, and speculation runs on mlx-vlm.+ # Speculation runs on mlx-vlm and takes an adapter only on a vision or zoo-multimodal load.lora = is_lora and not (is_vision or zoo_multimodal),)speculates = spec_reason is None@@ -6110,6 +6128,7 @@ class MLXInferenceBackend:use_vlm = (is_visionor zoo_multimodal+ or vlm_only_textor speculatesor (self._turboquant and not self._turboquant_refusal)or batches_quantized_text@@ -6232,7 +6251,8 @@ class MLXInferenceBackend:FastMLXModel.from_pretrained, load_source, load_kwargs, offload)except Exception as exc:- if is_vision or not use_vlm or is_metal_queue_dead(exc):+ # A vlm_only_text model has no mlx-lm route to fall back to.+ if is_vision or vlm_only_text or not use_vlm or is_metal_queue_dead(exc):raiselogger.warning("Load of %s through mlx-vlm for %s failed (%s); serving through mlx-lm",@@ -6244,8 +6264,8 @@ class MLXInferenceBackend:if model is not None and batches_quantized_text and not (is_vision or self._turboquant):_eligibility = _kv_quant_eligibility(model, True, _normalize_mlx_kv_bits(kv_bits))verdict, reason, _retainable = _eligibility- if verdict not in ("full", "partial"):- # Unquantized, the load gains nothing from mlx-vlm's batch.+ if verdict not in ("full", "partial") and not vlm_only_text:+ # Unquantized, the load gains nothing from mlx-vlm's batch; a vlm_only_text one stays on it.logger.info("MLX KV quantization not applied to %s (%s); serving through mlx-lm",model_name,diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.pyindex 15061a0e183..9dcde1fbe27 100644--- a/studio/backend/core/inference/mlx_memory.py+++ b/studio/backend/core/inference/mlx_memory.py@@ -785,9 +785,10 @@ def _probe(config: dict, dtype, n_tokens: int, kv_bits):failure = ValueError("no architecture module could build this config")with weightless_forward():for build, make_prompt_cache, model_class in _probe_models(config, dtype):- model = build()- cache = make_prompt_cache(model)try:+ # A tower that cannot stand alone (Raon-1 Mini's flat config) leaves the whole model to try.+ model = build()+ cache = make_prompt_cache(model)model(mx.zeros((1, n_tokens), dtype = mx.int32), cache = cache)except Exception as exc:failure, cache = exc, Nonediff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex cf53239383e..a1b89be9ca8 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -4489,6 +4489,29 @@ def test_a_text_model_only_mlx_vlm_builds_is_sized_on_mlx_vlm_and_served_as_textassert make_cache.__module__ == "mlx_vlm.models.cache"[email protected](not _HAVE_MLX, reason = "runs a lazy forward")+def test_a_tower_that_cannot_stand_alone_leaves_the_whole_model_to_size(monkeypatch):+ import mlx.core as mx+ from mlx_lm.models.cache import KVCache++ class Whole:+ def __call__(self, inputs, cache):+ keys = mx.zeros((1, 2, inputs.shape[1], 8))+ return cache[0].update_and_fetch(keys, keys)[0]++ def tower():+ raise ValueError("this architecture declares no text config")++ monkeypatch.setattr(+ mm,+ "_probe_models",+ lambda c, d: iter(+ [(tower, lambda m: [KVCache()], Whole), (Whole, lambda m: [KVCache()], Whole)]+ ),+ )+ assert len(mm._cache_plan({}, mx.bfloat16, None)[0]) == 1++class TestKvBytes:def test_a_hybrid_plan_is_far_cheaper_than_charging_every_layer_as_attention(self):hybrid = [_RECURRENT] * 30 + [_GROWING] * 10diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 0adeb6248e7..d72cfd9ce40 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -5055,6 +5055,108 @@ def test_zoo_multimodal_detection_fails_open_to_text(monkeypatch):assert mlx_inference._zoo_declares_multimodal("repo") is False+def _text_family_backend(+ monkeypatch,+ vlm_only,+ *,+ configs = None,+ fail_vlm_load = False,+ config_dir = None,+):+ """A backend over a stub unsloth-zoo that builds `raon1_mini` on mlx-vlm when `vlm_only`."""+ pytest.importorskip("mlx.core")+ from core.inference import mlx_inference+ from utils.models import model_config++ attempts, asked = [], []++ class _Loader:+ @staticmethod+ def from_pretrained(name, **kwargs):+ attempts.append(kwargs["text_only"])+ if fail_vlm_load and not kwargs["text_only"]:+ raise RuntimeError("mlx-vlm load failed")+ return SimpleNamespace(config = {}), SimpleNamespace()++ loader = types.ModuleType("unsloth_zoo.mlx.loader")+ loader.FastMLXModel = _Loader+ loader._is_vlm = lambda cfg: False+ loader._mlx_vlm_only_text_model = lambda cfg, model_type: (+ asked.append(model_type) or (vlm_only and model_type == "raon1_mini")+ )+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)+ configs = configs or {"org/raon": {"model_type": "raon1_mini"}}+ monkeypatch.setattr(+ model_config, "_read_raw_config_json", lambda name, token: configs.get(name)+ )+ monkeypatch.setattr(mlx_inference, "_classify_mlx_audio_type", lambda *a, **k: None)+ monkeypatch.setattr(mlx_inference, "_row_quantized_cache_gap", lambda: None)+ monkeypatch.setattr(+ mlx_inference, "_kv_quant_eligibility", lambda *a: ("none", "MLA cache", False)+ )+ monkeypatch.setattr(+ mlx_inference.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (8192, 8192, 8192)+ )+ config = SimpleNamespace(+ identifier = "org/raon", is_vision = False, is_lora = False, config_dir = config_dir+ )+ return mlx_inference.MLXInferenceBackend(), config, attempts, asked++[email protected]("vlm_only", [True, False])+def test_a_text_family_only_mlx_vlm_ships_is_loaded_and_served_there(monkeypatch, vlm_only):+ backend, config, attempts, asked = _text_family_backend(monkeypatch, vlm_only)+ backend.load_model(config)+ assert "raon1_mini" in asked and attempts == [not vlm_only]+ assert backend._is_vlm is vlm_only and not backend._reads_vision_input()+++def test_a_text_family_only_mlx_vlm_ships_is_routed_by_its_precision_folder_config(monkeypatch):+ # The repo root has no config; the loader builds from 8bit/, whose config declares the family.+ backend, config, attempts, _ = _text_family_backend(+ monkeypatch,+ True,+ configs = {"/snap/8bit": {"model_type": "raon1_mini"}},+ config_dir = "/snap/8bit",+ )+ backend.load_model(config)+ assert attempts == [False] and backend._is_vlm is True+++def test_a_multimodal_precision_folder_is_routed_by_its_own_config(monkeypatch):+ # The root config is a text family mlx-lm serves; the picked folder's is one zoo calls multimodal.+ backend, config, attempts, _ = _text_family_backend(+ monkeypatch,+ False,+ configs = {"org/raon": {"model_type": "qwen3"}, "/snap/8bit": {"model_type": "mm_family"}},+ config_dir = "/snap/8bit",+ )+ seen = []+ sys.modules["unsloth_zoo.mlx.loader"]._is_vlm = lambda cfg: (+ seen.append(cfg["model_type"]) or cfg["model_type"] == "mm_family"+ )+ backend.load_model(config)+ assert seen == ["mm_family"] and attempts == [False] and backend._is_vlm is True++[email protected]("vlm_only", [True, False])[email protected]("failure", ["kv_refused", "load_failed"])+def test_a_text_family_only_mlx_vlm_ships_never_falls_back_to_mlx_lm(+ monkeypatch, vlm_only, failure+):+ backend, config, attempts, _ = _text_family_backend(+ monkeypatch, vlm_only, fail_vlm_load = failure == "load_failed"+ )+ if failure == "load_failed" and vlm_only:+ with pytest.raises(RuntimeError, match = "mlx-vlm load failed"):+ backend.load_model(config, kv_quant = "8")+ assert attempts == [False]+ return+ backend.load_model(config, kv_quant = "8")+ assert attempts == ([False] if vlm_only else [False, True])+ assert backend._is_vlm is vlm_only and backend._kv_quant["kv_bits"] is None++def test_mlx_audio_input_normalizes_split_native_reasoning_channels(monkeypatch):from core.inference import mlx_inferencefrom core.inference.mlx_inference import MLXInferenceBackenddiff --git a/studio/backend/tests/test_mlx_precision_variants.py b/studio/backend/tests/test_mlx_precision_variants.pyindex 06f3f27de45..71764001205 100644--- a/studio/backend/tests/test_mlx_precision_variants.py+++ b/studio/backend/tests/test_mlx_precision_variants.py@@ -11,6 +11,7 @@ import jsonfrom pathlib import Pathfrom types import SimpleNamespace+import huggingface_hub.constantsimport pytestfrom hub.services.models import gguf_variants@@ -263,6 +264,9 @@ def test_gguf_quant_names_never_resolve_as_folders(monkeypatch, variant):assert reads == []+ huggingface_hub.constants.HF_HUB_OFFLINE, reason = "from_identifier's GGUF probe needs the Hub"+)def test_model_config_carries_the_folder_and_classifies_it(monkeypatch, tmp_path):from utils.models import model_config@@ -296,13 +300,14 @@ def test_model_config_carries_the_folder_and_classifies_it(monkeypatch, tmp_pathconfig = model_config.ModelConfig.from_identifier(_REPO, gguf_variant = "6-bit")assert config.identifier == _REPOassert config.gguf_variant == "6-bit"+ assert config.config_dir == str(folder)assert config.is_gguf is Falseassert checked == [str(folder)]assert fetched == ["6-bit/config.json"]checked.clear()plain = model_config.ModelConfig.from_identifier(_REPO)- assert plain.gguf_variant is None+ assert plain.gguf_variant is None and plain.config_dir is Noneassert checked == [_REPO]@@ -455,10 +460,11 @@ def test_a_resumed_folder_download_rebuilds_its_plan_from_the_manifest():def test_memory_estimate_prices_the_picked_folder(monkeypatch, tmp_path):+ from core.inference import mlx_memoryfrom routes import inference as rifolder = _cached_repo(monkeypatch, tmp_path, _REPO, "6-bit")- (folder / "config.json").write_text(json.dumps({"model_type": "qwen3"}))+ (folder / "config.json").write_text(json.dumps({"model_type": "raon1_mini"}))(folder / "model.safetensors").write_bytes(b"x")monkeypatch.setattr(ri, "_estimate_hf_cache_roots", lambda: [str(tmp_path / "hub")])config = SimpleNamespace(@@ -470,9 +476,9 @@ def test_memory_estimate_prices_the_picked_folder(monkeypatch, tmp_path):)(folder.parent / "config.json").write_text(json.dumps({"model_type": "qwen3"}))(folder.parent / "model.safetensors").write_bytes(b"x")- assert ri._local_mlx_model_dir(SimpleNamespace(**vars(config), gguf_variant = "6-bit")) == str(- folder- )+ picked = ri._local_mlx_model_dir(SimpleNamespace(**vars(config), gguf_variant = "6-bit"))+ assert picked == str(folder)+ assert mlx_memory._snapshot_config(picked)["model_type"] == "raon1_mini"@pytest.mark.parametrize("resident_folder, expected", [(None, "reuse"), ("6-bit", "load")])diff --git a/studio/backend/tests/test_mlx_repair.py b/studio/backend/tests/test_mlx_repair.pyindex b10e8dff43d..6ba7a19e7e1 100644--- a/studio/backend/tests/test_mlx_repair.py+++ b/studio/backend/tests/test_mlx_repair.py@@ -53,7 +53,7 @@ def test_uv_cmd_targets_this_interpreter_with_mlx_packages(monkeypatch):assert cmd is not Noneassert cmd[:5] == ["/usr/bin/uv", "pip", "install", "--python", sys.executable]assert set(mr.MLX_PACKAGES) <= set(cmd)- assert "mlx-vlm>=0.4.4,<=0.7.11" in cmd+ assert "mlx-vlm>=0.4.4,<=0.7.12" in cmd# Pinned, not floored: see _MLX_INSTALL_SPECS.assert "mlx==0.32.4" in cmdassert "mlx-lm>=0.31.3,<=0.32.2" in cmddiff --git a/studio/backend/utils/mlx_repair.py b/studio/backend/utils/mlx_repair.pyindex 69ebf8a2e40..2c08b0412ee 100644--- a/studio/backend/utils/mlx_repair.py+++ b/studio/backend/utils/mlx_repair.py@@ -34,7 +34,7 @@ _MLX_RUNTIME_IMPORTS = ("mlx.core", "mlx_lm", "mlx_lm.sample_utils", "mlx_vlm")_MLX_INSTALL_SPECS = {"mlx": "==0.32.4","mlx-lm": ">=0.31.3,<=0.32.2",- "mlx-vlm": ">=0.4.4,<=0.7.11",+ "mlx-vlm": ">=0.4.4,<=0.7.12",}MLX_PACKAGES = tuple(f"{name}{spec}" for name, spec in _MLX_INSTALL_SPECS.items())diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.pyindex 4fc3452d260..261ab0ce1a8 100644--- a/studio/backend/utils/models/model_config.py+++ b/studio/backend/utils/models/model_config.py@@ -4371,6 +4371,8 @@ class ModelConfig:# Repo a refused-Hub cached GGUF came from, for the download interlock.gguf_cache_repo: Optional[str] = Nonegguf_variant: Optional[str] = None # Quantization variant (e.g. "Q4_K_M")+ # A picked MLX precision folder's config, for route checks only; sizing needs the folder's weights.+ config_dir: Optional[str] = Nonebase_model: Optional[str] = None # Base model (for LoRAs)# Set for a GGUF only audio.cpp runs; loads via the native-audio worker, never llama-server.audio_cpp: Optional[Any] = None@@ -4861,6 +4863,7 @@ class ModelConfig:return cls(gguf_variant = precision_folder,+ config_dir = check_model if precision_folder else None,identifier = identifier,display_name = display_name,path = path,diff --git a/studio/install_python_stack.py b/studio/install_python_stack.pyindex e49886fa2ee..bfdfcb41ff8 100644--- a/studio/install_python_stack.py+++ b/studio/install_python_stack.py@@ -10381,7 +10381,7 @@ def _has_working_git() -> bool:_MLX_PINS: tuple[str, ...] = ("mlx==0.32.4", "mlx-metal==0.32.4")# A range, like mlx-vlm: 0.32 requires Python 3.11, so Python 3.10 resolves 0.31.3._MLX_LM_SPEC = "mlx-lm>=0.31.3,<=0.32.2"-_MLX_VLM_SPEC = "mlx-vlm>=0.4.4,<=0.7.11"+_MLX_VLM_SPEC = "mlx-vlm>=0.4.4,<=0.7.12"# Exact: llguidance.mlx / llguidance.hf are the API grammar_constraint.py binds to._LLGUIDANCE_PIN = "llguidance==1.8.0"_MLX_NAMES: tuple[str, ...] = tuple(spec.partition("==")[0] for spec in _MLX_PINS) + (diff --git a/tests/studio/install/test_dependency_pass_skips.py b/tests/studio/install/test_dependency_pass_skips.pyindex 64d50611ae7..6cc8d7ec7a3 100644--- a/tests/studio/install/test_dependency_pass_skips.py+++ b/tests/studio/install/test_dependency_pass_skips.py@@ -1021,12 +1021,12 @@ def test_the_pip_fallback_never_names_one_project_twice() -> None:"--upgrade-package","mlx-vlm","mlx==0.32.3",- "mlx-vlm>=0.4.4,<=0.7.11",+ "mlx-vlm>=0.4.4,<=0.7.12",))assert "--upgrade" in cmd and "--upgrade-package" not in cmdassert cmd.count("mlx") == 0 and cmd.count("mlx-vlm") == 0- assert "mlx==0.32.3" in cmd and "mlx-vlm>=0.4.4,<=0.7.11" in cmd+ assert "mlx==0.32.3" in cmd and "mlx-vlm>=0.4.4,<=0.7.12" in cmddef test_a_package_named_only_by_the_flag_is_still_passed() -> None:diff --git a/tests/studio/install/test_mlx_install.py b/tests/studio/install/test_mlx_install.pyindex d4df4fdfdcc..f4b783b3337 100644--- a/tests/studio/install/test_mlx_install.py+++ b/tests/studio/install/test_mlx_install.py@@ -316,7 +316,7 @@ def test_pin_floor_is_revisited_whenever_the_pins_move():assert _repair_specs() == {"mlx": "==0.32.4","mlx-lm": ">=0.31.3,<=0.32.2",- "mlx-vlm": ">=0.4.4,<=0.7.11",+ "mlx-vlm": ">=0.4.4,<=0.7.12",}assert (stack._MLX_MIN_PYTHON, stack._MLX_MIN_MACOS_MAJOR) == ((3, 10), 14)