Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: move MLX to mlx-vlm 0.7.12 and serve mlx-vlm-only text families (#16765)

Long Yixingbfdd8fa7d2d4
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index f36f34ef2ed..dddea6c68ea 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -2143,6 +2143,19 @@ def _zoo_declares_multimodal(model_name, hf_token = None) -> bool:
return False
+def _zoo_builds_text_on_mlx_vlm(model_name, hf_token = None) -> bool:
+ """A text family only mlx-vlm ships (Raon-1 Mini): unsloth_zoo builds it there, so it is served there."""
+ try:
+ from utils.models.model_config import _read_raw_config_json
+ from unsloth_zoo.mlx.loader import _mlx_vlm_only_text_model
+
+ config = _read_raw_config_json(model_name, hf_token) or {}
+ return bool(_mlx_vlm_only_text_model(config, config.get("model_type") or ""))
+ except Exception as exc: # noqa: BLE001 -- detection must never fail a load
+ logger.debug("unsloth_zoo text-route detection skipped for %s (%s)", model_name, exc)
+ return False
+
+
def _classify_mlx_audio_type(
model,
processor,
@@ -6064,8 +6077,13 @@ class MLXInferenceBackend:
model_name = config.identifier if hasattr(config, "identifier") else str(config)
is_vision = getattr(config, "is_vision", False)
is_lora = getattr(config, "is_lora", False)
- zoo_multimodal = not is_vision and _zoo_declares_multimodal(
- getattr(config, "base_model", None) if is_lora else model_name, hf_token
+ # Route by the config the loader reads.
+ route_config = getattr(config, "config_dir", None) or (
+ getattr(config, "base_model", None) if is_lora else model_name
+ )
+ zoo_multimodal = not is_vision and _zoo_declares_multimodal(route_config, hf_token)
+ vlm_only_text = not (is_vision or zoo_multimodal) and _zoo_builds_text_on_mlx_vlm(
+ route_config, hf_token
)
kv_bits, turboquant = parse_mlx_kv_quant(kv_quant)
self._turboquant = turboquant
@@ -6091,7 +6109,7 @@ class MLXInferenceBackend:
if speculates:
spec_reason = mlx_speculative.speculation_refusal(
distributed = is_distributed,
- # A text architecture's adapter loads only through mlx-lm, and speculation runs on mlx-vlm.
+ # Speculation runs on mlx-vlm and takes an adapter only on a vision or zoo-multimodal load.
lora = is_lora and not (is_vision or zoo_multimodal),
)
speculates = spec_reason is None
@@ -6110,6 +6128,7 @@ class MLXInferenceBackend:
use_vlm = (
is_vision
or zoo_multimodal
+ or vlm_only_text
or speculates
or (self._turboquant and not self._turboquant_refusal)
or batches_quantized_text
@@ -6232,7 +6251,8 @@ class MLXInferenceBackend:
FastMLXModel.from_pretrained, load_source, load_kwargs, offload
)
except Exception as exc:
- if is_vision or not use_vlm or is_metal_queue_dead(exc):
+ # A vlm_only_text model has no mlx-lm route to fall back to.
+ if is_vision or vlm_only_text or not use_vlm or is_metal_queue_dead(exc):
raise
logger.warning(
"Load of %s through mlx-vlm for %s failed (%s); serving through mlx-lm",
@@ -6244,8 +6264,8 @@ class MLXInferenceBackend:
if model is not None and batches_quantized_text and not (is_vision or self._turboquant):
_eligibility = _kv_quant_eligibility(model, True, _normalize_mlx_kv_bits(kv_bits))
verdict, reason, _retainable = _eligibility
- if verdict not in ("full", "partial"):
- # Unquantized, the load gains nothing from mlx-vlm's batch.
+ if verdict not in ("full", "partial") and not vlm_only_text:
+ # Unquantized, the load gains nothing from mlx-vlm's batch; a vlm_only_text one stays on it.
logger.info(
"MLX KV quantization not applied to %s (%s); serving through mlx-lm",
model_name,
diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.py
index 15061a0e183..9dcde1fbe27 100644
--- a/studio/backend/core/inference/mlx_memory.py
+++ b/studio/backend/core/inference/mlx_memory.py
@@ -785,9 +785,10 @@ def _probe(config: dict, dtype, n_tokens: int, kv_bits):
failure = ValueError("no architecture module could build this config")
with weightless_forward():
for build, make_prompt_cache, model_class in _probe_models(config, dtype):
- model = build()
- cache = make_prompt_cache(model)
try:
+ # A tower that cannot stand alone (Raon-1 Mini's flat config) leaves the whole model to try.
+ model = build()
+ cache = make_prompt_cache(model)
model(mx.zeros((1, n_tokens), dtype = mx.int32), cache = cache)
except Exception as exc:
failure, cache = exc, None
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index cf53239383e..a1b89be9ca8 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -4489,6 +4489,29 @@ def test_a_text_model_only_mlx_vlm_builds_is_sized_on_mlx_vlm_and_served_as_text
assert make_cache.__module__ == "mlx_vlm.models.cache"
[email protected](not _HAVE_MLX, reason = "runs a lazy forward")
+def test_a_tower_that_cannot_stand_alone_leaves_the_whole_model_to_size(monkeypatch):
+ import mlx.core as mx
+ from mlx_lm.models.cache import KVCache
+
+ class Whole:
+ def __call__(self, inputs, cache):
+ keys = mx.zeros((1, 2, inputs.shape[1], 8))
+ return cache[0].update_and_fetch(keys, keys)[0]
+
+ def tower():
+ raise ValueError("this architecture declares no text config")
+
+ monkeypatch.setattr(
+ mm,
+ "_probe_models",
+ lambda c, d: iter(
+ [(tower, lambda m: [KVCache()], Whole), (Whole, lambda m: [KVCache()], Whole)]
+ ),
+ )
+ assert len(mm._cache_plan({}, mx.bfloat16, None)[0]) == 1
+
+
class TestKvBytes:
def test_a_hybrid_plan_is_far_cheaper_than_charging_every_layer_as_attention(self):
hybrid = [_RECURRENT] * 30 + [_GROWING] * 10
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 0adeb6248e7..d72cfd9ce40 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -5055,6 +5055,108 @@ def test_zoo_multimodal_detection_fails_open_to_text(monkeypatch):
assert mlx_inference._zoo_declares_multimodal("repo") is False
+def _text_family_backend(
+ monkeypatch,
+ vlm_only,
+ *,
+ configs = None,
+ fail_vlm_load = False,
+ config_dir = None,
+):
+ """A backend over a stub unsloth-zoo that builds `raon1_mini` on mlx-vlm when `vlm_only`."""
+ pytest.importorskip("mlx.core")
+ from core.inference import mlx_inference
+ from utils.models import model_config
+
+ attempts, asked = [], []
+
+ class _Loader:
+ @staticmethod
+ def from_pretrained(name, **kwargs):
+ attempts.append(kwargs["text_only"])
+ if fail_vlm_load and not kwargs["text_only"]:
+ raise RuntimeError("mlx-vlm load failed")
+ return SimpleNamespace(config = {}), SimpleNamespace()
+
+ loader = types.ModuleType("unsloth_zoo.mlx.loader")
+ loader.FastMLXModel = _Loader
+ loader._is_vlm = lambda cfg: False
+ loader._mlx_vlm_only_text_model = lambda cfg, model_type: (
+ asked.append(model_type) or (vlm_only and model_type == "raon1_mini")
+ )
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)
+ configs = configs or {"org/raon": {"model_type": "raon1_mini"}}
+ monkeypatch.setattr(
+ model_config, "_read_raw_config_json", lambda name, token: configs.get(name)
+ )
+ monkeypatch.setattr(mlx_inference, "_classify_mlx_audio_type", lambda *a, **k: None)
+ monkeypatch.setattr(mlx_inference, "_row_quantized_cache_gap", lambda: None)
+ monkeypatch.setattr(
+ mlx_inference, "_kv_quant_eligibility", lambda *a: ("none", "MLA cache", False)
+ )
+ monkeypatch.setattr(
+ mlx_inference.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (8192, 8192, 8192)
+ )
+ config = SimpleNamespace(
+ identifier = "org/raon", is_vision = False, is_lora = False, config_dir = config_dir
+ )
+ return mlx_inference.MLXInferenceBackend(), config, attempts, asked
+
+
[email protected]("vlm_only", [True, False])
+def test_a_text_family_only_mlx_vlm_ships_is_loaded_and_served_there(monkeypatch, vlm_only):
+ backend, config, attempts, asked = _text_family_backend(monkeypatch, vlm_only)
+ backend.load_model(config)
+ assert "raon1_mini" in asked and attempts == [not vlm_only]
+ assert backend._is_vlm is vlm_only and not backend._reads_vision_input()
+
+
+def test_a_text_family_only_mlx_vlm_ships_is_routed_by_its_precision_folder_config(monkeypatch):
+ # The repo root has no config; the loader builds from 8bit/, whose config declares the family.
+ backend, config, attempts, _ = _text_family_backend(
+ monkeypatch,
+ True,
+ configs = {"/snap/8bit": {"model_type": "raon1_mini"}},
+ config_dir = "/snap/8bit",
+ )
+ backend.load_model(config)
+ assert attempts == [False] and backend._is_vlm is True
+
+
+def test_a_multimodal_precision_folder_is_routed_by_its_own_config(monkeypatch):
+ # The root config is a text family mlx-lm serves; the picked folder's is one zoo calls multimodal.
+ backend, config, attempts, _ = _text_family_backend(
+ monkeypatch,
+ False,
+ configs = {"org/raon": {"model_type": "qwen3"}, "/snap/8bit": {"model_type": "mm_family"}},
+ config_dir = "/snap/8bit",
+ )
+ seen = []
+ sys.modules["unsloth_zoo.mlx.loader"]._is_vlm = lambda cfg: (
+ seen.append(cfg["model_type"]) or cfg["model_type"] == "mm_family"
+ )
+ backend.load_model(config)
+ assert seen == ["mm_family"] and attempts == [False] and backend._is_vlm is True
+
+
[email protected]("vlm_only", [True, False])
[email protected]("failure", ["kv_refused", "load_failed"])
+def test_a_text_family_only_mlx_vlm_ships_never_falls_back_to_mlx_lm(
+ monkeypatch, vlm_only, failure
+):
+ backend, config, attempts, _ = _text_family_backend(
+ monkeypatch, vlm_only, fail_vlm_load = failure == "load_failed"
+ )
+ if failure == "load_failed" and vlm_only:
+ with pytest.raises(RuntimeError, match = "mlx-vlm load failed"):
+ backend.load_model(config, kv_quant = "8")
+ assert attempts == [False]
+ return
+ backend.load_model(config, kv_quant = "8")
+ assert attempts == ([False] if vlm_only else [False, True])
+ assert backend._is_vlm is vlm_only and backend._kv_quant["kv_bits"] is None
+
+
def test_mlx_audio_input_normalizes_split_native_reasoning_channels(monkeypatch):
from core.inference import mlx_inference
from core.inference.mlx_inference import MLXInferenceBackend
diff --git a/studio/backend/tests/test_mlx_precision_variants.py b/studio/backend/tests/test_mlx_precision_variants.py
index 06f3f27de45..71764001205 100644
--- a/studio/backend/tests/test_mlx_precision_variants.py
+++ b/studio/backend/tests/test_mlx_precision_variants.py
@@ -11,6 +11,7 @@ import json
from pathlib import Path
from types import SimpleNamespace
+import huggingface_hub.constants
import pytest
from hub.services.models import gguf_variants
@@ -263,6 +264,9 @@ def test_gguf_quant_names_never_resolve_as_folders(monkeypatch, variant):
assert reads == []
+ huggingface_hub.constants.HF_HUB_OFFLINE, reason = "from_identifier's GGUF probe needs the Hub"
+)
def test_model_config_carries_the_folder_and_classifies_it(monkeypatch, tmp_path):
from utils.models import model_config
@@ -296,13 +300,14 @@ def test_model_config_carries_the_folder_and_classifies_it(monkeypatch, tmp_path
config = model_config.ModelConfig.from_identifier(_REPO, gguf_variant = "6-bit")
assert config.identifier == _REPO
assert config.gguf_variant == "6-bit"
+ assert config.config_dir == str(folder)
assert config.is_gguf is False
assert checked == [str(folder)]
assert fetched == ["6-bit/config.json"]
checked.clear()
plain = model_config.ModelConfig.from_identifier(_REPO)
- assert plain.gguf_variant is None
+ assert plain.gguf_variant is None and plain.config_dir is None
assert checked == [_REPO]
@@ -455,10 +460,11 @@ def test_a_resumed_folder_download_rebuilds_its_plan_from_the_manifest():
def test_memory_estimate_prices_the_picked_folder(monkeypatch, tmp_path):
+ from core.inference import mlx_memory
from routes import inference as ri
folder = _cached_repo(monkeypatch, tmp_path, _REPO, "6-bit")
- (folder / "config.json").write_text(json.dumps({"model_type": "qwen3"}))
+ (folder / "config.json").write_text(json.dumps({"model_type": "raon1_mini"}))
(folder / "model.safetensors").write_bytes(b"x")
monkeypatch.setattr(ri, "_estimate_hf_cache_roots", lambda: [str(tmp_path / "hub")])
config = SimpleNamespace(
@@ -470,9 +476,9 @@ def test_memory_estimate_prices_the_picked_folder(monkeypatch, tmp_path):
)
(folder.parent / "config.json").write_text(json.dumps({"model_type": "qwen3"}))
(folder.parent / "model.safetensors").write_bytes(b"x")
- assert ri._local_mlx_model_dir(SimpleNamespace(**vars(config), gguf_variant = "6-bit")) == str(
- folder
- )
+ picked = ri._local_mlx_model_dir(SimpleNamespace(**vars(config), gguf_variant = "6-bit"))
+ assert picked == str(folder)
+ assert mlx_memory._snapshot_config(picked)["model_type"] == "raon1_mini"
@pytest.mark.parametrize("resident_folder, expected", [(None, "reuse"), ("6-bit", "load")])
diff --git a/studio/backend/tests/test_mlx_repair.py b/studio/backend/tests/test_mlx_repair.py
index b10e8dff43d..6ba7a19e7e1 100644
--- a/studio/backend/tests/test_mlx_repair.py
+++ b/studio/backend/tests/test_mlx_repair.py
@@ -53,7 +53,7 @@ def test_uv_cmd_targets_this_interpreter_with_mlx_packages(monkeypatch):
assert cmd is not None
assert cmd[:5] == ["/usr/bin/uv", "pip", "install", "--python", sys.executable]
assert set(mr.MLX_PACKAGES) <= set(cmd)
- assert "mlx-vlm>=0.4.4,<=0.7.11" in cmd
+ assert "mlx-vlm>=0.4.4,<=0.7.12" in cmd
# Pinned, not floored: see _MLX_INSTALL_SPECS.
assert "mlx==0.32.4" in cmd
assert "mlx-lm>=0.31.3,<=0.32.2" in cmd
diff --git a/studio/backend/utils/mlx_repair.py b/studio/backend/utils/mlx_repair.py
index 69ebf8a2e40..2c08b0412ee 100644
--- a/studio/backend/utils/mlx_repair.py
+++ b/studio/backend/utils/mlx_repair.py
@@ -34,7 +34,7 @@ _MLX_RUNTIME_IMPORTS = ("mlx.core", "mlx_lm", "mlx_lm.sample_utils", "mlx_vlm")
_MLX_INSTALL_SPECS = {
"mlx": "==0.32.4",
"mlx-lm": ">=0.31.3,<=0.32.2",
- "mlx-vlm": ">=0.4.4,<=0.7.11",
+ "mlx-vlm": ">=0.4.4,<=0.7.12",
}
MLX_PACKAGES = tuple(f"{name}{spec}" for name, spec in _MLX_INSTALL_SPECS.items())
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 4fc3452d260..261ab0ce1a8 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -4371,6 +4371,8 @@ class ModelConfig:
# Repo a refused-Hub cached GGUF came from, for the download interlock.
gguf_cache_repo: Optional[str] = None
gguf_variant: Optional[str] = None # Quantization variant (e.g. "Q4_K_M")
+ # A picked MLX precision folder's config, for route checks only; sizing needs the folder's weights.
+ config_dir: Optional[str] = None
base_model: Optional[str] = None # Base model (for LoRAs)
# Set for a GGUF only audio.cpp runs; loads via the native-audio worker, never llama-server.
audio_cpp: Optional[Any] = None
@@ -4861,6 +4863,7 @@ class ModelConfig:
return cls(
gguf_variant = precision_folder,
+ config_dir = check_model if precision_folder else None,
identifier = identifier,
display_name = display_name,
path = path,
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py
index e49886fa2ee..bfdfcb41ff8 100644
--- a/studio/install_python_stack.py
+++ b/studio/install_python_stack.py
@@ -10381,7 +10381,7 @@ def _has_working_git() -> bool:
_MLX_PINS: tuple[str, ...] = ("mlx==0.32.4", "mlx-metal==0.32.4")
# A range, like mlx-vlm: 0.32 requires Python 3.11, so Python 3.10 resolves 0.31.3.
_MLX_LM_SPEC = "mlx-lm>=0.31.3,<=0.32.2"
-_MLX_VLM_SPEC = "mlx-vlm>=0.4.4,<=0.7.11"
+_MLX_VLM_SPEC = "mlx-vlm>=0.4.4,<=0.7.12"
# Exact: llguidance.mlx / llguidance.hf are the API grammar_constraint.py binds to.
_LLGUIDANCE_PIN = "llguidance==1.8.0"
_MLX_NAMES: tuple[str, ...] = tuple(spec.partition("==")[0] for spec in _MLX_PINS) + (
diff --git a/tests/studio/install/test_dependency_pass_skips.py b/tests/studio/install/test_dependency_pass_skips.py
index 64d50611ae7..6cc8d7ec7a3 100644
--- a/tests/studio/install/test_dependency_pass_skips.py
+++ b/tests/studio/install/test_dependency_pass_skips.py
@@ -1021,12 +1021,12 @@ def test_the_pip_fallback_never_names_one_project_twice() -> None:
"--upgrade-package",
"mlx-vlm",
"mlx==0.32.3",
- "mlx-vlm>=0.4.4,<=0.7.11",
+ "mlx-vlm>=0.4.4,<=0.7.12",
)
)
assert "--upgrade" in cmd and "--upgrade-package" not in cmd
assert cmd.count("mlx") == 0 and cmd.count("mlx-vlm") == 0
- assert "mlx==0.32.3" in cmd and "mlx-vlm>=0.4.4,<=0.7.11" in cmd
+ assert "mlx==0.32.3" in cmd and "mlx-vlm>=0.4.4,<=0.7.12" in cmd
def test_a_package_named_only_by_the_flag_is_still_passed() -> None:
diff --git a/tests/studio/install/test_mlx_install.py b/tests/studio/install/test_mlx_install.py
index d4df4fdfdcc..f4b783b3337 100644
--- a/tests/studio/install/test_mlx_install.py
+++ b/tests/studio/install/test_mlx_install.py
@@ -316,7 +316,7 @@ def test_pin_floor_is_revisited_whenever_the_pins_move():
assert _repair_specs() == {
"mlx": "==0.32.4",
"mlx-lm": ">=0.31.3,<=0.32.2",
- "mlx-vlm": ">=0.4.4,<=0.7.11",
+ "mlx-vlm": ">=0.4.4,<=0.7.12",
}
assert (stack._MLX_MIN_PYTHON, stack._MLX_MIN_MACOS_MAJOR) == ((3, 10), 14)