Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(studio): MLX memory estimate no longer allocates the model, and sizes DeepSeek V4 / V4.1 (#14325)
Long Yixing075395868dcc
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex bda64ad75f1..ef35446d0bd 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -2735,21 +2735,23 @@ def _asking_the_architecture(model_dir, question):_probe_models,_runtime_dtype,_snapshot_config,+ weightless_forward,)config = _snapshot_config(model_dir)if config is None:return None- for build, make_prompt_cache, _ in _probe_models(config, _runtime_dtype()):- model = build()- try:- model(- mx.zeros((1, _PROBE_SHORT), dtype = mx.int32),- cache = make_prompt_cache(model),- )- except Exception:- continue- return question(model, make_prompt_cache)+ with weightless_forward():+ for build, make_prompt_cache, _ in _probe_models(config, _runtime_dtype()):+ model = build()+ try:+ model(+ mx.zeros((1, _PROBE_SHORT), dtype = mx.int32),+ cache = make_prompt_cache(model),+ )+ except Exception:+ continue+ return question(model, make_prompt_cache)except Exception as exc:logger.debug("MLX architecture probe unavailable for %s: %s", model_dir, exc)return Nonediff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.pyindex 5f175db3960..037c65af123 100644--- a/studio/backend/core/inference/mlx_memory.py+++ b/studio/backend/core/inference/mlx_memory.py@@ -5,6 +5,7 @@from __future__ import annotations+import contextlibimport contextvarsimport globimport functools@@ -142,7 +143,7 @@ def mlx_shard_files(model_dir: str, config: Optional[dict] = None) -> list:]if config is None:return [path for _, path in names]- if _loads_as_vision(config):+ if _builds_on_mlx_vlm(config):return _indexed_shards(model_dir) or [path for name, path in names if name != "consolidated.safetensors"]@@ -252,7 +253,7 @@ def _resident_bytes(model_dir: str, config: dict, quantize: bool) -> int:model,config,spec,- is_vlm = _loads_as_vision(config),+ is_vlm = _builds_on_mlx_vlm(config),)return sum(value.nbytes for _, value in tree_flatten(model.parameters()) if hasattr(value, "nbytes")@@ -319,6 +320,18 @@ def _loads_as_vision(config: dict) -> bool:return bool(_is_vlm(SimpleNamespace(**config)))+def _builds_on_mlx_vlm(config: dict) -> bool:+ """Whether mlx-vlm builds the weights: on a vision route, or for a text model unsloth_zoo builds+ there because mlx-lm cannot; that one is still served as text."""+ if _loads_as_vision(config):+ return True+ try:+ from unsloth_zoo.mlx.loader import _mlx_vlm_only_text_model+ return bool(_mlx_vlm_only_text_model(config, config.get("model_type") or ""))+ except Exception:+ return False++def _routes_to_diffusion(config: dict) -> bool:"""Whether mlx-vlm would divert this load to a diffusion generator."""try:@@ -387,7 +400,7 @@ def _loader_config(arch, config: dict):def _whole_model(config: dict, dtype):if config.get("model_file"):raise ValueError("this checkpoint carries its own model module")- if _loads_as_vision(config):+ if _builds_on_mlx_vlm(config):from mlx_vlm.utils import get_model_and_argsarch = get_model_and_args(config)[0]model = arch.Model(_loader_config(arch, config))@@ -416,7 +429,7 @@ def _probe_models(config: dict, dtype):return _build- if not _loads_as_vision(config):+ if not _builds_on_mlx_vlm(config):from mlx_lm.models.cache import make_prompt_cachefrom mlx_lm.utils import _get_classes@@ -622,6 +635,37 @@ def _entries(cache, quantized = None) -> list:return entries+_DEFERRED_EVAL = threading.local()+_DEFERRABLE_LOCK = threading.Lock()+++def _deferrable(evaluate):+ def call(*args, **kwargs):+ if getattr(_DEFERRED_EVAL, "depth", 0):+ return None+ return evaluate(*args, **kwargs)++ call._unsloth_deferrable = True+ return call+++def weightless_forward():+ """Defer ``mx.eval`` / ``mx.async_eval`` on this thread while a weightless tower runs. Model code+ evaluates mid-forward (mlx-vlm's per-layer ``async_eval``), which on random weights materializes+ the whole model; the probes read only shapes, which need no evaluation."""+ import mlx.core as mx++ with _DEFERRABLE_LOCK:+ if not getattr(mx.eval, "_unsloth_deferrable", False):+ mx.eval, mx.async_eval = _deferrable(mx.eval), _deferrable(mx.async_eval)+ _DEFERRED_EVAL.depth = getattr(_DEFERRED_EVAL, "depth", 0) + 1+ try:+ yield+ finally:+ _DEFERRED_EVAL.depth -= 1++def _probe(config: dict, dtype, n_tokens: int, kv_bits):import mlx.core as mx@@ -630,21 +674,22 @@ def _probe(config: dict, dtype, n_tokens: int, kv_bits):layers = Nonewidths = (0,) * len(_config_widths({}))failure = ValueError("no architecture module could build this config")- for build, make_prompt_cache, model_class in _probe_models(config, dtype):- model = build()- cache = make_prompt_cache(model)- try:- model(mx.zeros((1, n_tokens), dtype = mx.int32), cache = cache)- except Exception as exc:- failure, cache = exc, None- continue- whole_prompt = _declines_to_chunk(model_class, model)- layers = _tower_layers(model)- widths = _tower_widths(model)- break- if cache is None:- raise failure- quantized = _quantize_like_runtime(cache, kv_bits)+ with weightless_forward():+ for build, make_prompt_cache, model_class in _probe_models(config, dtype):+ model = build()+ cache = make_prompt_cache(model)+ try:+ model(mx.zeros((1, n_tokens), dtype = mx.int32), cache = cache)+ except Exception as exc:+ failure, cache = exc, None+ continue+ whole_prompt = _declines_to_chunk(model_class, model)+ layers = _tower_layers(model)+ widths = _tower_widths(model)+ break+ if cache is None:+ raise failure+ quantized = _quantize_like_runtime(cache, kv_bits)entries = _entries(cache, quantized)if quantized is not None:converted = [leaf for entry in quantized for leaf in _sub_caches(entry)]@@ -957,7 +1002,7 @@ def _extra_tensors_refused(model, config: dict, extras: list) -> Optional[str]:except ValueError as refusal:return str(refusal)- vision = _loads_as_vision(config)+ vision = _builds_on_mlx_vlm(config)table = _KNOWN_VLM_EXTRA_WEIGHT_FILTERS if vision else _KNOWN_MLX_LM_STRICT_FALLBACKSrule = table.get(model_type)if rule is not None and _message_matches_known_fallback(message, rule):diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex b5f0e5ee4d2..ffd31073444 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -4248,6 +4248,63 @@ class TestCacheWidthName:]+@_NEEDS_MLX+def test_a_weightless_probe_never_runs_the_evaluations_its_model_asks_for(monkeypatch):+ import threading++ import mlx.core as mx+ from mlx_lm.models.cache import KVCache++ elsewhere = []++ def evaluates():+ before, other = mx.get_active_memory(), mx.ones((16, 1024, 1024))+ mx.eval(other)+ elsewhere.append(mx.get_active_memory() - before >= other.nbytes)++ class Tower:+ def __call__(self, inputs, cache):+ weights = mx.ones((256, 1024, 1024)) # 1 GiB once evaluated+ mx.async_eval(weights)+ mx.eval(weights)+ thread = threading.Thread(target = evaluates)+ thread.start()+ thread.join()+ keys = mx.zeros((1, 2, inputs.shape[1], 8)) + weights[0, 0, 0]+ return cache[0].update_and_fetch(keys, keys)[0]++ monkeypatch.setattr(+ mm, "_probe_models", lambda c, d: iter([(Tower, lambda m: [KVCache()], Tower)])+ )+ mx.reset_peak_memory()+ assert len(mm._cache_plan({}, mx.bfloat16, None)[0]) == 1+ assert mx.get_peak_memory() < 2**28+ from core.inference.mlx_inference import mlx_kv_quant_is_refused++ monkeypatch.setattr(mm, "_snapshot_config", lambda d: {})+ mlx_kv_quant_is_refused("weightless", 8)+ assert mx.get_peak_memory() < 2**28+ assert elsewhere and all(elsewhere)+ mx.eval(mx.ones((64, 1024, 1024)))+ assert mx.get_peak_memory() >= 2**28++[email protected](not _HAVE_MLX, reason = "resolves an mlx-vlm architecture")+def test_a_text_model_only_mlx_vlm_builds_is_sized_on_mlx_vlm_and_served_as_text(monkeypatch):+ zoo, asked = _types.ModuleType("unsloth_zoo.mlx.loader"), []+ zoo._mlx_vlm_only_text_model = lambda config, model_type: (+ asked.append(model_type) or "engram_config" in config+ )+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", zoo)+ refused = {"model_type": "deepseek_v4", "engram_config": {}}+ assert mm._builds_on_mlx_vlm(refused)+ assert not mm._builds_on_mlx_vlm({"model_type": "deepseek_v4"})+ assert asked == ["deepseek_v4", "deepseek_v4"]+ assert not mm._loads_as_vision(refused)+ _, make_cache, _ = next(mm._probe_models(refused, None))+ assert make_cache.__module__ == "mlx_vlm.models.cache"++class TestKvBytes:def test_a_hybrid_plan_is_far_cheaper_than_charging_every_layer_as_attention(self):hybrid = [_RECURRENT] * 30 + [_GROWING] * 10diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 94e0d6528ff..1a434cd77ff 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -7189,6 +7189,7 @@ def test_the_width_probe_judges_the_tower_the_sizing_would_price(monkeypatch):_probe_models = lambda *_a: iter([(lambda t = t: t, lambda tower: list(tower.entries), None) for t in towers]),+ weightless_forward = contextlib.nullcontext,),)marks = len(rewound)