Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): MLX memory estimate no longer allocates the model, and sizes DeepSeek V4 / V4.1 (#14325)

Long Yixing075395868dcc
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index bda64ad75f1..ef35446d0bd 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -2735,21 +2735,23 @@ def _asking_the_architecture(model_dir, question):
_probe_models,
_runtime_dtype,
_snapshot_config,
+ weightless_forward,
)
config = _snapshot_config(model_dir)
if config is None:
return None
- for build, make_prompt_cache, _ in _probe_models(config, _runtime_dtype()):
- model = build()
- try:
- model(
- mx.zeros((1, _PROBE_SHORT), dtype = mx.int32),
- cache = make_prompt_cache(model),
- )
- except Exception:
- continue
- return question(model, make_prompt_cache)
+ with weightless_forward():
+ for build, make_prompt_cache, _ in _probe_models(config, _runtime_dtype()):
+ model = build()
+ try:
+ model(
+ mx.zeros((1, _PROBE_SHORT), dtype = mx.int32),
+ cache = make_prompt_cache(model),
+ )
+ except Exception:
+ continue
+ return question(model, make_prompt_cache)
except Exception as exc:
logger.debug("MLX architecture probe unavailable for %s: %s", model_dir, exc)
return None
diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.py
index 5f175db3960..037c65af123 100644
--- a/studio/backend/core/inference/mlx_memory.py
+++ b/studio/backend/core/inference/mlx_memory.py
@@ -5,6 +5,7 @@
from __future__ import annotations
+import contextlib
import contextvars
import glob
import functools
@@ -142,7 +143,7 @@ def mlx_shard_files(model_dir: str, config: Optional[dict] = None) -> list:
]
if config is None:
return [path for _, path in names]
- if _loads_as_vision(config):
+ if _builds_on_mlx_vlm(config):
return _indexed_shards(model_dir) or [
path for name, path in names if name != "consolidated.safetensors"
]
@@ -252,7 +253,7 @@ def _resident_bytes(model_dir: str, config: dict, quantize: bool) -> int:
model,
config,
spec,
- is_vlm = _loads_as_vision(config),
+ is_vlm = _builds_on_mlx_vlm(config),
)
return sum(
value.nbytes for _, value in tree_flatten(model.parameters()) if hasattr(value, "nbytes")
@@ -319,6 +320,18 @@ def _loads_as_vision(config: dict) -> bool:
return bool(_is_vlm(SimpleNamespace(**config)))
+def _builds_on_mlx_vlm(config: dict) -> bool:
+ """Whether mlx-vlm builds the weights: on a vision route, or for a text model unsloth_zoo builds
+ there because mlx-lm cannot; that one is still served as text."""
+ if _loads_as_vision(config):
+ return True
+ try:
+ from unsloth_zoo.mlx.loader import _mlx_vlm_only_text_model
+ return bool(_mlx_vlm_only_text_model(config, config.get("model_type") or ""))
+ except Exception:
+ return False
+
+
def _routes_to_diffusion(config: dict) -> bool:
"""Whether mlx-vlm would divert this load to a diffusion generator."""
try:
@@ -387,7 +400,7 @@ def _loader_config(arch, config: dict):
def _whole_model(config: dict, dtype):
if config.get("model_file"):
raise ValueError("this checkpoint carries its own model module")
- if _loads_as_vision(config):
+ if _builds_on_mlx_vlm(config):
from mlx_vlm.utils import get_model_and_args
arch = get_model_and_args(config)[0]
model = arch.Model(_loader_config(arch, config))
@@ -416,7 +429,7 @@ def _probe_models(config: dict, dtype):
return _build
- if not _loads_as_vision(config):
+ if not _builds_on_mlx_vlm(config):
from mlx_lm.models.cache import make_prompt_cache
from mlx_lm.utils import _get_classes
@@ -622,6 +635,37 @@ def _entries(cache, quantized = None) -> list:
return entries
+_DEFERRED_EVAL = threading.local()
+_DEFERRABLE_LOCK = threading.Lock()
+
+
+def _deferrable(evaluate):
+ def call(*args, **kwargs):
+ if getattr(_DEFERRED_EVAL, "depth", 0):
+ return None
+ return evaluate(*args, **kwargs)
+
+ call._unsloth_deferrable = True
+ return call
+
+
+def weightless_forward():
+ """Defer ``mx.eval`` / ``mx.async_eval`` on this thread while a weightless tower runs. Model code
+ evaluates mid-forward (mlx-vlm's per-layer ``async_eval``), which on random weights materializes
+ the whole model; the probes read only shapes, which need no evaluation."""
+ import mlx.core as mx
+
+ with _DEFERRABLE_LOCK:
+ if not getattr(mx.eval, "_unsloth_deferrable", False):
+ mx.eval, mx.async_eval = _deferrable(mx.eval), _deferrable(mx.async_eval)
+ _DEFERRED_EVAL.depth = getattr(_DEFERRED_EVAL, "depth", 0) + 1
+ try:
+ yield
+ finally:
+ _DEFERRED_EVAL.depth -= 1
+
+
def _probe(config: dict, dtype, n_tokens: int, kv_bits):
import mlx.core as mx
@@ -630,21 +674,22 @@ def _probe(config: dict, dtype, n_tokens: int, kv_bits):
layers = None
widths = (0,) * len(_config_widths({}))
failure = ValueError("no architecture module could build this config")
- for build, make_prompt_cache, model_class in _probe_models(config, dtype):
- model = build()
- cache = make_prompt_cache(model)
- try:
- model(mx.zeros((1, n_tokens), dtype = mx.int32), cache = cache)
- except Exception as exc:
- failure, cache = exc, None
- continue
- whole_prompt = _declines_to_chunk(model_class, model)
- layers = _tower_layers(model)
- widths = _tower_widths(model)
- break
- if cache is None:
- raise failure
- quantized = _quantize_like_runtime(cache, kv_bits)
+ with weightless_forward():
+ for build, make_prompt_cache, model_class in _probe_models(config, dtype):
+ model = build()
+ cache = make_prompt_cache(model)
+ try:
+ model(mx.zeros((1, n_tokens), dtype = mx.int32), cache = cache)
+ except Exception as exc:
+ failure, cache = exc, None
+ continue
+ whole_prompt = _declines_to_chunk(model_class, model)
+ layers = _tower_layers(model)
+ widths = _tower_widths(model)
+ break
+ if cache is None:
+ raise failure
+ quantized = _quantize_like_runtime(cache, kv_bits)
entries = _entries(cache, quantized)
if quantized is not None:
converted = [leaf for entry in quantized for leaf in _sub_caches(entry)]
@@ -957,7 +1002,7 @@ def _extra_tensors_refused(model, config: dict, extras: list) -> Optional[str]:
except ValueError as refusal:
return str(refusal)
- vision = _loads_as_vision(config)
+ vision = _builds_on_mlx_vlm(config)
table = _KNOWN_VLM_EXTRA_WEIGHT_FILTERS if vision else _KNOWN_MLX_LM_STRICT_FALLBACKS
rule = table.get(model_type)
if rule is not None and _message_matches_known_fallback(message, rule):
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index b5f0e5ee4d2..ffd31073444 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -4248,6 +4248,63 @@ class TestCacheWidthName:
]
+@_NEEDS_MLX
+def test_a_weightless_probe_never_runs_the_evaluations_its_model_asks_for(monkeypatch):
+ import threading
+
+ import mlx.core as mx
+ from mlx_lm.models.cache import KVCache
+
+ elsewhere = []
+
+ def evaluates():
+ before, other = mx.get_active_memory(), mx.ones((16, 1024, 1024))
+ mx.eval(other)
+ elsewhere.append(mx.get_active_memory() - before >= other.nbytes)
+
+ class Tower:
+ def __call__(self, inputs, cache):
+ weights = mx.ones((256, 1024, 1024)) # 1 GiB once evaluated
+ mx.async_eval(weights)
+ mx.eval(weights)
+ thread = threading.Thread(target = evaluates)
+ thread.start()
+ thread.join()
+ keys = mx.zeros((1, 2, inputs.shape[1], 8)) + weights[0, 0, 0]
+ return cache[0].update_and_fetch(keys, keys)[0]
+
+ monkeypatch.setattr(
+ mm, "_probe_models", lambda c, d: iter([(Tower, lambda m: [KVCache()], Tower)])
+ )
+ mx.reset_peak_memory()
+ assert len(mm._cache_plan({}, mx.bfloat16, None)[0]) == 1
+ assert mx.get_peak_memory() < 2**28
+ from core.inference.mlx_inference import mlx_kv_quant_is_refused
+
+ monkeypatch.setattr(mm, "_snapshot_config", lambda d: {})
+ mlx_kv_quant_is_refused("weightless", 8)
+ assert mx.get_peak_memory() < 2**28
+ assert elsewhere and all(elsewhere)
+ mx.eval(mx.ones((64, 1024, 1024)))
+ assert mx.get_peak_memory() >= 2**28
+
+
[email protected](not _HAVE_MLX, reason = "resolves an mlx-vlm architecture")
+def test_a_text_model_only_mlx_vlm_builds_is_sized_on_mlx_vlm_and_served_as_text(monkeypatch):
+ zoo, asked = _types.ModuleType("unsloth_zoo.mlx.loader"), []
+ zoo._mlx_vlm_only_text_model = lambda config, model_type: (
+ asked.append(model_type) or "engram_config" in config
+ )
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", zoo)
+ refused = {"model_type": "deepseek_v4", "engram_config": {}}
+ assert mm._builds_on_mlx_vlm(refused)
+ assert not mm._builds_on_mlx_vlm({"model_type": "deepseek_v4"})
+ assert asked == ["deepseek_v4", "deepseek_v4"]
+ assert not mm._loads_as_vision(refused)
+ _, make_cache, _ = next(mm._probe_models(refused, None))
+ assert make_cache.__module__ == "mlx_vlm.models.cache"
+
+
class TestKvBytes:
def test_a_hybrid_plan_is_far_cheaper_than_charging_every_layer_as_attention(self):
hybrid = [_RECURRENT] * 30 + [_GROWING] * 10
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 94e0d6528ff..1a434cd77ff 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -7189,6 +7189,7 @@ def test_the_width_probe_judges_the_tower_the_sizing_would_price(monkeypatch):
_probe_models = lambda *_a: iter(
[(lambda t = t: t, lambda tower: list(tower.entries), None) for t in towers]
),
+ weightless_forward = contextlib.nullcontext,
),
)
marks = len(rewound)