Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: price MLX loads in the memory panel and fit an unpinned context to available memory (#10287)

Long Yixing98284907b802
diff --git a/studio/backend/core/inference/memory_contract.py b/studio/backend/core/inference/memory_contract.py
index 1e9c6c04520..e567a8dfa49 100644
--- a/studio/backend/core/inference/memory_contract.py
+++ b/studio/backend/core/inference/memory_contract.py
@@ -89,6 +89,7 @@ def build_memory_estimate(
context_is_pinned: bool = True,
inherited_device_pin: bool = False,
spec_unpriced: bool = False,
+ context_fitted: Optional[int] = None,
moe_offload_unmodelled: bool = False,
gpu_bytes: Any = _UNSET,
compute_bytes: Any = _UNSET,
@@ -163,6 +164,7 @@ def build_memory_estimate(
else int(n_ctx or 0)
),
native_context = native_context,
+ context_fitted = context_fitted,
cache_type_kv = getattr(breakdown, "cache_type_kv", None),
n_parallel = int(getattr(breakdown, "n_parallel", 1) or 1),
layer_count = getattr(breakdown, "layer_count", None),
@@ -198,6 +200,7 @@ def project_estimate_memory_response(estimate: MemoryEstimate) -> dict:
"kv_estimable": estimate.kv_estimable,
"kv_on_gpu": estimate.kv_on_gpu,
"n_ctx": estimate.n_ctx,
+ "context_fitted": estimate.context_fitted,
"cache_type_kv": estimate.cache_type_kv,
"n_parallel": estimate.n_parallel,
"layer_count": estimate.layer_count,
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 63b2c828a6b..d52bcc0a81b 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -1595,11 +1595,81 @@ MLX_KV_BITS_CHOICES = (8, 6, 5, 4, 3, 2)
MLX_TURBOQUANT_BITS_CHOICES = (4, 3.5, 3, 2)
# Quantization group size; a head dim that is not a multiple makes mx.quantize raise.
MLX_KV_GROUP_SIZE = 64
+MLX_PREFILL_CHUNK_FALLBACK = 2048
+
+
+def _generation_step(*, vision: bool, drafted: bool):
+ """The function that would run this generation's prefill."""
+ if vision:
+ from mlx_vlm.generate.ar import generate_step as vlm_generate_step
+ return vlm_generate_step
+ if drafted:
+ from mlx_lm.generate import speculative_generate_step
+ return speculative_generate_step
+ from mlx_lm.generate import generate_step
+
+ return generate_step
+
+
+def _generation_default(setting: str, fallback: int, *, vision: bool, drafted: bool) -> int:
+ """A prefill setting as the runtime that would run generation defaults to it."""
+ import inspect
+ try:
+ found = (
+ inspect.signature(_generation_step(vision = vision, drafted = drafted))
+ .parameters[setting]
+ .default
+ )
+ if isinstance(found, bool) or not isinstance(found, int) or found <= 0:
+ raise ValueError(f"the runtime states no usable {setting}")
+ return found
+ except Exception as exc:
+ logger.debug("Cannot read the runtime's %s: %s", setting, exc)
+ return fallback
+
+
+def mlx_vlm_snapshot_store_available() -> bool:
+ """Whether a vision load can build the snapshot store, which decides two things about it."""
+ if _prompt_cache_max_bytes() <= 0:
+ return False
+ try:
+ from mlx_vlm.generate import GenerationResult
+ except Exception as exc:
+ logger.debug("Cannot tell whether mlx-vlm carries the snapshot grid: %s", exc)
+ return False
+ return hasattr(GenerationResult, "cached_tokens")
+
+
+def mlx_vlm_pins_the_prefill_step(config) -> bool:
+ """Whether EVERY request to this vision load prefills on the pinned grid."""
+ if not mlx_vlm_snapshot_store_available():
+ return False
+ return bool(MLXInferenceBackend._vlm_media_token_ids(config))
+
+
+def mlx_prefill_chunk(
+ *,
+ vision: bool = False,
+ drafted: bool = False,
+ config = None,
+) -> int:
+ """Most tokens one prefill step takes; the last step of a prompt may be shorter."""
+ if vision and mlx_vlm_pins_the_prefill_step(config):
+ return vlm_prefill_step()
+ return _generation_default(
+ "prefill_step_size", MLX_PREFILL_CHUNK_FALLBACK, vision = vision, drafted = drafted
+ )
+
+
# Surfaced with the resolved setting so an API client sees the reuse cost too.
MLX_KV_QUANT_NO_REUSE = (
"The installed mlx-lm cannot measure a quantized cache entry, so prompt-cache "
"reuse across turns is disabled while this is on."
)
+# The cap _configure_memory_limits installs, so a fitted context names a load MLX would allocate.
+MLX_MEMORY_LIMIT_FRACTION = 0.85
+
+
MLX_TURBOQUANT_TEXT_LOAD = (
"TurboQuant is not available for this model: it could not be loaded through the runtime "
"TurboQuant requires. The model is served without it."
@@ -1695,17 +1765,16 @@ def _mlx_rng_key_words():
what lets the rewind below stay unconditional. A key that reads but is not two words is not
the same as an unreadable one: the rewind no longer works on the installed mlx, so say so
before declining."""
- import mlx.core as mx
-
try:
+ import mlx.core as mx
words = mx.random.state[0].tolist()
except Exception:
return None
if len(words) != 2:
logger.warning(
"MLX exposes a %d-word random key; Unsloth can only rewind the "
- "two-word form, so the KV quantization probe will not restore the "
- "PRNG and sampling after a load may differ from an unprobed run.",
+ "two-word form, so sizing a load will not restore the PRNG and "
+ "sampling after a load may differ from an unsized run.",
len(words),
)
return None
@@ -1715,24 +1784,26 @@ def _mlx_rng_key_words():
def _as_uint32_pair(high, low):
"""Both words as uint32, or None if either is not a 32-bit word at all.
- mx.random.seed takes a uint64 and raises outside [0, 2**64); that raise would land in the
- probe's finally and replace the probe's own outcome. So the words are range-checked here rather
- than passed through, which is what lets the rewind below stay unguarded.
-
- Range-checked and not simply masked, though. A negative reads as the two's complement of the
- uint32 mlx stores, so reinterpreting it loses nothing. A value at or above 2**32 is not a 32-bit
- word under any reading, and masking it would turn a key we cannot represent into a plausible
- wrong one: (2**32, 0) would restore as (0, 0), the probe would report success, and sampling
- would silently diverge from an unprobed run. Decline instead, which is the outcome the caller
- already handles.
+ mx.random.seed takes a uint64 and raises outside [0, 2**64); that raise would
+ land in the rewind and replace the outcome of whatever it was guarding. So the
+ words are range-checked here rather than passed through, which is what lets the
+ rewind below stay unguarded.
+
+ Range-checked and not simply masked, though. A negative reads as the two's
+ complement of the uint32 mlx stores, so reinterpreting it loses nothing. A
+ value at or above 2**32 is not a 32-bit word under any reading, and masking
+ it would turn a key we cannot represent into a plausible wrong one: (2**32, 0)
+ would restore as (0, 0), the rewind would report success, and sampling would
+ silently diverge from an unsized run. Decline instead, which is the outcome
+ the caller already handles.
"""
converted = []
for word in (high, low):
if not -(2**31) <= word < 2**32:
logger.warning(
"MLX exposed a random key word of %d, which is not a 32-bit "
- "word; the KV quantization probe will not restore the PRNG and "
- "sampling after a load may differ from an unprobed run.",
+ "word; sizing a load will not restore the PRNG and sampling "
+ "after a load may differ from an unsized run.",
word,
)
return None
@@ -1747,16 +1818,26 @@ def _restore_mlx_rng_key(words):
unsigned 64-bit range. Unguarded on purpose: the range check in _as_uint32_pair removes the
only way this can raise, so there is no failure to swallow, and a blanket except here would
be a failure indistinguishable from an intentional no-op."""
- import mlx.core as mx
-
if words is None:
return
+ import mlx.core as mx
+
pair = _as_uint32_pair(int(words[0]), int(words[1]))
if pair is None:
return
mx.random.seed((pair[0] << 32) | pair[1])
+@contextmanager
+def mlx_rng_preserved():
+ """Leave the PRNG where it was found around weightless builds."""
+ rng_key = _mlx_rng_key_words()
+ try:
+ yield
+ finally:
+ _restore_mlx_rng_key(rng_key)
+
+
def _kv_entry_windowed(entry):
return any(getattr(entry, name, None) is not None for name in ("max_size", "window_size"))
@@ -1812,11 +1893,19 @@ def _quantize_kv_entries(entries, bits):
return entries
-def _kv_quant_probe(language_model, entries, bits):
+def _kv_quant_probe(
+ language_model,
+ entries,
+ bits,
+ evaluate = True,
+):
"""``(converted, skipped, failure, retainable)``: runs a token through the converted cache,
- since static proxies (declared head_dim, window spelling) proved wrong both ways."""
+ since static proxies (declared head_dim, window spelling) proved wrong both ways. Unevaluated,
+ the same graph is only built, which is what a weightless tower affords."""
import mlx.core as mx
+ settle = mx.eval if evaluate else (lambda _arrays: None)
+
targets, held = _kv_quant_targets(entries)
skipped = len(entries) - len(targets) - held
if not targets:
@@ -1828,21 +1917,21 @@ def _kv_quant_probe(language_model, entries, bits):
try:
try:
language_model(mx.array([[0]]), cache = entries)
- mx.eval([getattr(entry, "state", None) for entry in entries])
+ settle([getattr(entry, "state", None) for entry in entries])
except Exception as exc:
if is_metal_queue_dead(exc):
raise
return 0, 0, f"its cache could not be exercised ({type(exc).__name__})", True
try:
_quantize_kv_entries(entries, bits)
- mx.eval([entries[index].state for index in targets])
+ settle([entries[index].state for index in targets])
except Exception as exc:
if is_metal_queue_dead(exc):
raise
return 0, 0, f"MLX cannot quantize it ({type(exc).__name__})", True
try:
language_model(mx.array([[0]]), cache = entries)
- mx.eval([getattr(entry, "state", None) for entry in entries])
+ settle([getattr(entry, "state", None) for entry in entries])
except Exception as exc:
if is_metal_queue_dead(exc):
raise
@@ -2273,6 +2362,91 @@ def _kv_entry_is_bounded(entry, window):
return cap > getattr(entry, "keep", 0)
+def mlx_memory_budget(*, retains_history: bool = True) -> Optional[int]:
+ """Bytes a load's own footprint may occupy here, or None where Metal cannot say."""
+ import mlx.core as mx
+
+ if not mx.metal.is_available():
+ return None
+ recommended = mx.device_info().get("max_recommended_working_set_size")
+ if not recommended or recommended <= 0:
+ return None
+ budget = int(recommended * MLX_MEMORY_LIMIT_FRACTION)
+ if retains_history:
+ budget -= _prompt_cache_max_bytes(recommended / 1e9)
+ return budget if budget > 0 else None
+
+
+def _snapshot_dir(model, model_name: str) -> Optional[str]:
+ """Where the files this load actually opened are, or None if none are local."""
+ resolved = getattr(model, "_src_path", None)
+ if resolved and os.path.isdir(str(resolved)):
+ return str(resolved)
+ if os.path.isdir(model_name):
+ return model_name
+ from utils.utils import hf_cache_snapshot_dir
+
+ snapshot = hf_cache_snapshot_dir(model_name)
+ return str(snapshot) if snapshot is not None else None
+
+
+def mlx_fit_to_memory(
+ model_dir,
+ ceiling,
+ *,
+ load_in_4bit: bool,
+ retains_history: bool,
+ kv_bits = None,
+) -> Optional[int]:
+ """The context a load of *model_dir* is held to, priced at the KV width its cache takes."""
+ budget = mlx_memory_budget(retains_history = retains_history)
+ if budget is None or not ceiling or not model_dir:
+ return None
+ with mlx_rng_preserved():
+ try:
+ from core.inference.mlx_memory import mlx_fit_context
+
+ fitted = mlx_fit_context(
+ model_dir,
+ budget_bytes = budget,
+ max_ctx = int(ceiling),
+ kv_bits = kv_bits,
+ load_in_4bit = load_in_4bit,
+ )
+ logger.debug("MLX fit for %s: %s tokens under %.1f GB", model_dir, fitted, budget / 1e9)
+ return fitted
+ except Exception as exc:
+ logger.debug("MLX context fit unavailable for %s: %s", model_dir, exc)
+ return None
+
+
+def _fitted_context(
+ model,
+ model_name: str,
+ ceiling,
+ *,
+ load_in_4bit: bool,
+ retains_history: bool,
+ kv_bits = None,
+ is_vlm = False,
+):
+ """The fit for a resident model and the eligibility verdict it was priced under."""
+ try:
+ model_dir = _snapshot_dir(model, model_name)
+ except Exception as exc:
+ logger.debug("MLX snapshot unavailable for %s: %s", model_name, exc)
+ return None, None
+ verdict = None if kv_bits is None else _kv_quant_eligibility(model, is_vlm, kv_bits)
+ fitted = mlx_fit_to_memory(
+ model_dir,
+ ceiling,
+ load_in_4bit = load_in_4bit,
+ retains_history = retains_history,
+ kv_bits = kv_bits if verdict is None or verdict[0] in ("full", "partial") else None,
+ )
+ return fitted, verdict
+
+
def _kv_window_enforced(model, is_vlm, window):
"""Whether a cache built at *window* is bounded in every layer, None where none could be built
to judge: both runtimes defer to a model's own make_cache and ignore max_kv_size there."""
@@ -2286,6 +2460,69 @@ def _kv_window_enforced(model, is_vlm, window):
return None
+def _asking_the_architecture(model_dir, question):
+ """Put *question* to the tower the sizing would price, built weightless from the checkpoint."""
+ with mlx_rng_preserved():
+ try:
+ import mlx.core as mx
+
+ from core.inference.mlx_memory import (
+ _PROBE_SHORT,
+ _probe_models,
+ _runtime_dtype,
+ _snapshot_config,
+ )
+
+ config = _snapshot_config(model_dir)
+ if config is None:
+ return None
+ for build, make_prompt_cache, _ in _probe_models(config, _runtime_dtype()):
+ model = build()
+ try:
+ model(
+ mx.zeros((1, _PROBE_SHORT), dtype = mx.int32),
+ cache = make_prompt_cache(model),
+ )
+ except Exception:
+ continue
+ return question(model, make_prompt_cache)
+ except Exception as exc:
+ logger.debug("MLX architecture probe unavailable for %s: %s", model_dir, exc)
+ return None
+
+
+# Slider steps resend the width, and each answer is a weightless build.
+_kv_refusal_cache: dict = {}
+_kv_refusal_lock = threading.Lock()
+
+
+def mlx_kv_quant_is_refused(model_dir, bits) -> bool:
+ """Whether a load would keep this checkpoint's cache at full width despite *bits*, by the
+ eligibility probe's own rule (nothing to convert, or a converted cache the model cannot
+ attend over)."""
+
+ def refused(model, make_prompt_cache):
+ converted, _skipped, failure, _retainable = _kv_quant_probe(
+ model, make_prompt_cache(model), bits, evaluate = False
+ )
+ return failure is not None or not converted
+
+ try:
+ from core.inference.mlx_memory import _checkpoint_fingerprint
+ key = (os.path.realpath(model_dir), bits, _checkpoint_fingerprint(model_dir))
+ except Exception:
+ return _asking_the_architecture(model_dir, refused) is True
+ with _kv_refusal_lock:
+ if key in _kv_refusal_cache:
+ return _kv_refusal_cache[key]
+ answer = _asking_the_architecture(model_dir, refused) is True
+ with _kv_refusal_lock:
+ _kv_refusal_cache[key] = answer
+ while len(_kv_refusal_cache) > 32:
+ _kv_refusal_cache.pop(next(iter(_kv_refusal_cache)))
+ return answer
+
+
def _turboquant_reserved_positions(count, bits):
"""Entry positions mlx-vlm reserves, learned from a control cache rather than a fixed number."""
@@ -2377,7 +2614,12 @@ def _turboquant_status(
mx.clear_cache()
-def _kv_quant_status(requested_bits, model, is_vlm):
+def _kv_quant_status(
+ requested_bits,
+ model,
+ is_vlm,
+ eligibility = None,
+):
"""Resolve a requested bit width against this model into a status dict."""
status = {
"requested_kv_bits": requested_bits,
@@ -2388,7 +2630,9 @@ def _kv_quant_status(requested_bits, model, is_vlm):
}
if requested_bits is None:
return status
- verdict, reason, retainable = _kv_quant_eligibility(model, is_vlm, requested_bits)
+ verdict, reason, retainable = eligibility or _kv_quant_eligibility(
+ model, is_vlm, requested_bits
+ )
status["eligibility"] = verdict
status["reason"] = reason
if verdict in ("full", "partial"):
@@ -3941,7 +4185,7 @@ class MLXInferenceBackend:
if not rec_bytes or rec_bytes <= 0:
return
rec_gb = rec_bytes / 1e9
- memory_limit_gb = rec_gb * 0.85
+ memory_limit_gb = rec_gb * MLX_MEMORY_LIMIT_FRACTION
wired_limit_gb = min(rec_gb, memory_limit_gb)
mx.set_memory_limit(int(memory_limit_gb * 1e9))
mx.set_wired_limit(int(wired_limit_gb * 1e9))
@@ -3997,11 +4241,20 @@ class MLXInferenceBackend:
)
return enforced
- def _resolve_kv_policy(self, is_vlm, kv_bits, max_seq_length, served):
+ def _resolve_kv_policy(
+ self,
+ is_vlm,
+ kv_bits,
+ max_seq_length,
+ served,
+ fitted = False,
+ eligibility = None,
+ ):
"""The quantization status, cache window and per-request budget this load will run with.
- A rotating cache cannot be quantized, so with kv_bits an enforceable pin becomes a budget."""
- pinned = _positive_int(max_seq_length) is not None
+ A rotating cache cannot be quantized, so with kv_bits an enforceable pin or fitted window
+ becomes a budget."""
+ pinned = _positive_int(max_seq_length) is not None or fitted
# Tri-state: True bounded, False confirmed unbounded, None unjudgeable. Only True installs a bound; the other
# two stay apart so a client can tell them apart.
confirmed = self._kv_cache_window_enforceable(served)
@@ -4011,7 +4264,12 @@ class MLXInferenceBackend:
self._model, kv_bits, getattr(self, "_turboquant_refusal", "")
)
else:
- quant = _kv_quant_status(_normalize_mlx_kv_bits(kv_bits), self._model, is_vlm)
+ quant = _kv_quant_status(
+ _normalize_mlx_kv_bits(kv_bits),
+ self._model,
+ is_vlm,
+ eligibility,
+ )
if quant["kv_bits"] is None and enforceable:
logger.info("MLX KV cache limited to %d tokens", int(served))
return quant, int(served), True, None
@@ -4180,6 +4438,26 @@ class MLXInferenceBackend:
_served_ctx, _native_ctx, _max_ctx = self._resolve_context_lengths(
self._model, max_seq_length
)
+ _priceable = not (is_distributed or is_lora or dtype is not None)
+ # TurboQuant's layout is not one the planner prices, and a refused one stays full width:
+ # fit either at full width, which only ever under-promises.
+ _requested_bits = None if self._turboquant else _normalize_mlx_kv_bits(kv_bits)
+ _fitted_ctx, _eligibility = (
+ (None, None)
+ if not _priceable or _positive_int(max_seq_length) is not None
+ else _fitted_context(
+ self._model,
+ model_name,
+ _served_ctx,
+ load_in_4bit = load_in_4bit,
+ retains_history = not use_vlm or mlx_vlm_snapshot_store_available(),
+ kv_bits = _requested_bits,
+ is_vlm = use_vlm,
+ )
+ )
+ if _fitted_ctx:
+ logger.info("MLX context fitted to %d tokens from %d", _fitted_ctx, _served_ctx or 0)
+ _served_ctx = _fitted_ctx
self._served_context = _served_ctx
# Classified now so a cache the model cannot attend over is refused here, not on a token.
(
@@ -4187,7 +4465,14 @@ class MLXInferenceBackend:
self._kv_cache_window,
_ctx_enforced,
self._kv_context_budget,
- ) = self._resolve_kv_policy(use_vlm, kv_bits, max_seq_length, _served_ctx)
+ ) = self._resolve_kv_policy(
+ use_vlm,
+ kv_bits,
+ max_seq_length,
+ _served_ctx,
+ fitted = _fitted_ctx is not None,
+ eligibility = _eligibility,
+ )
if self._kv_quant["kv_bits"] is not None:
logger.info(
"MLX KV cache quantization: %s-bit (%s eligibility)",
@@ -4260,13 +4545,13 @@ class MLXInferenceBackend:
# Parity with llama.cpp's requested_n_ctx: the served window cannot say whether anything was asked for,
# and that decides reuse and "pinned".
"requested_context_length": _positive_int(max_seq_length) or 0,
- # Nothing measures the machine, so window and ceiling coincide.
"native_context_length": _native_ctx,
"max_context_length": _max_ctx,
# Whether the served window actually bounds the cache: True confirmed on a real cache, False confirmed
# unbounded, None nothing could be built to judge. Without it the API reports a limit a client cannot tell
# from an enforced one.
"context_length_enforced": _ctx_enforced,
+ "context_length_fitted": _fitted_ctx,
"context_unbounded_when_batched": self._vision_batch_is_available(),
"mlx_context_budget": self._kv_context_budget,
"mlx_kv_bits": self._kv_quant["kv_bits"],
diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.py
new file mode 100644
index 00000000000..35665079252
--- /dev/null
+++ b/studio/backend/core/inference/mlx_memory.py
@@ -0,0 +1,1165 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""What an MLX load would occupy, priced before anything is loaded."""
+
+from __future__ import annotations
+
+import glob
+import functools
+import json
+import os
+import posixpath
+import threading
+from collections import OrderedDict
+from dataclasses import dataclass
+from typing import Optional
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+__all__ = [
+ "MLX_FIT_MIN_CONTEXT",
+ "MLX_KV_BLOCK",
+ "MLX_PREFILL_CHUNK",
+ "MlxMemoryBreakdown",
+ "mlx_fit_context",
+ "mlx_memory_breakdown",
+ "mlx_shard_files",
+ "mlx_weight_bytes",
+]
+
+MLX_KV_BLOCK = 256
+
+_PROBE_SHORT = 8
+_PROBE_LONG = 40
+
+MLX_PREFILL_CHUNK = 2048
+
+MLX_FIT_MIN_CONTEXT = 4096
+_KV_GROUP_SIZE = 64
+
+_ATTENTION_LIVE = 3.5
+_COMPUTE_BASE_BYTES = int(0.64 * 1024**3)
+
+_RECURRENT_DTYPE_SIZE = 4
+
+_QUANT_SCORE_DTYPE_SIZE = 4
+
+_QUANT_SCORE_LIVE = 1.0
+
+_WIDTH_NAMES = {"float64": "f64", "float32": "f32", "bfloat16": "bf16", "float16": "f16"}
+
+_CACHE_BOUND_ATTRS = ("max_size", "chunk_size")
+
+_MAX_SAFETENSORS_HEADER = 100_000_000
+
+
+@dataclass
+class MlxMemoryBreakdown:
+ weights_bytes: int
+ kv_bytes: int
+ compute_bytes: int
+ total_bytes: int
+ gpu_bytes: int
+ n_ctx: int
+ layer_count: Optional[int] = None
+ cache_type_kv: Optional[str] = None
+ kv_estimable: bool = True
+ kv_on_gpu: bool = True
+ n_parallel: int = 1
+ gpu_layers: Optional[int] = None
+
+
+def _snapshot_config(model_dir: str) -> Optional[dict]:
+ path = os.path.join(model_dir, "config.json")
+ try:
+ with open(path, "r", encoding = "utf-8") as handle:
+ config = json.load(handle)
+ except Exception as exc:
+ logger.debug("MLX estimate could not read %s: %s", path, exc)
+ return None
+ if not isinstance(config, dict):
+ return None
+ return _as_the_loader_reads_it(config)
+
+
+def _as_the_loader_reads_it(config: dict) -> dict:
+ import contextlib
+ import io
+
+ try:
+ from unsloth_zoo.mlx.loader import _mlx_vlm_config_override_data
+ except Exception:
+ return config
+ try:
+ with contextlib.redirect_stdout(io.StringIO()):
+ patched = _mlx_vlm_config_override_data(config)
+ except Exception:
+ return config
+ return patched if isinstance(patched, dict) else config
+
+
+def _escapes_snapshot(shard: str) -> bool:
+ # Lexical, not realpath: hub snapshot files are symlinks into ../../blobs by design.
+ norm = posixpath.normpath(shard.replace("\\", "/"))
+ return norm.startswith("/") or norm.split("/")[0] == ".." or norm[1:2] == ":"
+
+
+def _indexed_shards(model_dir: str) -> list:
+ index = os.path.join(model_dir, "model.safetensors.index.json")
+ if not os.path.isfile(index):
+ return []
+ try:
+ with open(index, encoding = "utf-8") as handle:
+ weight_map = json.load(handle).get("weight_map", {})
+ except (ValueError, OSError):
+ return []
+ if not isinstance(weight_map, dict):
+ raise ValueError("model.safetensors.index.json has no weight map")
+ if any(not isinstance(shard, str) for shard in weight_map.values()):
+ raise ValueError("model.safetensors.index.json names a non-string shard")
+ if any(_escapes_snapshot(shard) for shard in weight_map.values()):
+ raise ValueError("model.safetensors.index.json names a shard outside the snapshot")
+ named = sorted(set(weight_map.values()))
+ return [
+ os.path.join(model_dir, shard)
+ for shard in named
+ if os.path.isfile(os.path.join(model_dir, shard))
+ ]
+
+
+def mlx_shard_files(model_dir: str, config: Optional[dict] = None) -> list:
+ """The shards the package that would load this model actually reads."""
+ names = [
+ (os.path.basename(path), path)
+ for path in glob.glob(os.path.join(model_dir, "*.safetensors"))
+ ]
+ if config is None:
+ return [path for _, path in names]
+ if _loads_as_vision(config):
+ return _indexed_shards(model_dir) or [
+ path for name, path in names if name != "consolidated.safetensors"
+ ]
+ return [path for name, path in names if name.startswith("model")]
+
+
+def _shard_bytes(model_dir: str, config: Optional[dict] = None) -> int:
+ total = 0
+ for shard in mlx_shard_files(model_dir, config):
+ try:
+ total += os.path.getsize(os.path.realpath(shard))
+ except OSError:
+ continue
+ return total
+
+
+_SAFETENSORS_DTYPES = {
+ "F64": "float32",
+ "F32": "float32",
+ "F16": "float16",
+ "BF16": "bfloat16",
+ "I64": "int64",
+ "I32": "int32",
+ "I16": "int16",
+ "I8": "int8",
+ "U64": "uint64",
+ "U32": "uint32",
+ "U16": "uint16",
+ "U8": "uint8",
+ "BOOL": "bool_",
+}
+
+
+def _checkpoint_tensors(model_dir: str, config: Optional[dict], dtype):
+ import mlx.core as mx
+
+ tensors = {}
+ for shard in mlx_shard_files(model_dir, config):
+ with open(shard, "rb") as handle:
+ length = int.from_bytes(handle.read(8), "little")
+ limit = min(_MAX_SAFETENSORS_HEADER, os.fstat(handle.fileno()).st_size - 8)
+ if not 0 < length <= limit:
+ raise ValueError(f"{shard} declares a {length}-byte safetensors header")
+ header = json.loads(handle.read(length))
+ for name, meta in header.items():
+ if name == "__metadata__":
+ continue
+ stored = getattr(mx, _SAFETENSORS_DTYPES.get(meta["dtype"], "float32"))
+ if stored in (mx.float32, mx.float16, mx.bfloat16):
+ stored = dtype
+ tensors[name] = mx.zeros(tuple(meta["shape"]), dtype = stored)
+ return tensors
+
+
+def _resident_bytes(model_dir: str, config: dict, quantize: bool) -> int:
+ """Parameter bytes the load leaves resident, at the width it holds them."""
+ import contextlib
+ import io
+
+ from mlx.utils import tree_flatten
+ from unsloth_zoo.mlx.loader import (
+ _apply_mlx_quantization,
+ _resolve_mlx_quantization_spec,
+ )
+
+ spec = (
+ None
+ if not quantize
+ else _resolve_mlx_quantization_spec(
+ load_in_4bit = True,
+ load_in_8bit = False,
+ load_in_16bit = False,
+ load_in_fp8 = False,
+ load_in_mxfp4 = False,
+ load_in_nvfp4 = False,
+ full_finetuning = False,
+ q_bits = None,
+ q_group_size = None,
+ q_mode = None,
+ mlx_quantization_config = None,
+ quantization_config = None,
+ quant_predicate = None,
+ quantize_modules = None,
+ force_requantize = False,
+ )
+ )
+ if quantize and not spec.enabled:
+ raise ValueError("the loader would quantize nothing")
+
+ dtype = _runtime_dtype()
+ model = _whole_model(config, dtype)
+ tensors = _checkpoint_tensors(model_dir, config, dtype)
+ if not tensors:
+ raise ValueError("this checkpoint ships no tensors")
+ if hasattr(model, "sanitize"):
+ tensors = model.sanitize(tensors)
+ model.load_weights(list(tensors.items()), strict = False)
+ orphaned = [name for name, _ in tree_flatten(model.parameters()) if name not in tensors]
+ if orphaned:
+ raise ValueError(
+ f"{len(orphaned)} parameters this checkpoint does not supply, "
+ f"beginning {orphaned[0]}"
+ )
+ if quantize:
+ with contextlib.redirect_stdout(io.StringIO()):
+ model, _ = _apply_mlx_quantization(
+ model,
+ config,
+ spec,
+ is_vlm = _loads_as_vision(config),
+ )
+ return sum(
+ value.nbytes for _, value in tree_flatten(model.parameters()) if hasattr(value, "nbytes")
+ )
+
+
+def mlx_weight_bytes(
+ model_dir: str,
+ config: Optional[dict] = None,
+ load_in_4bit: bool = False,
+) -> int:
+ """Weight bytes an MLX load would leave resident."""
+ shards = _shard_bytes(model_dir, config)
+ if not config:
+ return shards
+ if config.get("quantization") or config.get("quantization_config"):
+ return shards
+ try:
+ return _resident_bytes(model_dir, config, quantize = load_in_4bit)
+ except Exception as exc:
+ logger.debug("MLX estimate priced %s as stored: %s", model_dir, exc)
+ return shards
+
+
+def _runtime_dtype():
+ import mlx.core as mx
+ chip = mx.device_info().get("device_name", "") or ""
+ return mx.float16 if chip.startswith(("Apple M1", "Apple M2")) else mx.bfloat16
+
+
+def _generation_settings(config: dict) -> tuple:
+ """``(prefill chunk, kv group size)`` a load of this model runs at."""
+ try:
+ from core.inference.mlx_inference import MLX_KV_GROUP_SIZE, mlx_prefill_chunk
+ except Exception as exc:
+ logger.debug("MLX estimate cannot reach the loader's generation settings: %s", exc)
+ return MLX_PREFILL_CHUNK, _KV_GROUP_SIZE
+ vision = _loads_as_vision(config)
+ if vision and _routes_to_diffusion(config):
+ raise ValueError("mlx-vlm would divert this to a diffusion generator")
+ return (
+ mlx_prefill_chunk(vision = vision, config = config if vision else None),
+ MLX_KV_GROUP_SIZE,
+ )
+
+
+def _loads_as_vision(config: dict) -> bool:
+ from types import SimpleNamespace
+ try:
+ from utils.models.model_config import _is_vlm
+ except Exception:
+ return any(
+ key in config
+ for key in ("vision_config", "img_processor", "image_token_index", "projector_config")
+ )
+ return bool(_is_vlm(SimpleNamespace(**config)))
+
+
+def _routes_to_diffusion(config: dict) -> bool:
+ """Whether mlx-vlm would divert this load to a diffusion generator."""
+ try:
+ from mlx_vlm.generate.diffusion import is_diffusion_model
+ from mlx_vlm.utils import get_model_and_args
+ except ImportError:
+ return False
+ arch = get_model_and_args(config)[0]
+ loader_config = _loader_config(arch, config)
+ if (
+ getattr(loader_config, "canvas_length", None) is None
+ and getattr(loader_config, "mask_token_id", None) is None
+ ):
+ return False
+ return bool(is_diffusion_model(arch.Model(loader_config), {}))
+
+
+def _declines_to_chunk(model_class, model) -> bool:
+ """Whether this model's runtime feeds the whole prompt to a single step."""
+ return bool(
+ getattr(model_class, "no_chunked_prefill", False)
+ or getattr(model, "no_chunked_prefill", False)
+ )
+
+
+def _tower_needs_parent(arch) -> bool:
+ import inspect
+
+ tower = getattr(arch, "LanguageModel", None)
+ if tower is None or tower.__init__ is object.__init__:
+ return True
+ try:
+ parameters = inspect.signature(tower.__init__).parameters
+ except (TypeError, ValueError):
+ return False
+ required = [
+ name
+ for name, value in parameters.items()
+ if name != "self"
+ and value.default is inspect.Parameter.empty
+ and value.kind
+ in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+ ]
+ return len(required) > 1
+
+
+def _loader_config(arch, config: dict):
+ """Config resolved as the loader does (``from_dict`` leaves nested sub-configs as dicts)."""
+ from mlx_vlm.utils import update_module_configs
+
+ config = dict(config)
+ config.setdefault("text_config", config.pop("llm_config", {}))
+ config.setdefault("vision_config", {})
+ config.setdefault("audio_config", {})
+
+ model_config = arch.ModelConfig.from_dict(config)
+ model_config = update_module_configs(
+ model_config,
+ arch,
+ config,
+ ["text", "vision", "perceiver", "projector", "audio"],
+ )
+ return model_config
+
+
+def _whole_model(config: dict, dtype):
+ if config.get("model_file"):
+ raise ValueError("this checkpoint carries its own model module")
+ if _loads_as_vision(config):
+ from mlx_vlm.utils import get_model_and_args
+ arch = get_model_and_args(config)[0]
+ model = arch.Model(_loader_config(arch, config))
+ else:
+ from mlx_lm.utils import _get_classes
+ model_class, args_class = _get_classes(config)
+ model = model_class(args_class.from_dict(config))
+ model.set_dtype(dtype)
+ model.eval()
+ return model
+
+
+def _probe_models(config: dict, dtype):
+ """Language towers to try, in order, with the cache builder and class owning each."""
+ if not config.get("model_type"):
+ raise ValueError("config.json declares no model_type")
+ if config.get("model_file"):
+ raise ValueError("this checkpoint carries its own model module")
+
+ def prepared(build):
+ def _build():
+ model = build()
+ model.set_dtype(dtype)
+ model.eval()
+ return model
+
+ return _build
+
+ if not _loads_as_vision(config):
+ from mlx_lm.models.cache import make_prompt_cache
+ from mlx_lm.utils import _get_classes
+
+ try:
+ model_class, args_class = _get_classes(config)
+ except Exception as exc:
+ raise ValueError(f"mlx-lm cannot resolve this architecture: {exc}")
+ yield (
+ prepared(lambda: model_class(args_class.from_dict(config))),
+ make_prompt_cache,
+ model_class,
+ )
+ return
+
+ from mlx_vlm.models.cache import make_prompt_cache
+ from mlx_vlm.utils import get_model_and_args
+
+ try:
+ arch = get_model_and_args(config)[0]
+ except Exception as exc:
+ raise ValueError(f"mlx-vlm cannot resolve this architecture: {exc}")
+
+ if not _tower_needs_parent(arch):
+
+ def _tower():
+ text_config = getattr(_loader_config(arch, config), "text_config", None)
+ if text_config is None:
+ raise ValueError("this architecture declares no text config")
+ return arch.LanguageModel(text_config)
+
+ yield prepared(_tower), make_prompt_cache, getattr(arch, "Model", None)
+
+ def _whole_tower():
+ whole = arch.Model(_loader_config(arch, config))
+ return getattr(whole, "language_model", whole)
+
+ yield prepared(_whole_tower), make_prompt_cache, getattr(arch, "Model", None)
+
+
+def _quantize_like_runtime(cache, kv_bits: Optional[int]):
+ """The converted cache a load ends up with, or None if nothing in it converts."""
+ if not kv_bits:
+ return None
+ from core.inference.mlx_inference import _quantize_kv_entries
+
+ try:
+ converted = _quantize_kv_entries(list(cache), kv_bits)
+ except Exception:
+ return None
+ return converted if any(new is not old for new, old in zip(converted, cache)) else None
+
+
+def _sub_caches(entry):
+ inner = getattr(entry, "caches", None)
+ if not isinstance(inner, (list, tuple)):
+ return (entry,)
+ return tuple(sub for cache in inner for sub in _sub_caches(cache))
+
+
+def _block(entry) -> int:
+ """Allocation granularity of the storage behind ``entry``, in tokens."""
+ step = getattr(entry, "step", None)
+ return step if isinstance(step, int) and step > 0 else 1
+
+
+def _bound_spec(entry):
+ for name in _CACHE_BOUND_ATTRS:
+ value = getattr(entry, name, None)
+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:
+ return type(entry), name, value
+ return None
+
+
+def _entry_bytes(entry) -> int:
+ import mlx.core as mx
+
+ state = entry.state
+ if isinstance(state, mx.array):
+ state = (state,)
+ elif not isinstance(state, (list, tuple)):
+ return 0
+ total = 0
+ for item in state:
+ if isinstance(item, mx.array):
+ total += item.size * item.dtype.size
+ elif isinstance(item, (list, tuple)):
+ total += sum(sub.size * sub.dtype.size for sub in item if isinstance(sub, mx.array))
+ return total
+
+
+def _width_name(dtype) -> str:
+ name = str(dtype).rsplit(".", 1)[-1]
+ return _WIDTH_NAMES.get(name, name)
+
+
+def _cache_width_name(
+ plan, kv_bits, converted: bool, full_width: str, n_ctx: int, prefill_chunk: int
+) -> str:
+ """How to caption the cache, from the width each growing entry ends up at."""
+ charged: dict = {}
+ for slot in plan:
+ if slot["slope"] <= 0:
+ continue
+ quantized = converted and slot["converts"]
+ name = f"{kv_bits}-bit" if quantized else full_width
+ const, slope = ("quant_const", "quant_slope") if quantized else ("const", "slope")
+ charged[name] = (
+ charged.get(name, 0.0)
+ + slot[const]
+ + slot[slope] * _held_tokens(slot, n_ctx, prefill_chunk, True)
+ )
+ if not charged:
+ return full_width
+ return "/".join(sorted(charged, key = lambda name: charged[name], reverse = True))
+
+
+def _conv_width(entry) -> int:
+ """Total channel width of a linear-attention layer's convolution states."""
+ import mlx.core as mx
+
+ state = entry.state
+ if not isinstance(state, (list, tuple)):
+ return 0
+ return sum(
+ int(item.shape[-1]) for item in state if isinstance(item, mx.array) and item.ndim == 3
+ )
+
+
+def _tower_layers(model) -> Optional[int]:
+ seen = model
+ for _ in range(3):
+ layers = getattr(seen, "layers", None)
+ if isinstance(layers, list) and layers:
+ return len(layers)
+ seen = getattr(seen, "model", None) or getattr(seen, "language_model", None)
+ if seen is None:
+ return None
+ return None
+
+
+def _tower_widths(model):
+ """The widths in ``_config_widths``' order as the tower was BUILT, or zeros."""
+ node, depth = model, 0
+ while node is not None and depth < 3:
+ for name in ("args", "config"):
+ resolved = getattr(node, name, None)
+ hidden = (
+ 0 if resolved is None else _config_width(getattr(resolved, "hidden_size", None))
+ )
+ if hidden > 0:
+ heads = _config_width(getattr(resolved, "num_attention_heads", None))
+ return (
+ hidden,
+ _config_width(getattr(resolved, "intermediate_size", None))
+ or _config_width(getattr(resolved, "moe_intermediate_size", None)),
+ heads,
+ _config_width(getattr(resolved, "num_key_value_heads", None)) or heads,
+ _config_width(getattr(resolved, "head_dim", None))
+ or (hidden // heads if heads else 0),
+ _config_width(getattr(resolved, "v_head_dim", None)),
+ )
+ node = getattr(node, "model", None) or getattr(node, "language_model", None)
+ depth += 1
+ return (0,) * len(_config_widths({}))
+
+
+def _config_widths(config: dict):
+ text = config.get("text_config") or config
+ hidden = _config_width(text.get("hidden_size"))
+ heads = _config_width(text.get("num_attention_heads"))
+ return (
+ hidden,
+ _config_width(text.get("intermediate_size"))
+ or _config_width(text.get("moe_intermediate_size")),
+ heads,
+ _config_width(text.get("num_key_value_heads")) or heads,
+ _config_width(text.get("head_dim")) or (hidden // heads if heads else 0),
+ _config_width(text.get("v_head_dim")),
+ )
+
+
+def _probe(config: dict, dtype, n_tokens: int, kv_bits):
+ import mlx.core as mx
+
+ cache = None
+ whole_prompt = False
+ layers = None
+ widths = (0,) * len(_config_widths({}))
+ failure = ValueError("no architecture module could build this config")
+ for build, make_prompt_cache, model_class in _probe_models(config, dtype):
+ model = build()
+ cache = make_prompt_cache(model)
+ try:
+ model(mx.zeros((1, n_tokens), dtype = mx.int32), cache = cache)
+ except Exception as exc:
+ failure, cache = exc, None
+ continue
+ whole_prompt = _declines_to_chunk(model_class, model)
+ layers = _tower_layers(model)
+ widths = _tower_widths(model)
+ break
+ if cache is None:
+ raise failure
+ quantized = _quantize_like_runtime(cache, kv_bits)
+ entries = []
+ for index, entry in enumerate(cache):
+ converts = quantized is not None and quantized[index] is not entry
+ for leaf in _sub_caches(entry):
+ entries.append(
+ {
+ "bytes": _entry_bytes(leaf),
+ "quant_bytes": _entry_bytes(leaf),
+ "bound_spec": _bound_spec(leaf),
+ "conv_width": _conv_width(leaf),
+ "block": _block(leaf),
+ "converts": converts,
+ }
+ )
+ if quantized is not None:
+ converted = [leaf for entry in quantized for leaf in _sub_caches(entry)]
+ if len(converted) == len(entries):
+ for slot, leaf in zip(entries, converted):
+ slot["quant_bytes"] = _entry_bytes(leaf)
+ return {
+ "entries": entries,
+ "quantized": quantized is not None,
+ "whole_prompt": whole_prompt,
+ "layers": layers,
+ "widths": widths,
+ }
+
+
+def _cache_plan(config: dict, dtype, kv_bits):
+ """Per-entry ``const + slope * T``, solved from two lazy probes."""
+ near = _probe(config, dtype, _PROBE_SHORT, kv_bits)
+ far = _probe(config, dtype, _PROBE_LONG, kv_bits)
+ if not near["entries"] or len(near["entries"]) != len(far["entries"]):
+ raise ValueError("cache probe returned no comparable entries")
+ span = _PROBE_LONG - _PROBE_SHORT
+
+ def solve(key):
+ return [
+ (n[key] - (f[key] - n[key]) / span * _PROBE_SHORT, (f[key] - n[key]) / span)
+ for n, f in zip(near["entries"], far["entries"])
+ ]
+
+ plan = []
+ for slot, (const, slope), (q_const, q_slope) in zip(
+ near["entries"], solve("bytes"), solve("quant_bytes")
+ ):
+ plan.append(
+ {
+ "const": const,
+ "slope": slope,
+ "quant_const": q_const,
+ "quant_slope": q_slope,
+ "bound_spec": slot["bound_spec"],
+ "conv_width": slot["conv_width"],
+ "block": slot["block"],
+ "converts": slot["converts"] and near["quantized"],
+ }
+ )
+ return (
+ plan,
+ 0 if near["quantized"] else None,
+ {
+ "whole_prompt": near["whole_prompt"],
+ "layers": near["layers"],
+ "widths": near["widths"],
+ },
+ )
+
+
[email protected]_cache(maxsize = 256)
+def _bounded_peak(cache_type, attribute: str, value: int, n_ctx: int, prefill_chunk: int) -> int:
+ """Largest slot count a bounded cache reaches, by driving the real class."""
+ import mlx.core as mx
+
+ cache = cache_type(**{attribute: value})
+ settled = value + 4 * (prefill_chunk + (getattr(cache, "step", 0) or MLX_KV_BLOCK))
+ n_ctx = min(n_ctx, settled)
+ peak, processed = 0, 0
+
+ def call(width):
+ nonlocal peak
+ trim = getattr(cache, "maybe_trim_front", None)
+ if callable(trim):
+ trim()
+ cache.update_and_fetch(
+ mx.zeros((1, 1, width, 1), dtype = mx.bfloat16),
+ mx.zeros((1, 1, width, 1), dtype = mx.bfloat16),
+ )
+ keys = getattr(cache, "keys", None)
+ peak = max(peak, 0 if keys is None else int(keys.shape[2]))
+
+ while n_ctx - processed > 1:
+ width = min(prefill_chunk, n_ctx - processed - 1)
+ call(width)
+ processed += width
+ call(1)
+ call(1)
+ return peak
+
+
+def _held_tokens(
+ entry,
+ n_ctx: int,
+ prefill_chunk: int,
+ decoding: bool = True,
+) -> int:
+ tokens = n_ctx
+ block = entry.get("block") or MLX_KV_BLOCK
+ if entry["slope"] > 0:
+ tokens += 1 if decoding else 0
+ if block > 1:
+ tokens = -(-tokens // block) * block
+ spec = entry.get("bound_spec")
+ if spec:
+ tokens = _bounded_peak(*spec, n_ctx, prefill_chunk)
+ return tokens
+
+
+def _line_bytes(
+ plan,
+ n_ctx: int,
+ prefill_chunk: int,
+ quantized: bool,
+ decoding: bool = True,
+) -> int:
+ const, slope = ("quant_const", "quant_slope") if quantized else ("const", "slope")
+ return int(
+ sum(
+ entry[const] + entry[slope] * _held_tokens(entry, n_ctx, prefill_chunk, decoding)
+ for entry in plan
+ )
+ )
+
+
+def _kv_bytes(plan, n_ctx: int, quant_start, prefill_chunk: int):
+ """Peak cache bytes across the run, and where the cache is quantized from."""
+ if quant_start is None:
+ return _line_bytes(plan, n_ctx, prefill_chunk, quantized = False), None
+ # The load converts the cache before prefill, so no full-width copy is ever held.
+ return _line_bytes(plan, n_ctx, prefill_chunk, quantized = True), quant_start
+
+
+def _config_width(value) -> int:
+ """A width from the config, which is not always a single number."""
+ if isinstance(value, bool):
+ return 0
+ if isinstance(value, (int, float)):
+ return int(value)
+ if isinstance(value, (list, tuple)):
+ widths = [int(v) for v in value if isinstance(v, (int, float)) and not isinstance(v, bool)]
+ return max(widths) if widths else 0
+ return 0
+
+
+def _widest_quantized_scores(n_ctx: int, boundary: int, prefill_chunk: int) -> int:
+ widest = 0
+ remaining = n_ctx - boundary - 1
+ if remaining > 0:
+ whole, rest = divmod(remaining, prefill_chunk)
+ if whole:
+ widest = prefill_chunk * (boundary + whole * prefill_chunk)
+ if rest:
+ widest = max(widest, rest * (n_ctx - 1))
+ return max(widest, n_ctx + 1)
+
+
+def _dequantized_row_bytes(widths, dtype_size: int, kv_bits, kv_group_size, prefill_chunk):
+ """Bytes per cached token one attention call holds where the runtime dequantizes the cache."""
+ _hidden, _intermediate, heads, kv_heads, head_dim, value_dim = widths
+ if not (heads and kv_heads and head_dim and kv_bits):
+ return None
+ try:
+ import mlx.core as mx
+
+ from unsloth_zoo.mlx.attention import (
+ _row_bytes,
+ dequantizing_is_smaller,
+ fused_kernel_exists,
+ )
+ except Exception as exc:
+ logger.debug("MLX estimate cannot reach the quantized-attention wrapper: %s", exc)
+ return None
+ try:
+ dtype = mx.float32 if dtype_size == 4 else mx.bfloat16
+
+ def rows(width):
+ return mx.quantize(
+ mx.zeros((1, kv_heads, 1, width), dtype = dtype),
+ group_size = kv_group_size,
+ bits = kv_bits,
+ )
+
+ keys, values = rows(head_dim), rows(value_dim or head_dim)
+ queries = mx.zeros((1, heads, prefill_chunk, head_dim), dtype = dtype)
+ if not fused_kernel_exists(queries, keys, values, kv_group_size, "causal"):
+ return None
+ if not dequantizing_is_smaller(queries, keys, values, kv_group_size):
+ return None
+ return kv_heads * sum(_row_bytes(cache, kv_group_size, dtype) for cache in (keys, values))
+ except Exception as exc:
+ logger.debug("MLX estimate cannot route the quantized attention: %s", exc)
+ return None
+
+
+def _compute_bytes(
+ widths,
+ dtype_size: int,
+ prefill_chunk: int,
+ plan,
+ n_ctx: int,
+ quant_boundary,
+ kv_bits = None,
+ kv_group_size: int = _KV_GROUP_SIZE,
+) -> int:
+ """Transient buffers a prefill step holds, on top of weights and cache."""
+ hidden, intermediate, heads = widths[:3]
+ if hidden <= 0:
+ return 0
+ intermediate = intermediate or 4 * hidden
+ attention_width = (hidden + intermediate) * dtype_size * _ATTENTION_LIVE
+ recurrent_width = sum(int(entry["conv_width"]) for entry in plan) * _RECURRENT_DTYPE_SIZE
+ total = _COMPUTE_BASE_BYTES + prefill_chunk * (attention_width + recurrent_width)
+ if quant_boundary is not None:
+ score = heads * _QUANT_SCORE_DTYPE_SIZE * _QUANT_SCORE_LIVE
+ row = _dequantized_row_bytes(widths, dtype_size, kv_bits, kv_group_size, prefill_chunk)
+ if row is None:
+ total += _widest_quantized_scores(n_ctx, quant_boundary, prefill_chunk) * score
+ else:
+ total += max(row * n_ctx, (n_ctx + 1) * score)
+ return int(total)
+
+
+def _load_is_refused(model_dir: str, config: dict, load_in_4bit: bool) -> Optional[str]:
+ try:
+ from unsloth_zoo.mlx.loader import (
+ _ensure_quantization_compatible,
+ _get_existing_mlx_quantization,
+ _resolve_mlx_quantization_spec,
+ )
+ except Exception:
+ return None
+
+ import contextlib
+ import io
+
+ existing = _get_existing_mlx_quantization(config)
+ if isinstance(existing, dict) and existing.get("quant_method") == "bitsandbytes":
+ return "bitsandbytes weights mlx-lm cannot read"
+
+ try:
+ with contextlib.redirect_stdout(io.StringIO()):
+ spec = _resolve_mlx_quantization_spec(
+ load_in_4bit = bool(load_in_4bit),
+ load_in_8bit = False,
+ load_in_16bit = False,
+ load_in_fp8 = False,
+ load_in_mxfp4 = False,
+ load_in_nvfp4 = False,
+ full_finetuning = False,
+ q_bits = None,
+ q_group_size = None,
+ q_mode = None,
+ mlx_quantization_config = None,
+ quantization_config = None,
+ quant_predicate = None,
+ quantize_modules = None,
+ force_requantize = False,
+ )
+ _ensure_quantization_compatible(config, spec, "this checkpoint")
+ except ValueError as exc:
+ return str(exc)
+ except Exception:
+ return None
+
+ return _refused_by_its_own_tensors(model_dir, config)
+
+
+def _extra_tensors_refused(model, config: dict, extras: list) -> Optional[str]:
+ try:
+ from unsloth_zoo.mlx.loader import (
+ _KNOWN_MLX_LM_STRICT_FALLBACKS,
+ _KNOWN_VLM_EXTRA_WEIGHT_FILTERS,
+ _gemma4_unused_shared_kv_weight,
+ _message_matches_known_fallback,
+ _raise_if_qk_norm_version_gap,
+ )
+ except Exception:
+ return None
+
+ model_type = config.get("model_type")
+ message = f"Received {len(extras)} parameters not in model: \n{extras}."
+ try:
+ _raise_if_qk_norm_version_gap(model_type, message, ValueError(message))
+ except ValueError as refusal:
+ return str(refusal)
+
+ vision = _loads_as_vision(config)
+ table = _KNOWN_VLM_EXTRA_WEIGHT_FILTERS if vision else _KNOWN_MLX_LM_STRICT_FALLBACKS
+ rule = table.get(model_type)
+ if rule is not None and _message_matches_known_fallback(message, rule):
+ allowed = rule.get("allowed_extra", frozenset())
+ shared_kv = rule.get("allow_shared_kv", False)
+ extras = [
+ name
+ for name in extras
+ if name not in allowed
+ and not (shared_kv and _gemma4_unused_shared_kv_weight(model, name))
+ ]
+ if not extras:
+ return None
+
+ return f"{len(extras)} tensors this architecture has no home for, " f"beginning {extras[0]}"
+
+
+def _refused_by_its_own_tensors(model_dir: str, config: dict) -> Optional[str]:
+ """Refusals only the checkpoint's own tensors reveal."""
+ try:
+ from mlx.utils import tree_flatten
+ from unsloth_zoo.mlx.loader import _raise_if_qk_norm_version_gap
+ except Exception:
+ return None
+
+ import contextlib
+ import io
+
+ try:
+ with contextlib.redirect_stdout(io.StringIO()):
+ dtype = _runtime_dtype()
+ model = _whole_model(config, dtype)
+ tensors = _checkpoint_tensors(model_dir, config, dtype)
+ if not tensors:
+ return None
+ if hasattr(model, "sanitize"):
+ tensors = model.sanitize(tensors)
+ built = {name for name, _ in tree_flatten(model.parameters())}
+ except Exception:
+ return None
+
+ unhoused = sorted(
+ name for name in set(tensors) - built if not name.endswith((".scales", ".biases"))
+ )
+ if unhoused:
+ refusal = _extra_tensors_refused(model, config, unhoused)
+ if refusal:
+ return refusal
+
+ try:
+ with contextlib.redirect_stdout(io.StringIO()):
+ model.load_weights(list(tensors.items()), strict = True)
+ except Exception as exc:
+ try:
+ _raise_if_qk_norm_version_gap(config.get("model_type"), str(exc), exc)
+ except ValueError as refusal:
+ return str(refusal)
+ except Exception:
+ return None
+ return None
+
+
+@dataclass
+class _MlxSizing:
+ """What a load costs before a context is named, so a search over contexts probes once."""
+
+ weights: int
+ dtype: object
+ plan: list
+ quant_start: Optional[int]
+ facts: dict
+ widths: tuple
+ chunk: int
+ kv_bits: Optional[int]
+ kv_group_size: int
+
+
+_SIZING_CACHE_MAX = 16
+_sizing_cache: "OrderedDict[tuple, Optional[_MlxSizing]]" = OrderedDict()
+_sizing_lock = threading.Lock()
+
+
+def _checkpoint_fingerprint(model_dir: str) -> tuple:
+ """Size and mtime of every config, index and shard file, so a changed download re-sizes."""
+ root = os.path.realpath(model_dir)
+ found = []
+ for pattern in ("*.json", "**/*.safetensors"):
+ for path in sorted(glob.glob(os.path.join(root, pattern), recursive = True)):
+ try:
+ stat = os.stat(path)
+ except OSError:
+ continue
+ found.append((os.path.relpath(path, root), stat.st_size, stat.st_mtime_ns))
+ return tuple(found)
+
+
+def _size_load(
+ model_dir: str, kv_bits: Optional[int], prefill_chunk: Optional[int], load_in_4bit: bool
+) -> Optional[_MlxSizing]:
+ """Cached ``_size_load_uncached``: every estimate prices the fit and the breakdown, and every
+ slider step prices again, each otherwise rebuilding the same weightless architecture."""
+ try:
+ key = (
+ os.path.realpath(model_dir),
+ kv_bits,
+ prefill_chunk,
+ bool(load_in_4bit),
+ os.environ.get("UNSLOTH_MLX_PROMPT_CACHE_BYTES"),
+ _checkpoint_fingerprint(model_dir),
+ )
+ except Exception:
+ return _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit)
+ with _sizing_lock:
+ if key in _sizing_cache:
+ _sizing_cache.move_to_end(key)
+ return _sizing_cache[key]
+ sizing = _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit)
+ with _sizing_lock:
+ _sizing_cache[key] = sizing
+ while len(_sizing_cache) > _SIZING_CACHE_MAX:
+ _sizing_cache.popitem(last = False)
+ return sizing
+
+
+def _clear_sizing_cache() -> None:
+ with _sizing_lock:
+ _sizing_cache.clear()
+
+
+def _size_load_uncached(
+ model_dir: str, kv_bits: Optional[int], prefill_chunk: Optional[int], load_in_4bit: bool
+) -> Optional[_MlxSizing]:
+ """Everything about a load that does not move with the context, or None if it cannot be sized."""
+ config = _snapshot_config(model_dir)
+ if config is None:
+ return None
+ refusal = _load_is_refused(model_dir, config, load_in_4bit)
+ if refusal is not None:
+ logger.debug("MLX estimate refuses %s: %s", model_dir, refusal)
+ return None
+ try:
+ weights = mlx_weight_bytes(model_dir, config, load_in_4bit)
+ except Exception as exc:
+ logger.debug("MLX estimate cannot read the shards of %s: %s", model_dir, exc)
+ return None
+ if weights <= 0:
+ return None
+ try:
+ dtype = _runtime_dtype()
+ loaded_chunk, loaded_group = _generation_settings(config)
+ plan, quant_start, facts = _cache_plan(config, dtype, kv_bits)
+ widths = tuple(
+ tower or checkpoint
+ for tower, checkpoint in zip(facts["widths"], _config_widths(config))
+ )
+ chunk = prefill_chunk or loaded_chunk
+ except Exception as exc:
+ logger.debug(
+ "MLX estimate could not size %s (%s): %s", model_dir, config.get("model_type"), exc
+ )
+ return None
+ return _MlxSizing(
+ weights = weights,
+ dtype = dtype,
+ plan = plan,
+ quant_start = quant_start,
+ facts = facts,
+ widths = widths,
+ chunk = chunk,
+ kv_bits = kv_bits,
+ kv_group_size = loaded_group,
+ )
+
+
+def _priced_at(sizing: _MlxSizing, n_ctx: int) -> Optional[MlxMemoryBreakdown]:
+ try:
+ context = max(int(n_ctx or 0), 1)
+ whole_prompt = sizing.facts["whole_prompt"]
+ chunk = context if whole_prompt else sizing.chunk
+ kv, quant_boundary = _kv_bytes(sizing.plan, context, sizing.quant_start, chunk)
+ compute = _compute_bytes(
+ sizing.widths,
+ sizing.dtype.size,
+ chunk,
+ sizing.plan,
+ context,
+ quant_boundary,
+ sizing.kv_bits,
+ sizing.kv_group_size,
+ )
+ except Exception as exc:
+ logger.debug("MLX estimate could not price %s tokens: %s", n_ctx, exc)
+ return None
+ total = sizing.weights + kv + compute
+ return MlxMemoryBreakdown(
+ weights_bytes = sizing.weights,
+ kv_bytes = kv,
+ compute_bytes = compute,
+ total_bytes = total,
+ gpu_bytes = total,
+ n_ctx = context,
+ layer_count = sizing.facts["layers"],
+ cache_type_kv = _cache_width_name(
+ sizing.plan,
+ sizing.kv_bits,
+ quant_boundary is not None,
+ _width_name(sizing.dtype),
+ context,
+ chunk,
+ ),
+ )
+
+
+def mlx_memory_breakdown(
+ model_dir: str,
+ *,
+ n_ctx: int,
+ kv_bits: Optional[int] = None,
+ prefill_chunk: Optional[int] = None,
+ load_in_4bit: bool = False,
+) -> Optional[MlxMemoryBreakdown]:
+ """Price an MLX load, or None when it cannot honestly be sized."""
+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit)
+ return None if sizing is None else _priced_at(sizing, n_ctx)
+
+
+def mlx_fit_context(
+ model_dir: str,
+ *,
+ budget_bytes: int,
+ max_ctx: int,
+ min_ctx: int = MLX_FIT_MIN_CONTEXT,
+ kv_bits: Optional[int] = None,
+ prefill_chunk: Optional[int] = None,
+ load_in_4bit: bool = False,
+) -> Optional[int]:
+ """Largest context whose estimated footprint stays inside ``budget_bytes``."""
+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit)
+ if sizing is None:
+ return None
+ ceiling = max(int(max_ctx or 0), MLX_KV_BLOCK)
+ at_ceiling = _priced_at(sizing, ceiling)
+ if at_ceiling is None or at_ceiling.total_bytes <= budget_bytes:
+ return None
+ floor = max(-(-int(min_ctx or 0) // MLX_KV_BLOCK) * MLX_KV_BLOCK, MLX_KV_BLOCK)
+ low, high, best = floor, ceiling, None
+ while low <= high:
+ middle = (low + high) // 2
+ priced = _priced_at(sizing, middle)
+ if priced is None:
+ return None
+ if priced.total_bytes <= budget_bytes:
+ best = middle
+ low = middle + 1
+ else:
+ high = middle - 1
+ return None if best is None else (best // MLX_KV_BLOCK) * MLX_KV_BLOCK
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 9137f128cef..ac1c715f2ba 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -377,6 +377,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:
"max_context_length": model_info.get("max_context_length"),
"requested_context_length": model_info.get("requested_context_length"),
"context_length_enforced": model_info.get("context_length_enforced"),
+ "context_length_fitted": model_info.get("context_length_fitted"),
"context_unbounded_when_batched": model_info.get("context_unbounded_when_batched"),
"mlx_context_budget": model_info.get("mlx_context_budget"),
}
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 4c67f8433bf..6a3f6be5328 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -640,13 +640,12 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
_entry = (
_bm.get(mc.identifier) or _bm.get(getattr(backend, "active_model_name", None)) or {}
)
- # The whole group: the parent reports all four and can recompute none of
- # them once the worker holds the model.
for _ctx_field in (
"context_length",
"native_context_length",
"max_context_length",
"requested_context_length",
+ "context_length_fitted",
"mlx_context_budget",
):
try:
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 53123838982..68d58a2c5db 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -866,7 +866,9 @@ class EstimateMemoryRequest(BaseModel):
"""Settings a Load-Model panel is about to submit, priced before it submits them.
Every field mirrors the load request it previews, so the estimate answers for the
- command that would actually run. Header-only: nothing is read, touched or loaded.
+ command that would actually run. No model is loaded and no tensor data is read: a
+ GGUF is priced from its header, an MLX repo from safetensors headers and the shapes
+ of a graph that is built and never evaluated.
"""
model_path: str = Field(..., description = "Model identifier or local path")
@@ -884,6 +886,14 @@ class EstimateMemoryRequest(BaseModel):
description = "Context length to price (--ctx-size). 0 or omitted prices the "
"model's native context, which is what an Auto load asks for.",
)
+ load_in_4bit: bool = Field(
+ True,
+ description = "Quantize an unquantized checkpoint to 4 bits on load, as "
+ "/load does by default. Priced because it is the difference between a "
+ "checkpoint's shards and what those shards go resident as. A checkpoint "
+ "carrying its own quantization is priced at its shards, and asking for 4 "
+ "bits on top can make it unsizable, because the real load refuses that.",
+ )
cache_type_kv: Optional[str] = Field(
None,
description = "KV cache dtype to price. The biggest lever on this estimate at "
@@ -940,8 +950,30 @@ class EstimateMemoryRequest(BaseModel):
"-nkvo, --swa-full and the cache-type flags all move this estimate.",
)
+ max_seq_length: Optional[int] = Field(
+ None,
+ ge = 0,
+ le = 1048576,
+ description = "Context an MLX load would open at. Separate from n_ctx because "
+ "the two backends take the length from different fields -- /load reads n_ctx "
+ "for GGUF and max_seq_length for everything else -- and pricing one from the "
+ "other quotes a context the load will not use.",
+ )
+ mlx_kv_quant: Optional[str] = Field(
+ None,
+ description = "MLX KV cache quantization in /load's vocabulary ('auto', '8', 'tq-4', ...). "
+ "A TurboQuant choice is priced at full width, as the load fits it.",
+ )
+ mlx_kv_bits: Optional[int] = Field(
+ None,
+ description = "Superseded by mlx_kv_quant; read only when mlx_kv_quant is omitted. Not derived "
+ "from cache_type_kv: that is llama.cpp's setting, it survives in the config of "
+ "a model that never used it, and reading it here would quantize an estimate "
+ "for a load whose cache stays full width.",
+ )
+
_no_booleans = field_validator(
- "n_batch", "n_ubatch", "ctx_checkpoints", "n_ctx", mode = "before"
+ "n_batch", "n_ubatch", "ctx_checkpoints", "n_ctx", "max_seq_length", mode = "before"
)(LoadRequest._no_booleans.__func__)
# Unresolved, the reference reads as a Hub id and the panel is told the estimate is
# unavailable for a row it was offered.
@@ -957,7 +989,12 @@ class EstimateMemoryResponse(BaseModel):
description = "Cause when available is false: 'not_gguf', 'not_downloaded', "
"'unsupported_source' or 'unsizable'.",
)
- weights_bytes: int = Field(0, description = "Resident model files: weights, projector, drafter")
+ weights_bytes: int = Field(
+ 0,
+ description = "Resident weights, projector and drafter. The size of the files "
+ "for a GGUF load; for MLX, what those files go resident AS, which is smaller "
+ "wherever the load casts or quantizes them.",
+ )
kv_bytes: int = Field(0, description = "KV cache at the requested context and slots")
kv_checkpoint_bytes: int = Field(
0,
@@ -1013,6 +1050,13 @@ class EstimateMemoryResponse(BaseModel):
True, description = "False under --no-kv-offload, which moves the cache to host RAM"
)
n_ctx: int = Field(0, description = "Context length the estimate actually priced")
+ context_fitted: Optional[int] = Field(
+ None,
+ description = "The window MLX would fit to this machine's memory for a load that "
+ "names no Context Length, which is then what n_ctx prices. Null when the load names "
+ "one, when the machine holds the model's own window, or when the footprint of the "
+ "load cannot be described.",
+ )
cache_type_kv: Optional[str] = Field(
None, description = "KV dtype the estimate priced, after flags and fallbacks resolve"
)
@@ -1112,13 +1156,28 @@ class MemoryEstimate(BaseModel):
native_context: Optional[int] = Field(
None, description = "The model's own trained context length, when readable"
)
- cache_type_kv: Optional[str] = Field(None, description = "KV cache dtype the estimate priced")
+ cache_type_kv: Optional[str] = Field(
+ None,
+ description = (
+ "KV cache width the estimate priced, in the vocabulary of whichever backend "
+ "would load it: llama.cpp spellings such as f16 or q8_0 for a GGUF load, and "
+ "a dtype abbreviation or an N-bit width for an MLX one. An MLX cache whose "
+ "entries do not all end up at one width names each of them, the one carrying "
+ "the most of the cache first, as in bf16/4-bit"
+ ),
+ )
n_parallel: int = Field(1, description = "Slots the estimate priced, after the launch clamps")
layer_count: Optional[int] = Field(None, description = "GGUF block_count, when readable")
gpu_layers: Optional[int] = Field(None, description = "Layers placed on the GPU, when known")
moe_offload_unmodelled: bool = Field(
False, description = "--n-cpu-moe is set, so the GPU figure reads high"
)
+ context_fitted: Optional[int] = Field(
+ None,
+ description = "The window MLX would fit to this machine's memory for a load that "
+ "names no Context Length, and which n_ctx then prices. Null wherever the served "
+ "window was not chosen for this machine.",
+ )
context_is_pinned: bool = Field(
True,
description = "False only when the loader is free to shrink the context. A caller "
@@ -1259,6 +1318,14 @@ class _InferenceRuntimeFields(BaseModel):
"bounded while still reporting true -- so read it as the weaker claim it is."
),
)
+ context_length_fitted: Optional[int] = Field(
+ None,
+ description = (
+ "The window MLX fitted to this machine's memory, where that is shorter than the "
+ "model's own and no Context Length was requested. Null means context_length was not "
+ "chosen for this machine: a pin, or the model's own window."
+ ),
+ )
mlx_context_budget: Optional[int] = Field(
None,
description = (
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 62f1b9ae9a6..365640be0ca 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -7607,6 +7607,7 @@ def _unsloth_serving_fields(model_info: dict) -> dict:
"requested_parallel_slots": slots,
"parallel_slots": effective,
"context_length_enforced": model_info.get("context_length_enforced"),
+ "context_length_fitted": _positive_int_or_none(model_info.get("context_length_fitted")),
"context_unbounded_when_batched": bool(model_info.get("context_unbounded_when_batched")),
}
@@ -7632,6 +7633,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
mlx_context_budget = None,
chat_template_override_reason = None,
context_length_enforced = True,
+ context_length_fitted = None,
context_unbounded_when_batched = False,
# Older/custom backend doubles predate this additive runtime field.
preserve_thinking_default = bool(getattr(llama_backend, "preserve_thinking_default", False)),
@@ -13821,6 +13823,194 @@ def _local_gguf_main_path(config: ModelConfig) -> Optional[str]:
return None
+def _mlx_estimate_ceiling(model_dir: str) -> Optional[int]:
+ """The window this checkpoint declares, held to what a load may be asked for."""
+ from types import SimpleNamespace
+
+ from core.inference.mlx_inference import mlx_native_context_length
+ from core.inference.mlx_memory import _snapshot_config
+ from core.inference.runtime_context import MAX_REQUESTABLE_CONTEXT
+
+ try:
+ config = _snapshot_config(model_dir)
+ except Exception:
+ return None
+ if config is None:
+ return None
+ native = mlx_native_context_length(SimpleNamespace(config = config))
+ return None if native is None else min(int(native), MAX_REQUESTABLE_CONTEXT)
+
+
+def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_bits):
+ """The window a load naming no Context Length would be fitted to."""
+ from core.inference.mlx_inference import (
+ mlx_fit_to_memory,
+ mlx_vlm_snapshot_store_available,
+ )
+
+ ceiling = _mlx_estimate_ceiling(model_dir)
+ if not ceiling:
+ return None
+ return mlx_fit_to_memory(
+ model_dir,
+ ceiling,
+ load_in_4bit = load_in_4bit,
+ retains_history = (
+ not getattr(config, "is_vision", False) or mlx_vlm_snapshot_store_available()
+ ),
+ kv_bits = kv_bits,
+ )
+
+
+def _mlx_estimate_available() -> bool:
+ """Whether this host would actually run the load through MLX."""
+ try:
+ from utils.mlx_repair import is_apple_silicon, mlx_stack_blockers
+ except Exception:
+ return False
+ try:
+ return bool(is_apple_silicon() and not mlx_stack_blockers())
+ except Exception:
+ return False
+
+
+def _the_revision_that_loads(snapshots: list) -> list:
+ """Just the revision `main` names, where it names one that is here."""
+ from hub.utils.hf_cache_state import ref_snapshot_dir
+
+ paths = [Path(snapshot) for snapshot in snapshots]
+ if not paths:
+ return paths
+ pinned = ref_snapshot_dir(paths[0].parent.parent)
+ return [pinned] if pinned is not None else paths
+
+
+def _local_mlx_model_dir(config: ModelConfig) -> Optional[str]:
+ def _complete(directory: Path, shards) -> bool:
+ named = [
+ (Path(shard).parent, match)
+ for shard in shards
+ if (match := _re.fullmatch(r"(.+)-(\d+)-of-(\d+)\.safetensors", Path(shard).name))
+ ]
+ if not named:
+ return _index_agrees(directory)
+ for parent, match in named:
+ stem, total = match.group(1), match.group(3)
+ width, count = len(match.group(2)), int(total)
+ if any(
+ not (parent / f"{stem}-{i:0{width}d}-of-{total}.safetensors").is_file()
+ for i in range(1, count + 1)
+ ):
+ return False
+ return _index_agrees(directory)
+
+ def _index_agrees(directory: Path) -> bool:
+ """Whether an index that describes THIS directory finds all of it here."""
+ index = directory / "model.safetensors.index.json"
+ if not index.is_file():
+ return True
+ try:
+ with open(index, encoding = "utf-8") as handle:
+ weight_map = json.load(handle).get("weight_map", {})
+ named = {shard for shard in weight_map.values() if isinstance(shard, str)}
+ except (ValueError, OSError, AttributeError):
+ return True
+ here = {
+ path.relative_to(directory).as_posix() for path in directory.glob("**/*.safetensors")
+ }
+ return not (named & here) or named <= here
+
+ def _usable(path) -> bool:
+ directory = Path(path)
+ if not directory.is_dir() or not (directory / "config.json").is_file():
+ return False
+ from core.inference.mlx_memory import mlx_shard_files
+
+ try:
+ snapshot = json.loads((directory / "config.json").read_text(encoding = "utf-8"))
+ except Exception:
+ snapshot = None
+ try:
+ shards = mlx_shard_files(
+ str(directory), snapshot if isinstance(snapshot, dict) else None
+ )
+ except Exception:
+ return False
+ if not shards:
+ return False
+ return _complete(directory, shards)
+
+ candidate = getattr(config, "path", None)
+ if candidate and _usable(candidate):
+ return str(candidate)
+ identifier = candidate or getattr(config, "identifier", None)
+ if not identifier or getattr(config, "is_local", False):
+ return None
+ names = [_mlx_load_identifier(str(identifier))]
+ if "/" not in names[0]:
+ names.append(f"unsloth/{names[0]}")
+ try:
+ from utils.models.model_config import _iter_hf_cache_snapshots
+ roots = _estimate_hf_cache_roots()
+ except Exception:
+ return None
+ for root in roots or [None]:
+ for name in names:
+ try:
+ snapshots = _iter_hf_cache_snapshots(name, cache_dir = root)
+ except Exception:
+ continue
+ for snapshot in _the_revision_that_loads(list(snapshots)):
+ if _usable(snapshot):
+ return str(snapshot)
+ return None
+
+
+def _mlx_load_identifier(identifier: str) -> str:
+ """The repository the MLX load would actually open, not the one selected."""
+ try:
+ from unsloth_zoo.mlx.loader import _remap_unsloth_bnb_hub_id_for_mlx
+ except Exception:
+ return identifier
+ try:
+ return _remap_unsloth_bnb_hub_id_for_mlx(identifier, None)[0] or identifier
+ except Exception:
+ return identifier
+
+
+def _mlx_estimate_load_in_4bit(config, request) -> bool:
+ """The 4-bit setting the load resolves, not the one the panel sent."""
+ # Same order as the load route: a full-finetune output loads 16-bit before any tier check.
+ if not _effective_load_in_4bit(config, request.load_in_4bit):
+ return False
+ from utils.transformers_version import latest_tier_active_for
+
+ identifier = getattr(config, "identifier", None)
+ return not _offline_guarded(
+ (identifier, getattr(config, "base_model", None)),
+ latest_tier_active_for,
+ identifier,
+ request.hf_token,
+ )
+
+
+def _mlx_estimate_kv_bits(mlx_kv_quant, mlx_kv_bits = None) -> Optional[int]:
+ """The mx.quantize width the load would price, or None for a full-width cache."""
+ try:
+ from core.inference.mlx_inference import MLX_KV_BITS_CHOICES, parse_mlx_kv_quant
+ except Exception:
+ return None
+ if mlx_kv_quant is None:
+ if isinstance(mlx_kv_bits, bool) or not isinstance(mlx_kv_bits, int):
+ return None
+ return mlx_kv_bits if mlx_kv_bits in MLX_KV_BITS_CHOICES else None
+ bits, turboquant = parse_mlx_kv_quant(mlx_kv_quant)
+ # TurboQuant's layout is not one the planner prices; the load fits it at full width too.
+ if turboquant or bits not in MLX_KV_BITS_CHOICES:
+ return None
+ return bits
+
+
def _is_embedding_gguf(config: ModelConfig) -> bool:
"""Whether this GGUF's pooling type makes llama-server launch with --embedding.
@@ -16221,6 +16411,7 @@ def _npu_load_response(resident, status: str) -> LoadResponse:
max_context_length = model.max_context_length,
native_context_length = model.max_context_length,
context_length_enforced = True,
+ context_length_fitted = None,
supports_reasoning = model.reasoning,
reasoning_style = "enable_thinking",
supports_tools = model.tools,
@@ -18749,17 +18940,16 @@ async def estimate_memory(
fastapi_request: Request = None,
current_subject: str = Depends(get_current_subject),
):
- """Price a prospective GGUF load from its header, before anything is allocated.
+ """Price a prospective load from its metadata, before the load allocates anything.
- Reads only GGUF metadata and file sizes, so it is safe to call on every settings
- change: no model is loaded, no device is touched, nothing is downloaded. A model
- that is not on this disk answers ``not_downloaded``.
+ Reads GGUF metadata and file sizes -- or, for an MLX repo, safetensors headers and the
+ shapes of an architecture built and never evaluated -- so it is safe to call on every
+ settings change: no model is loaded, no tensor data is read, nothing is downloaded.
- The arithmetic is the loader's own KV, compute-buffer and companion sizing, which
- is the point. Weights times a constant is fine until the KV cache stops being a
- rounding error: at 262k tokens the cache can outweigh the weights, and it swings
- fourfold on the cache dtype alone. Where the header cannot supply the dims this
- answers ``kv_estimable = false`` rather than quoting an assumed total.
+ The arithmetic is the loader's own KV, compute-buffer and companion sizing. Weights
+ times a constant is fine until the KV cache stops being a rounding error: at 262k
+ tokens the cache can outweigh the weights. Where the metadata cannot supply the dims
+ this answers ``kv_estimable = false`` rather than quoting an assumed total.
"""
native_access_deferred = _defers_access_to_native_grant(request)
if account_access.managed_account() and not native_access_deferred:
@@ -18798,10 +18988,6 @@ async def estimate_memory(
requested_slots = _resolve_parallel_slots(request, fastapi_request)
def _estimate() -> EstimateMemoryResponse:
- resolved_slots = _effective_parallel_slots(
- requested_slots,
- diffusion_kind = False,
- )
config = _cached_estimate_config(
model_identifier,
request.gguf_variant,
@@ -18813,12 +18999,58 @@ async def estimate_memory(
if config is None:
return EstimateMemoryResponse(available = False, reason = "unsizable")
if not getattr(config, "is_gguf", False):
- # Safetensors / MLX allocate on a different plan; the GGUF arithmetic
- # would be a made-up number in a confident box.
- return EstimateMemoryResponse(available = False, reason = "not_gguf")
+ if not _mlx_estimate_available():
+ return EstimateMemoryResponse(available = False, reason = "not_gguf")
+ from core.inference.native_audio import is_native_audio_model
+
+ if is_native_audio_model(model_identifier):
+ return EstimateMemoryResponse(available = False, reason = "not_gguf")
+ if getattr(config, "is_lora", False):
+ return EstimateMemoryResponse(available = False, reason = "unsizable")
+ model_dir = _local_mlx_model_dir(config)
+ if not model_dir:
+ return EstimateMemoryResponse(available = False, reason = "not_downloaded")
+ from core.inference.mlx_inference import mlx_kv_quant_is_refused
+ from core.inference.mlx_memory import mlx_memory_breakdown
+
+ mlx_load_in_4bit = _mlx_estimate_load_in_4bit(config, request)
+ mlx_kv_bits = _mlx_estimate_kv_bits(request.mlx_kv_quant, request.mlx_kv_bits)
+ # A width the cache takes applies under a pin or a fit alike: the window is then a budget.
+ if mlx_kv_bits is not None and mlx_kv_quant_is_refused(model_dir, mlx_kv_bits):
+ mlx_kv_bits = None
+ mlx_named_ctx = request.max_seq_length or 0
+ mlx_fitted_ctx = None
+ if not mlx_named_ctx:
+ mlx_fitted_ctx = _mlx_estimate_fitted_context(
+ config, model_dir, mlx_load_in_4bit, mlx_kv_bits
+ )
+ mlx_priced_ctx = mlx_named_ctx or mlx_fitted_ctx or _mlx_estimate_ceiling(model_dir)
+ if not mlx_priced_ctx:
+ return EstimateMemoryResponse(available = False, reason = "unsizable")
+ mlx_breakdown = mlx_memory_breakdown(
+ model_dir,
+ n_ctx = mlx_priced_ctx,
+ kv_bits = mlx_kv_bits,
+ load_in_4bit = mlx_load_in_4bit,
+ )
+ if mlx_breakdown is None:
+ return EstimateMemoryResponse(available = False, reason = "unsizable")
+ return EstimateMemoryResponse(
+ **project_estimate_memory_response(
+ build_memory_estimate(
+ mlx_breakdown,
+ quant_file_bytes = 0,
+ context_fitted = mlx_fitted_ctx,
+ )
+ )
+ )
gguf_path = _local_gguf_main_path(config)
if not gguf_path:
return EstimateMemoryResponse(available = False, reason = "not_downloaded")
+ resolved_slots = _effective_parallel_slots(
+ requested_slots,
+ diffusion_kind = False,
+ )
# Price the files on this disk, not the repository they came from.
config = _localized_estimate_config(config, gguf_path)
@@ -19791,6 +20023,7 @@ async def get_status(current_subject: str):
max_context_length = _npu_model.max_context_length,
native_context_length = _npu_model.max_context_length,
context_length_enforced = True,
+ context_length_fitted = None,
supports_reasoning = _npu_model.reasoning,
reasoning_style = "enable_thinking",
supports_tools = _npu_model.tools,
@@ -30655,7 +30888,7 @@ def _openai_model_objects() -> list[dict]:
):
entry["task"] = _TTS_MODEL_TASK
- for _field in ("native_context_length", "max_context_length"):
+ for _field in ("native_context_length", "max_context_length", "context_length_fitted"):
_value = _positive_int_or_none(model_info.get(_field))
if _value is not None:
entry[_field] = _value
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index 553eda6dfca..f6359e0181d 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -1064,6 +1064,7 @@ class TestLoadHubDownloadExclusion:
"chat_template_override_reason",
# Constant True: llama.cpp allocates the window it reports.
"context_length_enforced",
+ "context_length_fitted",
# Constant False: each slot decodes against its own window.
"context_unbounded_when_batched",
# Read from requested_extra_args, which is what the load was invoked
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index 336c51678f8..b99b5154507 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -26,6 +26,7 @@ No GPU, no network, no model load: every GGUF here is a synthetic header on tmp_
"""
import inspect
+import json
import os
import sys
import types as _types
@@ -911,18 +912,390 @@ class TestEstimateMemoryRoute:
assert resp.available is False
assert resp.reason == "unsupported_source"
- def test_non_gguf_model_is_not_priced(self, monkeypatch):
- # Safetensors / MLX size their memory through a different allocator;
- # quoting the GGUF arithmetic for them is a made-up number in a box.
+ def test_the_estimate_asks_the_question_the_worker_asks_of_the_stack(self, monkeypatch):
+ import utils.mlx_repair as repair
+ monkeypatch.setattr(repair, "is_apple_silicon", lambda: True)
+ for blockers, available in (([], True), (["mlx-vlm 0.0.1 is too old"], False)):
+ monkeypatch.setattr(repair, "mlx_stack_blockers", lambda b = blockers: b)
+ assert ri._mlx_estimate_available() is available
+
+ def test_non_gguf_model_is_not_priced_without_mlx(self, monkeypatch):
monkeypatch.setattr(
ri,
"_cached_estimate_config",
lambda *a, **kw: SimpleNamespace(is_gguf = False, identifier = "org/model"),
)
+ monkeypatch.setattr(ri, "_mlx_estimate_available", lambda: False)
resp = _estimate(model_path = "org/model")
assert resp.available is False
assert resp.reason == "not_gguf"
+ def test_a_native_audio_load_is_not_priced_as_an_mlx_one(self, monkeypatch, tmp_path):
+ import core.inference.native_audio as native_audio
+
+ self._mlx_target(monkeypatch, str(tmp_path))
+ priced = []
+ monkeypatch.setattr(
+ ri, "_mlx_estimate_fitted_context", lambda *a, **kw: priced.append(a) or None
+ )
+ monkeypatch.setattr(native_audio, "is_native_audio_model", lambda name: name == "org/tts")
+ resp = _estimate(model_path = "org/tts")
+ assert (resp.available, resp.reason, priced) == (False, "not_gguf", [])
+ assert _estimate(model_path = "org/model").reason != "not_gguf"
+
+ @staticmethod
+ def _mlx_target(monkeypatch, model_dir, **config):
+ """A resolved non-GGUF config on an MLX host, with its weights at *model_dir*."""
+ monkeypatch.setattr(
+ ri,
+ "_cached_estimate_config",
+ lambda *a, **kw: SimpleNamespace(is_gguf = False, identifier = "org/model", **config),
+ )
+ monkeypatch.setattr(ri, "_mlx_estimate_available", lambda: True)
+ monkeypatch.setattr(ri, "_local_mlx_model_dir", lambda config: model_dir)
+
+ @staticmethod
+ def _record_breakdown(monkeypatch, **fields):
+ """Install a breakdown stub, returning the dict its call is recorded into."""
+ import core.inference.mlx_memory as mlx_memory
+
+ seen = {}
+
+ def _breakdown(
+ model_dir,
+ *,
+ n_ctx,
+ kv_bits = None,
+ load_in_4bit = False,
+ **kw,
+ ):
+ seen.update(
+ model_dir = model_dir, n_ctx = n_ctx, kv_bits = kv_bits, load_in_4bit = load_in_4bit
+ )
+ return mlx_memory.MlxMemoryBreakdown(n_ctx = n_ctx, **fields)
+
+ monkeypatch.setattr(mlx_memory, "mlx_memory_breakdown", _breakdown)
+ return seen
+
+ def test_mlx_model_not_on_disk_is_not_downloaded(self, monkeypatch):
+ self._mlx_target(monkeypatch, None)
+ resp = _estimate(model_path = "org/model")
+ assert resp.available is False
+ assert resp.reason == "not_downloaded"
+
+ def test_mlx_model_whose_cache_cannot_be_read_is_unsizable(self, monkeypatch):
+ self._mlx_target(monkeypatch, "/models/thing")
+ import core.inference.mlx_memory as mlx_memory
+
+ monkeypatch.setattr(mlx_memory, "mlx_memory_breakdown", lambda *a, **kw: None)
+ resp = _estimate(model_path = "org/model")
+ assert resp.available is False
+ assert resp.reason == "unsizable"
+ assert resp.total_bytes == 0
+
+ def test_mlx_model_is_priced_and_itemized(self, monkeypatch, tmp_path):
+ self._mlx_target(monkeypatch, "/models/thing")
+ seen = self._record_breakdown(
+ monkeypatch,
+ weights_bytes = 8_000_000_000,
+ kv_bytes = 600_000_000,
+ compute_bytes = 400_000_000,
+ total_bytes = 9_000_000_000,
+ gpu_bytes = 9_000_000_000,
+ layer_count = 36,
+ )
+ resp = _estimate(
+ model_path = "org/model",
+ max_seq_length = 16384,
+ mlx_kv_bits = 4,
+ cache_type_kv = "q8_0",
+ )
+ assert (resp.available, resp.reason, resp.kv_estimable) == (True, None, True)
+ assert (resp.weights_bytes, resp.kv_bytes, resp.compute_bytes) == (
+ 8_000_000_000,
+ 600_000_000,
+ 400_000_000,
+ )
+ assert (resp.total_bytes, resp.gpu_bytes, resp.n_ctx, resp.layer_count) == (
+ 9_000_000_000,
+ 9_000_000_000,
+ 16384,
+ 36,
+ )
+ assert seen == {
+ "model_dir": "/models/thing",
+ "n_ctx": 16384,
+ "kv_bits": 4,
+ "load_in_4bit": True,
+ }
+ _estimate(
+ model_path = "org/model",
+ max_seq_length = 16384,
+ mlx_kv_bits = 4,
+ load_in_4bit = False,
+ )
+ assert seen["load_in_4bit"] is False
+
+ import utils.transformers_version as tv
+
+ asked, guarded, real_guard = [], [], ri._offline_guarded
+ monkeypatch.setattr(
+ tv,
+ "latest_tier_active_for",
+ lambda name, token = None: asked.append((name, token)) or True,
+ )
+ monkeypatch.setattr(
+ ri,
+ "_offline_guarded",
+ lambda t, fn, *a, **kw: guarded.append(fn) or real_guard(t, fn, *a, **kw),
+ )
+ _estimate(model_path = "org/model", max_seq_length = 16384, load_in_4bit = True, hf_token = "tok")
+ assert seen["load_in_4bit"] is False
+ assert asked == [("org/model", "tok")] and guarded == [tv.latest_tier_active_for]
+ monkeypatch.setattr(tv, "latest_tier_active_for", lambda *a, **kw: False)
+
+ monkeypatch.setattr(
+ ri,
+ "_cached_estimate_config",
+ lambda *a, **kw: SimpleNamespace(
+ is_gguf = False,
+ identifier = "org/model",
+ is_lora = True,
+ path = str(tmp_path),
+ base_model = "org/base",
+ ),
+ )
+ seen.clear()
+ resp = _estimate(model_path = "org/model", max_seq_length = 16384, load_in_4bit = True)
+ assert (resp.available, resp.reason, resp.total_bytes) == (False, "unsizable", 0)
+ assert seen == {}
+
+ def test_a_full_finetune_output_is_priced_at_the_16_bit_it_loads_at(
+ self, monkeypatch, tmp_path
+ ):
+ # The load route resolves a Studio full-finetune output to 16-bit; pricing it 4-bit
+ # advertised a fitted window the real load could not hold.
+ self._mlx_target(monkeypatch, "/models/thing", is_lora = False, path = str(tmp_path))
+ seen = self._record_breakdown(
+ monkeypatch,
+ weights_bytes = 1,
+ kv_bytes = 1,
+ compute_bytes = 1,
+ total_bytes = 3,
+ gpu_bytes = 3,
+ layer_count = 2,
+ )
+ import utils.transformers_version as tv
+
+ monkeypatch.setattr(tv, "latest_tier_active_for", lambda *a, **kw: False)
+ monkeypatch.setattr(ri, "is_full_finetune_output", lambda path: path == str(tmp_path))
+ _estimate(model_path = "org/model", max_seq_length = 4096, load_in_4bit = True)
+ assert seen["load_in_4bit"] is False
+ monkeypatch.setattr(ri, "is_full_finetune_output", lambda path: False)
+ _estimate(model_path = "org/model", max_seq_length = 4096, load_in_4bit = True)
+ assert seen["load_in_4bit"] is True
+
+ def test_the_snapshot_priced_is_the_one_the_load_would_open(self, monkeypatch, tmp_path):
+ import utils.models.model_config as mc
+
+ asked, found = [], []
+ snaps, ref = tmp_path / "snapshots", tmp_path / "refs" / "main"
+ monkeypatch.setattr(ri, "_estimate_hf_cache_roots", lambda: [None])
+ monkeypatch.setattr(
+ mc, "_iter_hf_cache_snapshots", lambda name, cache_dir = None: asked.append(name) or found
+ )
+ if ri._mlx_estimate_available():
+ for alias in ("unsloth/Qwen3-4B-bnb-4bit", "unsloth/Qwen3-4B-unsloth-bnb-4bit"):
+ assert (
+ ri._local_mlx_model_dir(
+ SimpleNamespace(path = None, is_local = False, identifier = alias)
+ )
+ is None
+ )
+ assert asked == ["unsloth/Qwen3-4B", "unsloth/Qwen3-4B"]
+ for made in (
+ "stub/config.json",
+ "new/config.json",
+ "new/model.safetensors",
+ "old/config.json",
+ "old/model.safetensors",
+ ):
+ (snaps / made).parent.mkdir(parents = True, exist_ok = True)
+ (snaps / made).write_text("{}")
+ ref.parent.mkdir()
+ found[:], new_, old = (
+ [snaps / "stub", snaps / "new", snaps / "old"],
+ str(snaps / "new"),
+ str(snaps / "old"),
+ )
+ config = SimpleNamespace(path = None, is_local = False, is_lora = False, identifier = "org/model")
+ for named, expected in (
+ ("old", old),
+ ("stub", None),
+ ("collected", new_),
+ ("../snapshots/old", new_),
+ (None, new_),
+ ):
+ ref.write_text(named) if named else ref.unlink()
+ assert ri._local_mlx_model_dir(config) == expected
+
+ def test_a_part_finished_download_is_not_priced_as_a_smaller_model(self, tmp_path):
+ def index(weight_map):
+ (tmp_path / "model.safetensors.index.json").write_text(
+ json.dumps({"weight_map": weight_map})
+ )
+
+ (tmp_path / "config.json").write_text("{}")
+ config = SimpleNamespace(path = str(tmp_path), is_local = True, is_lora = False)
+ five = [f"model-0000{i}-of-00005.safetensors" for i in range(1, 6)]
+ for name in five[:2]:
+ (tmp_path / name).write_bytes(b"")
+ assert ri._local_mlx_model_dir(config) is None
+ for name in five[2:]:
+ (tmp_path / name).write_bytes(b"")
+ assert ri._local_mlx_model_dir(config) == str(tmp_path)
+ (tmp_path / "adapter-00001-of-00002.safetensors").write_bytes(b"")
+ assert ri._local_mlx_model_dir(config) == str(tmp_path)
+ (tmp_path / "config.json").write_text(json.dumps({"vision_config": {}}))
+ (tmp_path / "weights").mkdir()
+ nested = [f"model-0000{i}-of-00002.safetensors" for i in (1, 2)]
+ for name in nested:
+ (tmp_path / "weights" / name).write_bytes(b"")
+ index({n: f"weights/{n}" for n in nested})
+ assert ri._local_mlx_model_dir(config) == str(tmp_path)
+ (tmp_path / "weights" / nested[1]).unlink()
+ assert ri._local_mlx_model_dir(config) is None
+
+ def test_shard_names_that_count_nothing_are_counted_by_the_index(self, tmp_path):
+ (tmp_path / "config.json").write_text("{}")
+ config = SimpleNamespace(path = str(tmp_path), is_local = True, is_lora = False)
+ (tmp_path / "model.safetensors").write_bytes(b"")
+ assert ri._local_mlx_model_dir(config) == str(tmp_path)
+ (tmp_path / "model.safetensors").unlink()
+ (tmp_path / "weights").mkdir()
+ named = [f"model-{i:05d}.safetensors" for i in range(1, 45)] + [
+ "mtp.safetensors",
+ "weights/extra.safetensors",
+ ]
+ (tmp_path / "model.safetensors.index.json").write_text(
+ json.dumps({"weight_map": {n: n for n in named}})
+ )
+ (tmp_path / named[0]).write_bytes(b"")
+ assert ri._local_mlx_model_dir(config) is None
+ for name in named[1:]:
+ (tmp_path / name).write_bytes(b"")
+ assert ri._local_mlx_model_dir(config) == str(tmp_path)
+ (tmp_path / "model.safetensors.index.json").write_text(
+ json.dumps(
+ {
+ "weight_map": {
+ str(i): f"model-{i:05d}-of-00099.safetensors" for i in range(1, 100)
+ }
+ }
+ )
+ )
+ assert ri._local_mlx_model_dir(config) == str(tmp_path)
+
+ def test_an_mlx_estimate_prices_max_seq_length_not_n_ctx(self, monkeypatch):
+ self._mlx_target(monkeypatch, "/models/thing")
+ seen = self._record_breakdown(
+ monkeypatch,
+ weights_bytes = 1,
+ kv_bytes = 1,
+ compute_bytes = 1,
+ total_bytes = 3,
+ gpu_bytes = 3,
+ )
+ _estimate(model_path = "org/model", max_seq_length = 8192, n_ctx = 32768)
+ assert seen["n_ctx"] == 8192
+
+ def test_an_unpinned_mlx_estimate_prices_the_fit_and_reports_it(self, monkeypatch, tmp_path):
+ from core.inference import mlx_inference
+ from core.inference.runtime_context import MAX_REQUESTABLE_CONTEXT
+
+ write = (tmp_path / "config.json").write_text
+ write(json.dumps({"max_position_embeddings": 262_144}))
+ self._mlx_target(monkeypatch, str(tmp_path))
+ asked, fit = {}, {"answer": 24_576}
+ monkeypatch.setattr(
+ mlx_inference,
+ "mlx_fit_to_memory",
+ lambda model_dir, ceiling, **kw: (
+ asked.update(kw, ceiling = ceiling, dir = model_dir) or fit["answer"]
+ ),
+ )
+ seen = self._record_breakdown(
+ monkeypatch,
+ weights_bytes = 1,
+ kv_bytes = 1,
+ compute_bytes = 1,
+ total_bytes = 3,
+ gpu_bytes = 3,
+ )
+ widths, refuses = [], {"answer": False}
+ monkeypatch.setattr(
+ mlx_inference,
+ "mlx_kv_quant_is_refused",
+ lambda *a: widths.append(a) or refuses["answer"],
+ )
+
+ # The width the cache takes is priced into the fit, and the fitted window is a budget.
+ resp = _estimate(model_path = "org/model", n_ctx = 32_768, mlx_kv_bits = 4)
+ assert (seen["n_ctx"], seen["kv_bits"], resp.context_fitted) == (24_576, 4, 24_576)
+ assert widths == [(str(tmp_path), 4)]
+ assert asked == {
+ "ceiling": 262_144,
+ "retains_history": True,
+ "dir": str(tmp_path),
+ "kv_bits": 4,
+ "load_in_4bit": seen["load_in_4bit"],
+ }
+
+ # /load's own vocabulary prices the same width; TurboQuant is priced at full width,
+ # the width the load fits it at, and never asks the mx.quantize probe.
+ widths.clear()
+ resp = _estimate(model_path = "org/model", n_ctx = 32_768, mlx_kv_quant = "4")
+ assert (seen["kv_bits"], resp.context_fitted, widths) == (4, 24_576, [(str(tmp_path), 4)])
+ widths.clear()
+ resp = _estimate(model_path = "org/model", mlx_kv_quant = "tq-4", mlx_kv_bits = 4)
+ assert (asked["kv_bits"], seen["kv_bits"], widths) == (None, None, [])
+
+ # A width the load would refuse is priced at full width, fitted or pinned.
+ refuses["answer"] = True
+ resp = _estimate(model_path = "org/model", mlx_kv_bits = 4)
+ assert (asked["kv_bits"], seen["kv_bits"], resp.context_fitted) == (None, None, 24_576)
+ resp = _estimate(model_path = "org/model", max_seq_length = 8192, mlx_kv_bits = 4)
+ assert (seen["n_ctx"], seen["kv_bits"], resp.context_fitted) == (8192, None, None)
+
+ # A pin never displaces an accepted width; it is not fitted either.
+ refuses["answer"] = False
+ asked.clear()
+ resp = _estimate(model_path = "org/model", max_seq_length = 8192, mlx_kv_bits = 4)
+ assert (seen["n_ctx"], seen["kv_bits"], resp.context_fitted, asked) == (8192, 4, None, {})
+
+ widths.clear()
+ _estimate(model_path = "org/model")
+ assert (widths, asked["kv_bits"], seen["kv_bits"]) == ([], None, None)
+
+ fit["answer"] = None
+ write(json.dumps({"max_position_embeddings": MAX_REQUESTABLE_CONTEXT * 4}))
+ resp = _estimate(model_path = "org/model", mlx_kv_bits = 4)
+ assert resp.context_fitted is None
+ assert (seen["n_ctx"], seen["kv_bits"]) == (MAX_REQUESTABLE_CONTEXT, 4)
+ write(json.dumps({"model_type": "llama"}))
+ seen.clear()
+ assert _estimate(model_path = "org/model").reason == "unsizable"
+ assert seen == {}
+
+ write(json.dumps({"max_position_embeddings": 262_144}))
+ self._mlx_target(monkeypatch, str(tmp_path), is_vision = True)
+ for available in (True, False):
+ monkeypatch.setattr(
+ mlx_inference, "mlx_vlm_snapshot_store_available", lambda a = available: a
+ )
+ _estimate(model_path = "org/model")
+ assert asked["retains_history"] is available
+
def test_gguf_not_on_disk_is_not_downloaded(self, monkeypatch):
# No header to read and no reaching for the network on a slider drag.
monkeypatch.setattr(
@@ -3308,3 +3681,1091 @@ class TestFourMoreLaunchNormalizations:
)
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 8192, n_parallel = 4)
assert out.n_parallel == 4
+
+
+class TestMlxEstimateKvBits:
+ """Which requested KV widths reach the MLX planner, and which are dropped."""
+
+ @pytest.mark.parametrize(
+ "quant, bits, expected",
+ [
+ (None, 4, 4),
+ (None, 8, 8),
+ (None, 7, None),
+ (None, 0, None),
+ (None, None, None),
+ (None, True, None),
+ ("4", None, 4),
+ (" 8 ", 4, 8),
+ ("auto", 8, None),
+ ("tq-4", 4, None),
+ ("bogus", 4, None),
+ ],
+ )
+ def test_kv_bits_resolution(self, quant, bits, expected):
+ # An explicit mlx_kv_quant wins over the superseded width, as on /load.
+ assert ri._mlx_estimate_kv_bits(quant, bits) == expected
+
+
+import glob # noqa: E402
+from dataclasses import replace # noqa: E402
+
+from core.inference import mlx_memory as mm # noqa: E402
+
+
[email protected](autouse = True)
+def _fresh_sizing_cache():
+ # Tests re-price one checkpoint under different patched internals.
+ from core.inference import mlx_inference
+
+ mm._clear_sizing_cache()
+ mlx_inference._kv_refusal_cache.clear()
+ yield
+ mm._clear_sizing_cache()
+ mlx_inference._kv_refusal_cache.clear()
+
+
+def test_a_load_is_sized_once_until_its_files_change(monkeypatch, tmp_path):
+ (tmp_path / "config.json").write_text("{}")
+ shard = tmp_path / "model.safetensors"
+ shard.write_bytes(b"x")
+ built = []
+ monkeypatch.setattr(
+ mm, "_size_load_uncached", lambda *a: built.append(a) or f"sizing{len(built)}"
+ )
+ first = mm._size_load(str(tmp_path), 4, None, True)
+ assert mm._size_load(str(tmp_path), 4, None, True) is first and len(built) == 1
+ assert mm._size_load(str(tmp_path), None, None, True) != first and len(built) == 2
+ shard.write_bytes(b"xy")
+ assert mm._size_load(str(tmp_path), 4, None, True) != first and len(built) == 3
+
+
+try:
+ import importlib
+
+ importlib.import_module("mlx.core")
+ importlib.import_module("mlx_lm")
+ _HAVE_MLX = True
+except Exception:
+ _HAVE_MLX = False
+
+_NEEDS_MLX = pytest.mark.skipif(not _HAVE_MLX, reason = "needs a working MLX install")
+
+
+def _mlx_raise():
+ raise NotImplementedError
+
+
+def _plan_entry(**over):
+ """One cache plan entry: a growing key/value cache unless overridden."""
+ return {
+ "const": 0.0,
+ "slope": 4096.0,
+ "quant_const": 0.0,
+ "quant_slope": 1152.0,
+ "bound_spec": None,
+ "conv_width": 0,
+ "block": 256,
+ "converts": True,
+ **over,
+ }
+
+
+_GROWING = _plan_entry()
+_RECURRENT = _plan_entry(
+ const = 2_195_456.0,
+ slope = 0.0,
+ quant_const = 2_195_456.0,
+ quant_slope = 0.0,
+ conv_width = 8192,
+ block = 1,
+ converts = False,
+)
+_UNQUANTIZED = {"quant_start": None, "prefill_chunk": mm.MLX_PREFILL_CHUNK}
+
+
+def _caption(
+ plan,
+ bits = None,
+ converted = False,
+ n_ctx = 32768,
+):
+ return mm._cache_width_name(plan, bits, converted, "bf16", n_ctx, mm.MLX_PREFILL_CHUNK)
+
+
+def _local_snapshot(repo):
+ """This repo's snapshot directory, skipping the test when it is not here."""
+ hub = os.path.expanduser("~/.cache/huggingface/hub")
+ hits = glob.glob(
+ os.path.join(hub, "models--" + repo.replace("/", "--"), "snapshots", "*", "config.json")
+ )
+ if not hits:
+ pytest.skip(f"{repo} not on this disk")
+ return os.path.dirname(hits[0])
+
+
+def _on_bfloat16_chip():
+ import mlx.core as mx
+ if mm._runtime_dtype() is not mx.bfloat16:
+ pytest.skip("measured on a chip the loader gives bfloat16")
+
+
+class TestCacheWidthName:
+ def test_only_the_entries_that_grow_name_the_cache(self):
+ hybrid = [_RECURRENT] * 30 + [_GROWING] * 10
+ assert _caption(hybrid) == "bf16"
+ assert _caption(hybrid, 4, True) == "4-bit"
+ assert _caption([_RECURRENT]) == "bf16"
+
+ def test_a_converted_run_is_named_as_its_own_control_is_labelled(self):
+ assert [_caption([_GROWING] * 10, b, True) for b in (8, 6, 5, 4, 3, 2)] == [
+ "8-bit",
+ "6-bit",
+ "5-bit",
+ "4-bit",
+ "3-bit",
+ "2-bit",
+ ]
+ assert _caption([_GROWING] * 10, 4, False) == "bf16"
+
+ def test_an_entry_that_cannot_convert_keeps_its_width_through_a_converting_run(self):
+ staying = _plan_entry(converts = False, quant_slope = _GROWING["slope"])
+ assert _caption([staying] * 57 + [_GROWING] * 3, 4, True) == "bf16/4-bit"
+ heavy, light = (
+ _plan_entry(quant_slope = 100_000.0),
+ _plan_entry(converts = False, slope = 1.0, quant_slope = 1.0),
+ )
+ assert _caption([light] * 50 + [heavy] * 2, 4, True) == "4-bit/bf16"
+
+ @_NEEDS_MLX
+ def test_a_bounded_entry_is_weighed_by_what_it_holds_and_not_by_its_slope(self):
+ from mlx_lm.models.cache import ChunkedKVCache
+
+ bounded = _plan_entry(converts = False, bound_spec = (ChunkedKVCache, "chunk_size", 8192))
+ plan = [bounded] * 3 + [_plan_entry(quant_slope = 1152.0)]
+ assert _caption(plan, 4, True, 8192).startswith("bf16")
+ assert _caption(plan, 4, True, 131_072).startswith("4-bit")
+
+ @_NEEDS_MLX
+ def test_every_width_a_cache_is_built_at_has_a_caption(self):
+ import mlx.core as mx
+ assert [mm._width_name(d) for d in (mx.bfloat16, mx.float16, mx.float32)] == [
+ "bf16",
+ "f16",
+ "f32",
+ ]
+
+
+class TestKvBytes:
+ def test_a_hybrid_plan_is_far_cheaper_than_charging_every_layer_as_attention(self):
+ hybrid = [_RECURRENT] * 30 + [_GROWING] * 10
+ assert (
+ mm._kv_bytes(hybrid, 131_072, **_UNQUANTIZED)[0] * 3
+ < mm._kv_bytes([_GROWING] * 40, 131_072, **_UNQUANTIZED)[0]
+ )
+
+ def test_a_runtime_that_declines_to_chunk_is_recognised_on_either_half(self):
+ assert mm._declines_to_chunk(type("M", (), {"no_chunked_prefill": True}), object())
+ assert mm._declines_to_chunk(object, type("T", (), {"no_chunked_prefill": True})())
+ assert not mm._declines_to_chunk(object, object())
+
+ def test_a_cache_converted_before_prefill_is_never_charged_at_full_width(self):
+ plan = [_GROWING, _RECURRENT]
+ quantized = mm._line_bytes(plan, 32768, 2048, quantized = True)
+ assert mm._kv_bytes(plan, 32768, 0, 2048) == (quantized, 0)
+ assert mm._kv_bytes(plan, 2048, 0, 2048)[0] == mm._line_bytes(plan, 2048, 2048, True)
+
+ def test_one_failed_conversion_refuses_the_whole_request(self):
+ def _entry(converts):
+ return type(
+ "Entry",
+ (),
+ {
+ "max_size": None,
+ "window_size": None,
+ "to_quantized": lambda self, **kw: "converted" if converts else _mlx_raise(),
+ },
+ )()
+
+ assert mm._quantize_like_runtime([_entry(True), _entry(False)], 4) is None
+
+ def test_a_windowed_entry_keeps_its_width_while_the_rest_convert(self):
+ from core.inference.mlx_inference import MLX_KV_GROUP_SIZE
+
+ def _entry(**window):
+ attrs = {"max_size": None, "window_size": None, **window}
+ attrs["to_quantized"] = lambda self, **kw: ("converted", kw)
+ return type("Entry", (), attrs)()
+
+ full, ring, window = _entry(), _entry(max_size = 512), _entry(window_size = 512)
+ quantized = ("converted", {"group_size": MLX_KV_GROUP_SIZE, "bits": 4})
+ assert mm._quantize_like_runtime([full, ring], 4) == [quantized, ring]
+ # Past two entries the last full-attention one stays float, as the load keeps it.
+ last = _entry()
+ assert mm._quantize_like_runtime([full, last, ring, window], 4) == [
+ quantized,
+ last,
+ ring,
+ window,
+ ]
+ assert mm._quantize_like_runtime([full, ring, window], 4) is None
+ assert mm._quantize_like_runtime([ring, window], 4) is None
+ assert mm._quantize_like_runtime([full], None) is None
+
+ @pytest.mark.skipif(not _HAVE_MLX, reason = "drives a real cache class")
+ def test_a_bounded_window_leaves_the_request_full_width(self):
+ from mlx_vlm.models.unlimited_ocr.language import RingSlidingKVCache
+
+ entry = RingSlidingKVCache(512)
+ assert entry.to_quantized(group_size = 64, bits = 4) is not None
+ assert mm._quantize_like_runtime([entry], 4) is None
+
+
+class TestComputeBytes:
+ def test_an_unreadable_width_is_not_guessed(self):
+ assert mm._compute_bytes(mm._config_widths({}), 2, 2048, [_GROWING], 8192, None) == 0
+
+ def test_the_calibrated_term_matches_the_measurement_it_came_from(self):
+ measured = 610 * 1024**2
+ priced = mm._compute_bytes(
+ (1024, 3072, 16), 2, mm.MLX_PREFILL_CHUNK, [_GROWING] * 28, 8192, None
+ )
+ assert 0.7 * measured < priced < 1.5 * measured, (
+ f"prices {priced / 1024**2:.0f} MiB where the load measured "
+ f"{measured / 1024**2:.0f} MiB"
+ )
+
+
+@_NEEDS_MLX
+class TestTheQuantizedAttentionRouteIsTheOneZooTakes:
+ """unsloth-zoo routes each call between a score matrix and a dequantized copy of the cache."""
+
+ GQA = (3584, 18944, 32, 8, 128, 0)
+ MHA = (3584, 18944, 32, 32, 128, 0)
+ WIDE = (3584, 18944, 16, 2, 256, 0)
+
+ @pytest.mark.parametrize(
+ "widths, chunk, copies",
+ [
+ (GQA, 2048, True),
+ (GQA, 256, True),
+ (MHA, 2048, True),
+ (MHA, 256, False),
+ (WIDE, 2048, False),
+ (WIDE, 256, False),
+ ],
+ )
+ def test_both_halves_of_the_wrappers_choice_are_asked(self, widths, chunk, copies):
+ assert (mm._dequantized_row_bytes(widths, 2, 4, 64, chunk) is not None) is copies
+
+ def test_the_copy_charged_is_the_cache_the_wrapper_would_make(self):
+ import mlx.core as mx
+
+ from unsloth_zoo.mlx.attention import _row_bytes
+
+ cache = mx.quantize(mx.zeros((1, 8, 1, 128), dtype = mx.bfloat16), group_size = 64, bits = 4)
+ assert mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048) == (
+ 2 * 8 * _row_bytes(cache, 64, mx.bfloat16)
+ )
+
+ def test_a_copied_route_is_charged_the_copy_and_not_the_scores(self):
+ row = mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048)
+ floor = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, None)
+ priced = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, 2048, 4, 64)
+ scores = mm._widest_quantized_scores(32768, 2048, 2048) * 32 * mm._QUANT_SCORE_DTYPE_SIZE
+ assert priced - floor == row * 32768 < scores
+
+ def test_the_decode_step_still_scores_where_that_is_the_wider_of_the_two(self):
+ widths = (3584, 18944, 128, 1, 64, 0)
+ row = mm._dequantized_row_bytes(widths, 2, 8, 64, 2048)
+ floor = mm._compute_bytes(widths, 2, 2048, [], 32768, None)
+ charged = mm._compute_bytes(widths, 2, 2048, [], 32768, 2048, 8, 64) - floor
+ assert row is not None and charged == (32769) * 128 * mm._QUANT_SCORE_DTYPE_SIZE
+ assert charged > row * 32768
+
+ def test_a_zoo_that_cannot_route_it_keeps_the_scores_whole(self, monkeypatch):
+ import builtins
+
+ real = builtins.__import__
+
+ def _no_wrapper(name, *a, **kw):
+ if name == "unsloth_zoo.mlx.attention":
+ raise ImportError("this zoo predates the wrapper")
+ return real(name, *a, **kw)
+
+ monkeypatch.setattr(builtins, "__import__", _no_wrapper)
+ assert mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048) is None
+ floor = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, None)
+ priced = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, 2048, 4, 64)
+ assert priced - floor == (
+ mm._widest_quantized_scores(32768, 2048, 2048) * 32 * mm._QUANT_SCORE_DTYPE_SIZE
+ )
+
+ @pytest.mark.parametrize(
+ "stated, expected",
+ [
+ ({"num_key_value_heads": 8, "head_dim": 128}, (8, 128, 0)),
+ ({}, (32, 112, 0)),
+ ({"num_key_value_heads": 4}, (4, 112, 0)),
+ ({"head_dim": 64}, (32, 64, 0)),
+ ({"head_dim": 192, "v_head_dim": 128}, (32, 192, 128)),
+ ],
+ )
+ def test_the_geometry_the_route_reads_comes_off_the_checkpoint(self, stated, expected):
+ from types import SimpleNamespace
+
+ config = {"hidden_size": 3584, "num_attention_heads": 32, **stated}
+ assert mm._config_widths(config)[3:] == expected
+ assert mm._tower_widths(SimpleNamespace(args = SimpleNamespace(**config)))[3:] == expected
+
+ def test_a_tower_whose_values_are_narrower_keeps_the_scores(self):
+ assert mm._dequantized_row_bytes((3584, 18944, 32, 8, 128, 64), 2, 4, 64, 2048) is None
+ assert mm._dequantized_row_bytes((3584, 18944, 32, 8, 192, 128), 2, 4, 64, 2048) is None
+ assert mm._dequantized_row_bytes((3584, 18944, 32, 8, 128, 128), 2, 4, 64, 2048) == (
+ mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048)
+ )
+
+ def test_a_tower_that_states_nothing_is_still_priced_from_its_checkpoint(self):
+ assert len(mm._tower_widths(object())) == len(mm._config_widths({}))
+ config = {
+ "hidden_size": 3584,
+ "intermediate_size": 18944,
+ "num_attention_heads": 32,
+ "num_key_value_heads": 8,
+ "head_dim": 128,
+ }
+ merged = tuple(
+ tower or checkpoint
+ for tower, checkpoint in zip(mm._tower_widths(object()), mm._config_widths(config))
+ )
+ assert mm._dequantized_row_bytes(merged, 2, 4, 64, 2048) == (
+ mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048)
+ )
+
+ def test_an_unquantized_load_never_reaches_the_route(self):
+ assert mm._dequantized_row_bytes(self.GQA, 2, None, 64, 2048) is None
+ assert mm._dequantized_row_bytes((3584, 18944, 32, 0, 0, 0), 2, 4, 64, 2048) is None
+
+
[email protected](not _HAVE_MLX, reason = "imports architecture modules")
+class TestProbeFollowsTheLoadersRoute:
+ """The estimator must not price a load path that cannot be taken."""
+
+ def test_a_vision_model_is_never_priced_from_mlx_lms_copy_of_it(self):
+ import importlib
+
+ config = {"model_type": "kimi_vl", "vision_config": {"depth": 2}}
+ assert importlib.import_module("mlx_lm.models.kimi_vl")
+ assert {
+ model_class.__module__.split(".")[0]
+ for _, _, model_class in mm._probe_models(config, None)
+ } == {"mlx_vlm"}
+
+ @pytest.mark.parametrize(
+ "architecture, resolved",
+ [("DeepseekOCRForCausalLM", "deepseekocr"), ("DeepseekOCR2ForCausalLM", "deepseekocr_2")],
+ )
+ def test_a_config_is_read_as_the_loader_resolves_it_not_as_stated(
+ self, tmp_path, architecture, resolved
+ ):
+ (tmp_path / "config.json").write_text(
+ json.dumps({"model_type": "deepseek_vl_v2", "architectures": [architecture]})
+ )
+ assert mm._snapshot_config(str(tmp_path))["model_type"] == resolved
+
+ def test_a_checkpoint_supplying_its_own_module_is_refused_and_not_imported(self):
+ with pytest.raises(ValueError, match = "its own model module"):
+ list(mm._probe_models({"model_type": "qwen3", "model_file": "modeling.py"}, None))
+
+ def test_the_probe_hands_the_forward_pass_a_model_in_eval_mode(self):
+ import mlx.core as mx
+ config = json.loads(
+ (Path(_local_snapshot("unsloth/Qwen3-4B-Thinking-2507")) / "config.json").read_text()
+ )
+ assert next(iter(mm._probe_models(config, mx.bfloat16)))[0]().training is False
+
+ @pytest.mark.parametrize(
+ "chip, expected", [("Apple M1 Max", "float16"), ("Apple M3 Max", "bfloat16")]
+ )
+ def test_the_chip_decides_the_width_the_loader_installs(self, monkeypatch, chip, expected):
+ import mlx.core as mx
+ monkeypatch.setattr(mx, "device_info", lambda: {"device_name": chip})
+ assert mm._runtime_dtype() is getattr(mx, expected)
+
+
[email protected](not _HAVE_MLX, reason = "asks the installed runtime")
+class TestGenerationSettingsComeFromTheLoader:
+ """What a load prefills at is read off the runtime, except where Studio pins it."""
+
+ def test_each_path_is_read_from_the_function_that_would_run_it(self, monkeypatch):
+ import inspect
+
+ from core.inference import mlx_inference as mi
+
+ monkeypatch.setattr(mi, "mlx_vlm_snapshot_store_available", lambda: False)
+ for vision, drafted in ((False, False), (True, False), (False, True), (True, True)):
+ step = mi._generation_step(vision = vision, drafted = drafted)
+ assert mi.mlx_prefill_chunk(vision = vision, drafted = drafted) == (
+ inspect.signature(step).parameters["prefill_step_size"].default
+ )
+ assert mi._generation_step(vision = True, drafted = False).__module__.startswith("mlx_vlm")
+ assert mi._generation_step(vision = False, drafted = False).__module__.startswith("mlx_lm")
+ assert mi.mlx_prefill_chunk(drafted = True) != mi.mlx_prefill_chunk()
+ assert mi.mlx_prefill_chunk(vision = True, drafted = True) == mi.mlx_prefill_chunk(vision = True)
+
+ def test_a_pinned_vision_step_outranks_the_runtimes_own(self, monkeypatch):
+ from core.inference import mlx_inference as mi
+
+ vlm_default = mi._generation_default("prefill_step_size", 0, vision = True, drafted = False)
+ monkeypatch.setattr(mi, "vlm_prefill_step", lambda: vlm_default + 256)
+ media, bare = {"image_token_id": 151655}, {"model_type": "vlm"}
+ for store, config, expected in (
+ (True, media, vlm_default + 256),
+ (True, bare, vlm_default),
+ (False, media, vlm_default),
+ (True, None, vlm_default),
+ ):
+ monkeypatch.setattr(mi, "mlx_vlm_snapshot_store_available", lambda p = store: p)
+ assert mi.mlx_prefill_chunk(vision = True, config = config) == expected
+ assert mi.mlx_prefill_chunk(config = media) == mi.MLX_PREFILL_CHUNK_FALLBACK
+
+ def test_the_step_priced_and_the_store_that_pins_it_answer_together(self, monkeypatch):
+ """The estimate is only right while these two read the same capability."""
+ import importlib
+
+ from core.inference.mlx_inference import (
+ MLXInferenceBackend,
+ mlx_vlm_snapshot_store_available,
+ )
+
+ vlm_generate = importlib.import_module("mlx_vlm.generate")
+ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)
+ backend._memory_limits_applied = {}
+ for cached_tokens, built in ((True, True), (False, False)):
+ backend._vlm_snapshot_store = None
+ backend._vlm_snapshot_store_unavailable = False
+ monkeypatch.setattr(
+ vlm_generate,
+ "GenerationResult",
+ type("R", (), {"cached_tokens": 0} if cached_tokens else {}),
+ )
+ assert mlx_vlm_snapshot_store_available() is built
+ assert (backend._vlm_prompt_cache_store() is not None) is built
+
+ @pytest.mark.parametrize("chunk", [None, 0, True, "2048"])
+ def test_a_runtime_that_states_no_usable_value_falls_back(self, monkeypatch, chunk):
+ from core.inference import mlx_inference as mi
+
+ def _stated(**kw):
+ return lambda prefill_step_size = chunk: None
+
+ monkeypatch.setattr(mi, "_generation_step", _stated)
+ assert mi.mlx_prefill_chunk() == mi.MLX_PREFILL_CHUNK_FALLBACK
+
+ def test_the_estimator_tells_the_loader_which_package_would_load_it(self, monkeypatch):
+ from core.inference import mlx_inference as mi
+
+ seen = []
+ monkeypatch.setattr(mi, "mlx_prefill_chunk", lambda **kw: seen.append(kw["vision"]) or 2048)
+ vision = mm._generation_settings({"model_type": "kimi_vl", "vision_config": {"depth": 2}})
+ assert mm._generation_settings({"model_type": "qwen3"}) == vision == (2048, 64)
+ assert seen == [True, False]
+
+ def test_every_loader_quantizes_a_converted_cache_from_the_first_token(self, monkeypatch):
+ entry = {
+ "bytes": 1,
+ "quant_bytes": 1,
+ "bound_spec": None,
+ "conv_width": 0,
+ "block": 1,
+ "converts": True,
+ }
+ probe = {
+ "entries": [entry],
+ "quantized": True,
+ "whole_prompt": False,
+ "layers": 1,
+ "widths": (1, 1, 1),
+ }
+ monkeypatch.setattr(mm, "_probe", lambda *a, **kw: probe)
+ for vision in (True, False):
+ monkeypatch.setattr(mm, "_loads_as_vision", lambda _c, v = vision: v)
+ assert mm._cache_plan({}, None, 4)[1] == 0
+ probe["quantized"] = False
+ assert mm._cache_plan({}, None, 4)[1] is None
+
+ def test_a_host_without_the_loader_prices_the_fallback(self, monkeypatch):
+ import builtins
+
+ real = builtins.__import__
+
+ def _no_loader(name, *a, **kw):
+ if name == "core.inference.mlx_inference":
+ raise ImportError("no loader here")
+ return real(name, *a, **kw)
+
+ monkeypatch.setattr(builtins, "__import__", _no_loader)
+ assert mm._generation_settings({"model_type": "llama"}) == (mm.MLX_PREFILL_CHUNK, 64)
+
+
+@_NEEDS_MLX
[email protected]("reported, compute", [(None, 1_035_321_999), (512, 903_201_423)])
+def test_the_estimate_prices_the_chunk_the_loader_reports(monkeypatch, reported, compute):
+ _on_bfloat16_chip()
+ if reported is not None:
+ from core.inference import mlx_inference as mi
+ monkeypatch.setattr(mi, "mlx_prefill_chunk", lambda **kw: reported)
+ breakdown = mm.mlx_memory_breakdown(
+ _local_snapshot("unsloth/Qwen3-4B-Thinking-2507"),
+ n_ctx = 32768,
+ load_in_4bit = True,
+ kv_bits = 4,
+ )
+ assert breakdown is not None and breakdown.compute_bytes == compute
+ override = mm.mlx_memory_breakdown(
+ _local_snapshot("unsloth/Qwen3-4B-Thinking-2507"),
+ n_ctx = 32768,
+ load_in_4bit = True,
+ kv_bits = 4,
+ prefill_chunk = 512,
+ )
+ assert override.compute_bytes == 903_201_423
+
+
+@_NEEDS_MLX
+def test_a_model_mlx_vlm_would_diffuse_is_refused_rather_than_priced():
+ snapshot = _local_snapshot("mlx-community/diffusiongemma-26B-A4B-it-4bit")
+ assert mm._routes_to_diffusion(mm._snapshot_config(snapshot)) is True
+ assert mm.mlx_memory_breakdown(snapshot, n_ctx = 32768, load_in_4bit = True) is None
+
+
+@_NEEDS_MLX
+def test_an_ordinary_vision_model_is_not_mistaken_for_a_diffusion_one():
+ config = {"model_type": "kimi_vl", "vision_config": {"depth": 2}}
+ assert mm._routes_to_diffusion(config) is False
+ assert mm._generation_settings(config)[0] > 0
+
+
+@_NEEDS_MLX
+class TestDiffusionRouting:
+ """The markers gate a build; mlx-vlm's own predicate is the verdict."""
+
+ @staticmethod
+ def _resolving(
+ monkeypatch,
+ *,
+ canvas,
+ mask,
+ build = lambda config: object(),
+ ):
+ monkeypatch.setattr(
+ mm,
+ "_loader_config",
+ lambda arch, config: SimpleNamespace(canvas_length = canvas, mask_token_id = mask),
+ )
+ monkeypatch.setattr(
+ "mlx_vlm.utils.get_model_and_args",
+ lambda config: (SimpleNamespace(Model = build), None),
+ )
+
+ @pytest.mark.parametrize("verdict", [True, False])
+ def test_the_predicate_outranks_the_marker(self, monkeypatch, verdict):
+ from mlx_vlm.generate import diffusion as vlm_diffusion
+
+ self._resolving(monkeypatch, canvas = None, mask = 100)
+ monkeypatch.setattr(vlm_diffusion, "is_diffusion_model", lambda model, kw: verdict)
+ assert mm._routes_to_diffusion({"model_type": "whatever"}) is verdict
+
+ def test_a_diverted_load_refuses_without_needing_a_checkpoint(self, monkeypatch):
+ from mlx_vlm.generate import diffusion as vlm_diffusion
+
+ self._resolving(monkeypatch, canvas = 512, mask = None)
+ monkeypatch.setattr(vlm_diffusion, "is_diffusion_model", lambda model, kw: True)
+ monkeypatch.setattr(mm, "_loads_as_vision", lambda config: True)
+ with pytest.raises(ValueError, match = "diffusion generator"):
+ mm._generation_settings({"model_type": "whatever"})
+
+ def test_an_architecture_carrying_no_marker_is_never_built(self, monkeypatch):
+ def _explode(config):
+ raise AssertionError("built a wrapper for a config carrying no marker")
+
+ self._resolving(monkeypatch, canvas = None, mask = None, build = _explode)
+ assert mm._routes_to_diffusion({"model_type": "whatever"}) is False
+
+ def test_a_marked_model_that_cannot_be_placed_is_refused_not_assumed(self, monkeypatch):
+ def _explode(config):
+ raise RuntimeError("this wrapper cannot be built")
+
+ self._resolving(monkeypatch, canvas = 512, mask = None, build = _explode)
+ with pytest.raises(RuntimeError):
+ mm._routes_to_diffusion({"model_type": "whatever"})
+ with pytest.raises(RuntimeError):
+ mm._generation_settings({"model_type": "whatever", "vision_config": {"depth": 2}})
+
+
[email protected](not _HAVE_MLX, reason = "drives real cache classes")
+def test_the_peak_of_a_bounded_cache_is_measured_not_derived():
+ from mlx_lm.models import cache as C
+
+ for bound, attribute, value, n_ctx, chunk, expected in (
+ (C.RotatingKVCache, "max_size", 512, 100, 2048, 355),
+ (C.RotatingKVCache, "max_size", 512, 255, 2048, 510),
+ (C.RotatingKVCache, "max_size", 512, 1024, 2048, 1023),
+ (C.ChunkedKVCache, "chunk_size", 512, 4096, 2048, 2560),
+ (C.ChunkedKVCache, "chunk_size", 512, 2560, 2048, 2048),
+ (C.RotatingKVCache, "max_size", 512, 256, 1, 512),
+ (C.RotatingKVCache, "max_size", 512, 10**12, 2048, 2559),
+ ):
+ assert (
+ mm._bounded_peak(bound, attribute, value, n_ctx, chunk) == expected
+ ), f"{bound.__name__}({value}) at {n_ctx}"
+ assert mm._bound_spec(type("E", (), {"max_size": 512})())
+ assert mm._bound_spec(type("E", (), {"chunk_size": 512})())
+ spec = (C.RotatingKVCache, "max_size", 512)
+ entry = {"slope": 1.0, "block": mm.MLX_KV_BLOCK, "bound_spec": spec}
+ for n_ctx, charged in ((253, 508), (100, 355), (1024, 1023), (10**12, 2559)):
+ assert mm._held_tokens(entry, n_ctx, 2048) == charged
+
+
[email protected](not _HAVE_MLX, reason = "builds a real architecture")
+class TestPricingALoad:
+ _CONFIG = {
+ "model_type": "llama",
+ "hidden_size": 256,
+ "num_hidden_layers": 2,
+ "intermediate_size": 512,
+ "num_attention_heads": 4,
+ "num_key_value_heads": 4,
+ "rms_norm_eps": 1e-5,
+ "vocab_size": 512,
+ "rope_theta": 10000.0,
+ }
+
+ def _checkpoint(
+ self,
+ tmp_path,
+ *,
+ prefix = "",
+ ):
+ import mlx.core as mx
+ from mlx.utils import tree_flatten
+
+ config = dict(self._CONFIG, tie_word_embeddings = True)
+ (tmp_path / "config.json").write_text(json.dumps(config))
+ built = mm._whole_model(config, mx.bfloat16)
+ mx.save_safetensors(
+ str(tmp_path / "model.safetensors"),
+ {
+ prefix + name: mx.zeros(value.shape, dtype = mx.bfloat16)
+ for name, value in tree_flatten(built.parameters())
+ },
+ )
+ return config
+
+ def test_a_float32_checkpoint_goes_resident_at_the_width_the_chip_gives_it(self):
+ if mm._runtime_dtype().size != 2:
+ pytest.skip("measured on a chip the loader gives a 16-bit width")
+ snapshot = _local_snapshot("hf-internal-testing/tiny-random-LlamaForCausalLM")
+ config = json.loads((Path(snapshot) / "config.json").read_text())
+ assert mm.mlx_weight_bytes(snapshot, config, load_in_4bit = False) == 2_064_544
+
+ def test_a_quantizing_load_is_priced_below_the_shards(self, tmp_path):
+ config = self._checkpoint(tmp_path)
+ wide = mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = False)
+ quantized = mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = True)
+ assert (wide, quantized) == (2_886_144, 1_001_984)
+ assert quantized < wide < mm._shard_bytes(str(tmp_path), config) == 2_888_265
+
+ def test_shards_that_name_nothing_the_architecture_has_are_not_priced_from_it(self, tmp_path):
+ config = self._checkpoint(tmp_path, prefix = "draft.")
+ with pytest.raises(ValueError, match = "does not supply"):
+ mm._resident_bytes(str(tmp_path), config, quantize = True)
+ for flag in (True, False):
+ assert mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = flag) == (
+ mm._shard_bytes(str(tmp_path), config)
+ )
+
+
[email protected](not _HAVE_MLX, reason = "asks the installed loader")
+class TestALoadWithNoFootprintToQuote:
+ def test_quantization_metadata_no_load_can_honour_is_refused_not_priced(self, tmp_path):
+ per_module = {"model.layers.0.self_attn.q_proj": {"bits": 4, "group_size": 64}}
+ for spelling in ("quantization", "quantization_config"):
+ stated = {"model_type": "llama", spelling: per_module}
+ assert mm._load_is_refused(str(tmp_path), stated, True)
+ assert mm._load_is_refused(str(tmp_path), stated, False) is None
+ bnb = {"model_type": "llama", spelling: {"quant_method": "bitsandbytes"}}
+ for flag in (True, False):
+ assert "bitsandbytes" in (mm._load_is_refused(str(tmp_path), bnb, flag) or "")
+
+ @pytest.mark.parametrize(
+ "repo, names",
+ [
+ ("OsaurusAI/ZAYA1-VL-8B-MXFP4", "no home for"),
+ ("Qwen/Qwen3-Embedding-0.6B", "QK-norm"),
+ ("z-lab/Qwen3.6-27B-DFlash", "QK-norm"),
+ ("z-lab/gemma-4-31B-it-DFlash", "QK-norm"),
+ ("yuyijiong/Qwen3.5-4B-Eagle3", "no home for"),
+ ],
+ )
+ def test_a_checkpoint_no_load_starts_from_is_refused(self, repo, names):
+ snapshot = _local_snapshot(repo)
+ config = mm._snapshot_config(snapshot)
+ for flag in (True, False):
+ assert names in (mm._load_is_refused(snapshot, config, flag) or "")
+ assert mm.mlx_memory_breakdown(snapshot, n_ctx = 2048, load_in_4bit = flag) is None
+
+ def test_no_checkpoint_on_this_disk_is_refused_for_an_invented_reason(self):
+ hub = os.path.expanduser("~/.cache/huggingface/hub")
+ seen = accepted = 0
+ for path in glob.glob(os.path.join(hub, "models--*", "snapshots", "*", "config.json")):
+ directory = os.path.dirname(path)
+ config = mm._snapshot_config(directory)
+ if not isinstance(config, dict) or not config.get("model_type"):
+ continue
+ try:
+ if not mm.mlx_shard_files(directory, config):
+ continue
+ except Exception:
+ continue
+ seen += 1
+ refusals = [mm._load_is_refused(directory, config, flag) for flag in (True, False)]
+ for refusal in refusals:
+ assert refusal is None or any(
+ reason in refusal for reason in ("QK-norm", "no home for", "bitsandbytes")
+ ), f"{path}: {refusal}"
+ accepted += refusals[0] is None
+ if not seen:
+ pytest.skip("no checkpoints on this disk")
+ assert accepted > seen * 0.9, "the gate has started refusing wholesale"
+
+ def test_extras_a_registered_rule_filters_are_not_refused(self):
+ snapshot = _local_snapshot("google/gemma-4-E2B")
+ config = mm._snapshot_config(snapshot)
+ assert mm._load_is_refused(snapshot, config, True) is None
+ assert mm.mlx_weight_bytes(snapshot, config, load_in_4bit = True) == 7_510_668_230
+
+ _FILTERED = [
+ "language_model.model.per_layer_model_projection.biases",
+ "language_model.model.per_layer_model_projection.scales",
+ ]
+ _GEMMA4 = {"model_type": "gemma4", "vision_config": {"model_type": "gemma4"}}
+
+ @pytest.mark.parametrize(
+ "config, extras, names",
+ [
+ (_GEMMA4, _FILTERED, None),
+ (_GEMMA4, _FILTERED + ["mystery.weight"], "mystery.weight"),
+ ({"model_type": "an-architecture-with-no-rule"}, ["d2t", "fc.weight"], "d2t"),
+ ],
+ )
+ def test_a_rule_exempts_only_the_keys_it_names(self, config, extras, names):
+ refusal = mm._extra_tensors_refused(object(), config, extras)
+ assert refusal is None if names is None else (names in refusal and "no home for" in refusal)
+
+
+class TestShardsTheLoaderReads:
+ """Which safetensors beside a config actually become weights."""
+
+ _VISION = {"model_type": "llava", "vision_config": {"hidden_size": 8}}
+
+ def _spread(self, tmp_path, config):
+ for name in (
+ "model-00001.safetensors",
+ "adapter_model.safetensors",
+ "consolidated.safetensors",
+ ):
+ (tmp_path / name).write_bytes(b"x" * 100)
+ return sorted(os.path.basename(p) for p in mm.mlx_shard_files(str(tmp_path), config))
+
+ def test_the_text_loader_reads_model_shards_only(self, tmp_path):
+ assert self._spread(tmp_path, {"model_type": "llama"}) == ["model-00001.safetensors"]
+ adapter_only = tmp_path / "lora"
+ adapter_only.mkdir()
+ (adapter_only / "adapter_model.safetensors").write_bytes(b"x" * 100)
+ assert mm.mlx_shard_files(str(adapter_only), {"model_type": "llama"}) == []
+
+ def test_the_vision_loader_reads_the_index_before_the_directory(self, tmp_path):
+ assert self._spread(tmp_path, self._VISION) == [
+ "adapter_model.safetensors",
+ "model-00001.safetensors",
+ ]
+ (tmp_path / "model.safetensors.index.json").write_text(
+ json.dumps({"weight_map": {"a.weight": "model-00001.safetensors"}})
+ )
+ assert self._spread(tmp_path, self._VISION) == ["model-00001.safetensors"]
+
+ @_NEEDS_MLX
+ @pytest.mark.parametrize("declared", [0, 1_000_000, 1 << 40])
+ def test_a_shard_is_not_read_past_the_header_it_declares(self, tmp_path, declared):
+ import mlx.core as mx
+
+ shard = tmp_path / "model.safetensors"
+ body = json.dumps({"a.weight": {"dtype": "F16", "shape": [2], "data_offsets": [0, 4]}})
+ shard.write_bytes(declared.to_bytes(8, "little") + body.encode() + b"\0" * 4)
+ with pytest.raises(ValueError, match = "safetensors header"):
+ mm._checkpoint_tensors(str(tmp_path), {"model_type": "llama"}, mx.bfloat16)
+ shard.write_bytes(len(body).to_bytes(8, "little") + body.encode() + b"\0" * 4)
+ assert list(mm._checkpoint_tensors(str(tmp_path), {"model_type": "llama"}, mx.bfloat16))
+
+ @_NEEDS_MLX
+ def test_a_header_is_capped_below_the_shard_it_sits_in(self, tmp_path, monkeypatch):
+ import mlx.core as mx
+
+ body = json.dumps({"a.weight": {"dtype": "F16", "shape": [2], "data_offsets": [0, 4]}})
+ (tmp_path / "model.safetensors").write_bytes(
+ len(body).to_bytes(8, "little") + body.encode() + b"\0" * 4
+ )
+ read = lambda: mm._checkpoint_tensors(str(tmp_path), {"model_type": "llama"}, mx.bfloat16)
+ assert list(read())
+ monkeypatch.setattr(mm, "_MAX_SAFETENSORS_HEADER", len(body) - 1)
+ with pytest.raises(ValueError, match = "safetensors header"):
+ read()
+
+ @pytest.mark.parametrize(
+ "shard",
+ [
+ "../secret.safetensors",
+ "a/../../secret.safetensors",
+ "/etc/passwd",
+ "..\\secret.safetensors",
+ "C:/secret.safetensors",
+ ],
+ )
+ def test_an_index_naming_a_shard_outside_the_snapshot_is_refused(self, tmp_path, shard):
+ # A cached repo's index must not steer sizing at files beside or above its snapshot.
+ (tmp_path / "secret.safetensors").write_bytes(b"x" * 100)
+ snapshot = tmp_path / "snap"
+ snapshot.mkdir()
+ (snapshot / "model-00001.safetensors").write_bytes(b"x" * 100)
+ (snapshot / "model.safetensors.index.json").write_text(
+ json.dumps({"weight_map": {"a.weight": "model-00001.safetensors", "b.weight": shard}})
+ )
+ with pytest.raises(ValueError, match = "outside the snapshot"):
+ mm.mlx_shard_files(str(snapshot), self._VISION)
+
+ def test_a_nested_shard_inside_the_snapshot_is_still_read(self, tmp_path):
+ (tmp_path / "sub").mkdir()
+ (tmp_path / "sub" / "model-00001.safetensors").write_bytes(b"x" * 100)
+ (tmp_path / "model.safetensors.index.json").write_text(
+ json.dumps({"weight_map": {"a.weight": "sub/./model-00001.safetensors"}})
+ )
+ assert [os.path.basename(p) for p in mm.mlx_shard_files(str(tmp_path), self._VISION)] == [
+ "model-00001.safetensors"
+ ]
+
+ @pytest.mark.parametrize(
+ "weight_map", [["model-00001.safetensors"], "model-00001.safetensors", 7, True, None]
+ )
+ def test_an_index_whose_weight_map_is_not_a_mapping_is_fatal(self, tmp_path, weight_map):
+ (tmp_path / "model.safetensors.index.json").write_text(
+ json.dumps({"weight_map": weight_map})
+ )
+ with pytest.raises(ValueError, match = "no weight map"):
+ mm.mlx_shard_files(str(tmp_path), self._VISION)
+
+
+@_NEEDS_MLX
+ "repo, n_ctx, kv_bits, weights, kv, compute, layers, caption",
+ [
+ (
+ "unsloth/Qwen3-4B-Thinking-2507",
+ 32768,
+ None,
+ 2_822_044_672,
+ 4_869_586_944,
+ 863_355_535,
+ 36,
+ "bf16",
+ ),
+ (
+ "unsloth/Qwen3-4B-Thinking-2507",
+ 32768,
+ 4,
+ 2_822_044_672,
+ 1_369_571_328,
+ 1_035_321_999,
+ 36,
+ "4-bit",
+ ),
+ (
+ "unsloth/Qwen3.6-35B-A3B-UD-MLX-4bit",
+ 32768,
+ None,
+ 21_634_993_114,
+ 740_720_640,
+ 943_440_527,
+ 40,
+ "bf16",
+ ),
+ ("unsloth/gemma-3-270m-it", 8192, None, 392_058_112, 65_258_496, 725_729_935, 18, "bf16"),
+ (
+ "mlx-community/deepseek-vl2-tiny-4bit",
+ 4096,
+ None,
+ 2_099_712_075,
+ 267_386_880,
+ 803_717_775,
+ 12,
+ "bf16",
+ ),
+ (
+ "Qwen/Qwen2.5-VL-3B-Instruct",
+ 32768,
+ 4,
+ 3_520_856_064,
+ 342_392_832,
+ 753_582_735,
+ 36,
+ "4-bit",
+ ),
+ (
+ "Qwen/Qwen2.5-VL-3B-Instruct",
+ 5001,
+ 4,
+ 3_520_856_064,
+ 56_852_480,
+ 717_152_431,
+ 36,
+ "4-bit",
+ ),
+ (
+ "Qwen/Qwen2.5-VL-3B-Instruct",
+ 5000,
+ 4,
+ 3_520_856_064,
+ 56_852_480,
+ 717_151_119,
+ 36,
+ "4-bit",
+ ),
+ (
+ "mlx-community/gemma-3n-E2B-it-4bit",
+ 4096,
+ None,
+ 4_462_976_625,
+ 60_784_640,
+ 705_544_847,
+ 30,
+ "bf16",
+ ),
+ (
+ "mlx-community/Molmo-7B-D-0924-4bit",
+ 4096,
+ None,
+ 5_298_715_127,
+ 249_561_088,
+ 1_281_737_359,
+ 28,
+ "bf16",
+ ),
+ (
+ "hf-internal-testing/tiny-random-LlamaForCausalLM",
+ 512,
+ None,
+ 2_061_600,
+ 98_304,
+ 688_341_647,
+ 2,
+ "bf16",
+ ),
+ ],
+)
+def test_a_real_checkpoint_is_priced_to_the_byte(
+ repo, n_ctx, kv_bits, weights, kv, compute, layers, caption
+):
+ _on_bfloat16_chip()
+ breakdown = mm.mlx_memory_breakdown(
+ _local_snapshot(repo), n_ctx = n_ctx, load_in_4bit = True, kv_bits = kv_bits
+ )
+ assert breakdown is not None
+ assert (breakdown.weights_bytes, breakdown.kv_bytes) == (weights, kv)
+ assert breakdown.compute_bytes == compute
+ assert breakdown.layer_count == layers
+ assert breakdown.cache_type_kv == caption
+
+
+@_NEEDS_MLX
[email protected]("kv_bits, caption", [(6, "6-bit"), (8, "8-bit")])
+def test_the_caption_names_a_width_the_real_conversion_reaches(kv_bits, caption):
+ _on_bfloat16_chip()
+ breakdown = mm.mlx_memory_breakdown(
+ _local_snapshot("unsloth/Qwen3-4B-Thinking-2507"),
+ n_ctx = 32768,
+ load_in_4bit = True,
+ kv_bits = kv_bits,
+ )
+ assert breakdown is not None and breakdown.cache_type_kv == caption
+
+
+class TestTheContextSearch:
+ """The search itself, priced by a stub so it runs where MLX and the checkpoints do not."""
+
+ @staticmethod
+ def _linear(monkeypatch, unpriceable = ()):
+ """A byte per token, so every expected answer below is readable off the budget."""
+
+ def _priced(sizing, n_ctx):
+ return None if n_ctx in unpriceable else SimpleNamespace(total_bytes = n_ctx)
+
+ monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: object())
+ monkeypatch.setattr(mm, "_priced_at", _priced)
+
+ def test_the_answer_is_the_largest_whole_block_under_budget(self, monkeypatch):
+ self._linear(monkeypatch)
+ assert mm.mlx_fit_context("x", budget_bytes = 7000, max_ctx = 8192) == 6912
+
+ def test_a_floor_between_blocks_rounds_up_rather_than_under_the_minimum(self, monkeypatch):
+ self._linear(monkeypatch)
+ assert mm.mlx_fit_context("x", budget_bytes = 4200, max_ctx = 8192, min_ctx = 4100) is None
+ assert mm.mlx_fit_context("x", budget_bytes = 5000, max_ctx = 8192, min_ctx = 4100) == 4864
+
+ def test_a_context_that_cannot_be_priced_abandons_the_fit(self, monkeypatch):
+ self._linear(monkeypatch, unpriceable = (6144,))
+ assert mm.mlx_fit_context("x", budget_bytes = 7000, max_ctx = 8192) is None
+
+ def test_a_ceiling_that_already_fits_is_left_alone(self, monkeypatch):
+ self._linear(monkeypatch)
+ assert mm.mlx_fit_context("x", budget_bytes = 9000, max_ctx = 8192) is None
+
+ def test_a_load_that_cannot_be_sized_is_not_fitted(self, monkeypatch):
+ monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: None)
+ assert mm.mlx_fit_context("x", budget_bytes = 7000, max_ctx = 8192) is None
+
+
+@_NEEDS_MLX
[email protected]("budget_gib, fitted", [(6, 18_432), (12, 61_952), (24, 149_504)])
+def test_a_real_checkpoint_is_fitted_to_the_byte(budget_gib, fitted):
+ _on_bfloat16_chip()
+ snapshot = _local_snapshot("unsloth/Qwen3-4B-Thinking-2507")
+ budget = budget_gib * 1024**3
+ assert (
+ mm.mlx_fit_context(snapshot, budget_bytes = budget, max_ctx = 262_144, load_in_4bit = True)
+ == fitted
+ )
+ priced = mm.mlx_memory_breakdown(snapshot, n_ctx = fitted, load_in_4bit = True)
+ over = mm.mlx_memory_breakdown(snapshot, n_ctx = fitted + mm.MLX_KV_BLOCK, load_in_4bit = True)
+ assert priced.total_bytes <= budget < over.total_bytes
+ assert mm.mlx_memory_breakdown(snapshot, n_ctx = float("nan"), load_in_4bit = True) is None
+
+
+@_NEEDS_MLX
+ "repo, kv_bits, whole_prompt",
+ [
+ ("unsloth/Qwen3-4B-Thinking-2507", 4, False),
+ ("unsloth/Qwen3-4B-Thinking-2507", None, True),
+ ("unsloth/gemma-3-270m-it", None, False),
+ ("unsloth/Qwen3.6-35B-A3B-UD-MLX-4bit", None, False),
+ ],
+)
+def test_the_total_never_falls_as_the_context_grows(repo, kv_bits, whole_prompt):
+ sizing = mm._size_load(_local_snapshot(repo), kv_bits, None, True)
+ if whole_prompt:
+ forced = replace(sizing, facts = {**sizing.facts, "whole_prompt": True})
+ assert mm._priced_at(forced, 32_768).total_bytes > mm._priced_at(sizing, 32_768).total_bytes
+ sizing = forced
+ steps = range(mm.MLX_KV_BLOCK, 262_145, mm.MLX_KV_BLOCK)
+ totals = [mm._priced_at(sizing, n).total_bytes for n in steps]
+ assert totals == sorted(totals)
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 10eb382295b..f543b369e19 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -4807,12 +4807,13 @@ def test_the_load_policy_bounds_a_pin_only_where_the_bound_can_be_enforced(monke
kv_bits,
requested,
served = unset,
+ fitted = False,
):
backend = MLXInferenceBackend()
backend._model = model
backend._is_vlm = False
quant, window, enforced, budget = backend._resolve_kv_policy(
- False, kv_bits, requested, requested if served is unset else served
+ False, kv_bits, requested, requested if served is unset else served, fitted = fitted
)
return quant["kv_bits"], window, enforced, budget
@@ -4821,6 +4822,11 @@ def test_the_load_policy_bounds_a_pin_only_where_the_bound_can_be_enforced(monke
assert policy(honours, None, 8192) == (None, 8192, True, None)
assert policy(honours, None, 0, 262144) == (None, 262144, True, None) # nothing to yield to
+ # A window fitted to the machine is held like a pin: a budget under a width, else a bound.
+ assert policy(honours, 4, 0, 24576, fitted = True) == (4, None, False, 24576)
+ assert policy(honours, None, 0, 24576, fitted = True) == (None, 24576, True, None)
+ assert policy(owns, 4, 0, 24576, fitted = True) == (4, None, False, None)
+
assert policy(owns, 4, 8192) == (4, None, False, None)
assert policy(owns, None, 8192) == (None, None, False, None)
@@ -5383,6 +5389,366 @@ def test_the_mlx_mcp_snapshot_is_taken_under_the_same_guard_the_gguf_count_uses(
assert guard < snapshot, "the guard must be held across the snapshot, not after it"
[email protected](autouse = True)
+def _fresh_mlx_estimate_caches():
+ # Tests ask one path under different patched towers.
+ from core.inference import mlx_inference, mlx_memory
+
+ mlx_inference._kv_refusal_cache.clear()
+ mlx_memory._clear_sizing_cache()
+ yield
+ mlx_inference._kv_refusal_cache.clear()
+ mlx_memory._clear_sizing_cache()
+
+
+def test_a_width_refusal_is_asked_once_per_checkpoint_and_width(monkeypatch, tmp_path):
+ from core.inference import mlx_inference
+
+ (tmp_path / "config.json").write_text("{}")
+ asked = []
+ monkeypatch.setattr(
+ mlx_inference, "_asking_the_architecture", lambda d, q: asked.append(d) or True
+ )
+ assert mlx_inference.mlx_kv_quant_is_refused(str(tmp_path), 4) is True
+ assert mlx_inference.mlx_kv_quant_is_refused(str(tmp_path), 4) is True and len(asked) == 1
+ assert mlx_inference.mlx_kv_quant_is_refused(str(tmp_path), 8) is True and len(asked) == 2
+ (tmp_path / "config.json").write_text('{"changed": 1}')
+ assert mlx_inference.mlx_kv_quant_is_refused(str(tmp_path), 4) is True and len(asked) == 3
+
+
+def test_what_the_fit_is_asked_and_when_it_is_asked_at_all(monkeypatch, tmp_path):
+ from core.inference import mlx_inference
+
+ priced, rewound = [], []
+ monkeypatch.setitem(
+ sys.modules,
+ "core.inference.mlx_memory",
+ types.SimpleNamespace(
+ mlx_fit_context = lambda model_dir, **kw: (
+ priced.append(kw | {"dir": model_dir})
+ or rewound.append(f"priced@{kw['kv_bits']}")
+ or 24_576
+ )
+ ),
+ )
+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: 8 * 1024**3)
+ monkeypatch.setattr(
+ mlx_inference, "_mlx_rng_key_words", lambda: rewound.append("held") or ("key",)
+ )
+ monkeypatch.setattr(mlx_inference, "_restore_mlx_rng_key", rewound.append)
+ asked = []
+ verdict = {"answer": ("full", "", True)}
+ monkeypatch.setattr(
+ mlx_inference,
+ "_kv_quant_eligibility",
+ lambda *a: asked.append(a) or verdict["answer"],
+ )
+ model = object()
+
+ def fit(kv_bits = None, **kw):
+ priced.clear(), asked.clear(), rewound.clear()
+ return mlx_inference._fitted_context(
+ model,
+ str(tmp_path),
+ 262_144,
+ load_in_4bit = True,
+ retains_history = True,
+ kv_bits = kv_bits,
+ **kw,
+ )
+
+ assert fit() == (24_576, None)
+ assert asked == [] and rewound == ["held", "priced@None", ("key",)]
+ assert priced == [
+ {
+ "budget_bytes": 8 * 1024**3,
+ "max_ctx": 262_144,
+ "kv_bits": None,
+ "load_in_4bit": True,
+ "dir": str(tmp_path),
+ }
+ ]
+ # The width the cache will take is the width the fit is priced at.
+ for answer, bits in ((("full", "", True), 4), (("partial", "w", True), 4)):
+ verdict["answer"] = answer
+ assert fit(4, is_vlm = True) == (24_576, answer)
+ assert asked == [(model, True, 4)] and priced[0]["kv_bits"] == bits
+ for answer in (("refused", "r", True), ("none", "n", True)):
+ verdict["answer"] = answer
+ assert fit(4) == (24_576, answer) and priced[0]["kv_bits"] is None
+
+ monkeypatch.setitem(
+ sys.modules,
+ "core.inference.mlx_memory",
+ types.SimpleNamespace(
+ mlx_fit_context = lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("no"))
+ ),
+ )
+ assert fit() == (None, None) and rewound == ["held", ("key",)]
+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: None)
+ assert fit() == (None, None) and rewound == []
+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: 8 * 1024**3)
+ monkeypatch.setattr(mlx_inference, "_snapshot_dir", lambda model, name: None)
+ assert fit() == (None, None) and (priced, rewound) == ([], [])
+
+
+def test_the_fit_budget_leaves_the_prompt_history_its_own_room(monkeypatch):
+ from core.inference import mlx_inference
+
+ fake = types.SimpleNamespace(
+ metal = types.SimpleNamespace(is_available = lambda: True),
+ device_info = lambda: {"max_recommended_working_set_size": 100_000_000_000},
+ )
+ monkeypatch.setitem(sys.modules, "mlx.core", fake)
+ if "mlx" in sys.modules:
+ monkeypatch.setattr(sys.modules["mlx"], "core", fake, raising = False)
+ else:
+ # The parent package is imported first, so it must resolve on hosts without MLX.
+ monkeypatch.setitem(sys.modules, "mlx", types.SimpleNamespace(core = fake))
+ monkeypatch.delenv("UNSLOTH_MLX_PROMPT_CACHE_BYTES", raising = False)
+
+ assert mlx_inference.mlx_memory_budget() == 85_000_000_000 - 15_000_000_000
+ assert mlx_inference.mlx_memory_budget(retains_history = False) == 85_000_000_000
+ fake.metal.is_available = lambda: False
+ assert mlx_inference.mlx_memory_budget() is None
+
+
+def test_the_fit_asks_about_the_repo_the_loader_actually_opens(monkeypatch, tmp_path):
+ import utils.utils as backend_utils
+ from core.inference import mlx_inference
+
+ asked = []
+ monkeypatch.setattr(
+ backend_utils, "hf_cache_snapshot_dir", lambda name: asked.append(name) or tmp_path
+ )
+ assert mlx_inference._snapshot_dir(SimpleNamespace(_src_path = tmp_path), "org/alias") == str(
+ tmp_path
+ )
+ assert asked == []
+ assert mlx_inference._snapshot_dir(SimpleNamespace(), "org/alias") == str(tmp_path)
+ assert asked == ["org/alias"]
+
+
+def test_only_a_load_that_asked_for_nothing_is_fitted_to_the_machine(monkeypatch):
+ """A pin is the user's own answer about memory and is served verbatim."""
+ _install_fake_mlx(monkeypatch)
+ _install_fake_fast_mlx(monkeypatch, [])
+
+ from core.inference import mlx_inference
+
+ FULL = ("full", "", True)
+ asked, verdicts, probes = [], [], []
+ monkeypatch.setattr(
+ mlx_inference, "_kv_quant_eligibility", lambda *a: verdicts.append(a) or FULL
+ )
+
+ def load(
+ fitted = (24_576, FULL),
+ enforceable = bool,
+ model = "fake/text",
+ **kwargs,
+ ):
+ del asked[:], verdicts[:], probes[:]
+ monkeypatch.setattr(
+ mlx_inference,
+ "_fitted_context",
+ lambda _m, name, ceiling, **kw: asked.append((name, ceiling, kw)) or fitted,
+ )
+ monkeypatch.setattr(
+ mlx_inference.MLXInferenceBackend,
+ "_kv_cache_window_enforceable",
+ lambda _s, served: probes.append(served) or enforceable(served),
+ )
+ backend = mlx_inference.MLXInferenceBackend()
+ assert backend.load_model(
+ SimpleNamespace(
+ identifier = model,
+ is_vision = kwargs.pop("vision", False),
+ is_lora = kwargs.pop("lora", False),
+ ),
+ **kwargs,
+ )
+ return backend, backend.models[model]
+
+ backend, pinned = load(max_seq_length = 4096, kv_quant = "4")
+ assert (asked, verdicts) == ([], [(backend._model, False, 4)])
+ assert (pinned["context_length"], pinned["context_length_fitted"]) == (4096, None)
+ assert (pinned["mlx_kv_bits"], pinned["mlx_context_budget"]) == (4, 4096)
+ # A fitted window is held like a pin: the width takes the cache, the window a budget.
+ backend, info = load(max_seq_length = 0, kv_quant = "4")
+ assert (verdicts, probes) == ([], [24_576])
+ assert asked == [
+ ("fake/text", None, dict(load_in_4bit = True, retains_history = True, kv_bits = 4, is_vlm = False))
+ ]
+ assert (info["context_length"], info["context_length_fitted"]) == (24_576, 24_576)
+ assert (info["mlx_kv_bits"], info["mlx_context_budget"]) == (4, 24_576)
+ assert (info["context_length_enforced"], backend._kv_cache_window) == (False, None)
+
+ backend, info = load((24_576, None), max_seq_length = 0)
+ assert (info["mlx_kv_bits"], info["mlx_context_budget"]) == (None, None)
+ assert (info["context_length_enforced"], backend._kv_cache_window) == (True, 24_576)
+ refused = ("refused", "r", True)
+ backend, info = load((24_576, refused), max_seq_length = 0, kv_quant = "4")
+ assert (info["mlx_kv_bits"], backend._kv_cache_window, verdicts) == (None, 24_576, [])
+ _, info = load(enforceable = lambda _s: False, max_seq_length = 0, kv_quant = "4")
+ assert (info["mlx_kv_bits"], info["mlx_context_budget"], info["context_length"]) == (
+ 4,
+ None,
+ 24_576,
+ )
+ backend, info = load(enforceable = lambda _s: False, max_seq_length = 4096, kv_quant = "4")
+ assert (info["mlx_kv_bits"], verdicts) == (4, [(backend._model, False, 4)])
+
+ # TurboQuant's layout is not one the planner prices: fitted at full width, no mx.quantize probe.
+ monkeypatch.setitem(
+ sys.modules, "mlx_vlm.turboquant", SimpleNamespace(TurboQuantKVCache = object)
+ )
+ monkeypatch.setattr(
+ mlx_inference,
+ "_turboquant_status",
+ lambda _m, bits, refusal = "": dict(
+ requested_kv_bits = bits, kv_bits = bits, eligibility = "full", reason = "", note = ""
+ ),
+ )
+ load(max_seq_length = 0, kv_quant = "tq-4")
+ assert verdicts == [] and [kw["kv_bits"] for _, _, kw in asked] == [None]
+
+ for available, reserved in ((True, True), (False, False)):
+ monkeypatch.setattr(
+ mlx_inference, "mlx_vlm_snapshot_store_available", lambda a = available: a
+ )
+ load(model = "fake/vlm", vision = True, max_seq_length = 0)
+ assert asked == [
+ (
+ "fake/vlm",
+ None,
+ dict(
+ load_in_4bit = True,
+ retains_history = reserved,
+ kv_bits = None,
+ is_vlm = True,
+ ),
+ )
+ ]
+
+ for kwargs in (
+ dict(
+ parallel_mode = "tensor",
+ distributed_group = SimpleNamespace(rank = lambda: 0, size = lambda: 2),
+ ),
+ dict(dtype = "float32"),
+ dict(lora = True),
+ ):
+ load(max_seq_length = 0, **kwargs)
+ assert not asked
+
+
+def test_the_width_probe_judges_the_tower_the_sizing_would_price(monkeypatch):
+ """The load's eligibility rule, asked of the first tower that runs, and never evaluated."""
+ from core.inference import mlx_inference
+
+ fake_mx = types.SimpleNamespace(
+ array = lambda value: value,
+ zeros = lambda *_a, **_k: None,
+ int32 = None,
+ eval = lambda *_a: pytest.fail("a weightless tower must never be evaluated"),
+ )
+ monkeypatch.setitem(sys.modules, "mlx.core", fake_mx)
+ if "mlx" in sys.modules:
+ monkeypatch.setattr(sys.modules["mlx"], "core", fake_mx, raising = False)
+ else:
+ monkeypatch.setitem(sys.modules, "mlx", types.SimpleNamespace(core = fake_mx))
+
+ class Quantized:
+ keys, state, nbytes = "k", (), 1
+
+ class Entry:
+ keys, state, window_size = "k", (), None
+
+ def __init__(
+ self,
+ max_size = None,
+ converts = True,
+ ):
+ self.max_size = max_size
+ if converts:
+ self.to_quantized = lambda **_kw: Quantized()
+
+ class Tower:
+ def __init__(
+ self,
+ *entries,
+ runs = True,
+ attends_quantized = True,
+ ):
+ self.entries, self.runs, self.attends_quantized = entries, runs, attends_quantized
+
+ def __call__(
+ self,
+ *_a,
+ cache = None,
+ **_k,
+ ):
+ if not self.runs:
+ raise ValueError("this tower cannot run")
+ if not self.attends_quantized and any(isinstance(e, Quantized) for e in cache):
+ raise TypeError("shared KV takes arrays only")
+
+ rewound = []
+
+ def refused(*towers):
+ monkeypatch.setitem(
+ sys.modules,
+ "core.inference.mlx_memory",
+ types.SimpleNamespace(
+ _PROBE_SHORT = 8,
+ _runtime_dtype = lambda: None,
+ _snapshot_config = lambda _d: {"model_type": "x"},
+ _probe_models = lambda *_a: iter(
+ [(lambda t = t: t, lambda tower: list(tower.entries), None) for t in towers]
+ ),
+ ),
+ )
+ marks = len(rewound)
+ mlx_inference._kv_refusal_cache.clear()
+ answer = mlx_inference.mlx_kv_quant_is_refused("/d", 4)
+ assert rewound[marks] == "held" and rewound[-1] == ("key",)
+ return answer
+
+ monkeypatch.setattr(
+ mlx_inference, "_mlx_rng_key_words", lambda: rewound.append("held") or ("key",)
+ )
+ monkeypatch.setattr(mlx_inference, "_restore_mlx_rng_key", rewound.append)
+
+ assert refused(Tower(Entry())) is False
+ # Sliding-window layers keep their ring and the rest convert: partial, still quantized.
+ assert refused(Tower(Entry(), Entry(max_size = 512))) is False
+ assert refused(Tower(Entry(max_size = 512))) is True
+ assert refused(Tower(Entry(converts = False))) is True
+ assert refused(Tower()) is True
+ # Gemma 4 E2B / E4B: the conversion succeeds, attending over it does not.
+ assert refused(Tower(Entry(), Entry(max_size = 512), attends_quantized = False)) is True
+ broken = Entry()
+ broken.to_quantized = lambda **_kw: (_ for _ in ()).throw(ValueError("group size"))
+ assert refused(Tower(broken)) is True
+ assert refused(Tower(Entry(max_size = 512), runs = False), Tower(Entry())) is False
+ assert refused(Tower(Entry(), runs = False), Tower(Entry(max_size = 512))) is True
+ # Nothing that runs to ask: priced as requested, as the load would try it.
+ assert refused(Tower(Entry(max_size = 512), runs = False)) is False
+
+ monkeypatch.setitem(
+ sys.modules,
+ "core.inference.mlx_memory",
+ types.SimpleNamespace(
+ _snapshot_config = lambda _d: (_ for _ in ()).throw(RuntimeError("no")),
+ ),
+ )
+ marks = len(rewound)
+ mlx_inference._kv_refusal_cache.clear()
+ assert mlx_inference.mlx_kv_quant_is_refused("/d", 4) is False
+ assert rewound[marks:] == ["held", ("key",)]
+
+
class _TwoRowStream:
"""A vision batch reporting two rows, recording whatever is taken back."""
diff --git a/studio/backend/tests/test_non_gguf_reload_settings.py b/studio/backend/tests/test_non_gguf_reload_settings.py
index ee6eb41433c..789e5e87b0f 100644
--- a/studio/backend/tests/test_non_gguf_reload_settings.py
+++ b/studio/backend/tests/test_non_gguf_reload_settings.py
@@ -194,6 +194,43 @@ class TestNonGgufStatusReportsWhatTheLoadAskedFor:
assert response.load_in_4bit is False
assert response.requested_gpu_ids == [0, 1]
+ def test_a_window_fitted_to_the_machine_reaches_the_client(self, monkeypatch):
+ """Only the worker measured it, and nothing outside can recompute it."""
+ response = self._status_for(monkeypatch, {"context_length_fitted": 14336})
+
+ assert response.context_length_fitted == 14336
+
+ def test_every_projection_that_reports_a_bound_reports_how_it_was_chosen(self):
+ """Four hand-maintained projections, and a drop from any one of them is silent."""
+ lines = (_BACKEND / "routes" / "inference.py").read_text(encoding = "utf-8").split("\n")
+ orphaned = [
+ index + 1
+ for index, line in enumerate(lines)
+ if "context_length_enforced" in line
+ and not any(
+ "context_length_fitted" in lines[near]
+ for near in range(max(0, index - 6), min(len(lines), index + 7))
+ )
+ ]
+
+ assert orphaned == [], f"context_length_enforced without a fitted window at {orphaned}"
+
+ def test_the_parent_mirrors_every_context_field_the_worker_sends(self):
+ """The hop the route test cannot see: a field the mirror drops is one no API reports."""
+ from core.inference.orchestrator import _mirrored_model_entry
+
+ sent = {
+ "context_length": 14336,
+ "native_context_length": 40960,
+ "max_context_length": 40960,
+ "requested_context_length": 0,
+ "context_length_enforced": True,
+ "context_length_fitted": 14336,
+ }
+ mirrored = _mirrored_model_entry(dict(sent, display_name = "m"), "m")
+
+ assert {name: mirrored.get(name) for name in sent} == sent
+
def test_the_mlx_mirror_wins_over_the_stamped_spelling(self, monkeypatch):
"""#8125: the MLX worker mirrors the real context back as requested_context_length."""
response = self._status_for(
diff --git a/studio/frontend/src/features/model-picker/api/memory-estimate.ts b/studio/frontend/src/features/model-picker/api/memory-estimate.ts
index 588d89af955..163de0dd26b 100644
--- a/studio/frontend/src/features/model-picker/api/memory-estimate.ts
+++ b/studio/frontend/src/features/model-picker/api/memory-estimate.ts
@@ -18,7 +18,6 @@ export type MemoryEstimateReason =
export interface MemoryEstimate {
available: boolean;
reason: MemoryEstimateReason | null;
- /** Files that become resident: weights, projector, drafter. */
weightsBytes: number;
/** KV cache at the priced context and slot count. Meaningless unless `kvEstimable`. */
kvBytes: number;
@@ -50,6 +49,7 @@ export interface MemoryEstimate {
kvOnGpu: boolean;
/** What was actually priced, after overrides and clamps resolve. */
nCtx: number;
+ contextFitted: number | null;
cacheTypeKv: string | null;
nParallel: number;
layerCount: number | null;
@@ -67,6 +67,8 @@ export interface MemoryEstimateRequest {
nativePathToken?: string | null;
nCtx?: number | null;
cacheTypeKv?: string | null;
+ maxSeqLength?: number | null;
+ mlxKvQuant?: string | null;
nParallel?: number | null;
nBatch?: number | null;
nUbatch?: number | null;
@@ -100,6 +102,7 @@ const UNAVAILABLE: MemoryEstimate = {
kvEstimable: false,
kvOnGpu: true,
nCtx: 0,
+ contextFitted: null,
cacheTypeKv: null,
nParallel: 1,
layerCount: null,
@@ -124,6 +127,7 @@ interface ApiEstimateResponse {
kv_estimable: boolean;
kv_on_gpu: boolean;
n_ctx: number;
+ context_fitted?: number | null;
cache_type_kv: string | null;
n_parallel: number;
layer_count: number | null;
@@ -142,6 +146,8 @@ function estimateRequestBody(
native_path_lease: nativePathLease,
n_ctx: payload.nCtx ?? null,
cache_type_kv: payload.cacheTypeKv ?? null,
+ max_seq_length: payload.maxSeqLength ?? null,
+ mlx_kv_quant: payload.mlxKvQuant ?? null,
n_parallel: payload.nParallel ?? null,
n_batch: payload.nBatch ?? null,
n_ubatch: payload.nUbatch ?? null,
@@ -235,6 +241,7 @@ function toMemoryEstimate(body: ApiEstimateResponse): MemoryEstimate {
kvEstimable: flag(body.kv_estimable, false),
kvOnGpu: flag(body.kv_on_gpu, true),
nCtx: finiteCount(body.n_ctx, 0),
+ contextFitted: nullableCount(body.context_fitted),
cacheTypeKv:
typeof body.cache_type_kv === "string" ? body.cache_type_kv : null,
nParallel: finiteCount(body.n_parallel, 1),
@@ -266,10 +273,6 @@ export function resetMemoryEstimateRouteMemo(): void {
routeAbsentAt = null;
}
-/** Price a prospective GGUF load from its header. Allocates nothing, loads nothing. Never
- * throws for a backend answer: an absent route, an auth expiry, a 500, an HTML error page
- * served as 200 or a truncated body all come back as an unavailable estimate, so the panel
- * hides the row. The statuses are told apart only to decide whether the miss is memoable. */
export async function fetchMemoryEstimate(
payload: MemoryEstimateRequest,
signal?: AbortSignal,
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index 3c8b56d3ca1..72058ee3f2c 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -81,7 +81,13 @@ import {
loadManagedLlamaFlags,
subscribeLlamaFlagCatalog,
} from "../api/llama-flags";
-import { resolveEstimateContext } from "../model-config/estimate-context";
+import { type MemoryEstimate } from "../api/memory-estimate";
+import {
+ resolveEstimateContext,
+ resolveMlxEstimateContext,
+ resolveMlxServedWindow,
+ shouldRequestMemoryEstimate,
+} from "../model-config/estimate-context";
import { resolveReclaimableMemoryCredit } from "../model-config/memory-fit";
import {
resolveResidentEstimateRequest,
@@ -427,6 +433,7 @@ function MaxSeqLengthSetting({
inputRef,
isMlx,
pinned,
+ fittedToMemory,
windowUnknown,
}: {
value: number;
@@ -436,6 +443,7 @@ function MaxSeqLengthSetting({
inputRef?: Ref<NumericValueInputHandle>;
isMlx?: boolean;
pinned?: boolean;
+ fittedToMemory?: boolean;
windowUnknown?: boolean;
}) {
// MLX sizes itself when unpinned, so the control is the GGUF path's Context Length and
@@ -448,7 +456,11 @@ function MaxSeqLengthSetting({
<span className={LABEL_CLASS}>{label}</span>
<InfoHint>
{isMlx
- ? "Tokens of context the model is sized for."
+ ? "Tokens of context the model is sized for." +
+ (fittedToMemory
+ ? " Fitted to this machine's memory, which is less than the model's own " +
+ "window. Set a length to ask for a different one."
+ : "")
: "Maximum context window in tokens. Applies on load."}
</InfoHint>
</div>
@@ -2015,6 +2027,7 @@ export function ModelConfigPage({
const loadedGpuIds = useChatRuntimeStore((s) => s.loadedGpuIds);
const loadedGpuIndexKind = useChatRuntimeStore((s) => s.loadedGpuIndexKind);
const loadedCpuFallback = useChatRuntimeStore((s) => s.loadedCpuFallback);
+ const residentModelLoading = useChatRuntimeStore((s) => s.modelLoading);
const residentEstimateSettings = useChatRuntimeStore(
useShallow(selectResidentEstimateSettings),
);
@@ -2741,18 +2754,6 @@ export function ModelConfigPage({
mlxNativeWindow == null
? null
: Math.min(mlxNativeWindow, MAX_SEQ_LENGTH_MAX);
- const mlxServedWindow =
- (targetIsMlx && isActiveModel ? servedWindow(loadedContextLength) : null) ??
- mlxProspectiveWindow;
- const maxSeqLengthValue =
- servedWindow(savedContextPin(config)) ??
- mlxServedWindow ??
- clampMaxSeqLength(DEFAULT_MAX_SEQ_LENGTH, nativeMaxSeqLength);
- // The slider picks a request, so it stops at the widest a load may make.
- const maxSeqLengthMax = Math.min(
- MAX_SEQ_LENGTH_MAX,
- Math.max(nativeMaxSeqLength, maxSeqLengthValue),
- );
// An auto-fit-below-native GGUF shows activeLoadedContext while
// customContextLength stays null. If the user fixes GPU Layers (Manual) and
// remembers, pin that shown context so a later fresh load keeps the fitted
@@ -2790,7 +2791,11 @@ export function ModelConfigPage({
// tri-state, not through resolvedIsDiffusion: a GGUF still being classified may be
// DiffusionGemma, and guessing paints a footprint from the wrong plan that never clears.
const memoryEstimateRequest =
- target.isGguf && classifiedIsDiffusion === false
+ shouldRequestMemoryEstimate({
+ isGguf: Boolean(target.isGguf),
+ isAppleUnifiedMemory,
+ classifiedIsDiffusion,
+ })
? {
modelPath: target.id,
ggufVariant: target.ggufVariant ?? null,
@@ -2809,6 +2814,10 @@ export function ModelConfigPage({
(target.isGguf === true && activePresetSource === "builtin-default"),
),
cacheTypeKv: runtimeConfig.kvCacheDtype,
+ maxSeqLength: target.isGguf
+ ? null
+ : resolveMlxEstimateContext(savedContextPin(config)),
+ mlxKvQuant: runtimeConfig.mlxKvQuant ?? null,
nParallel: runtimeConfig.nParallel,
nBatch: runtimeConfig.nBatch,
nUbatch: runtimeConfig.nUbatch,
@@ -2834,12 +2843,34 @@ export function ModelConfigPage({
}
: null;
const memoryEstimate = useMemoryEstimate(memoryEstimateRequest);
+ const mlxFittedWindow = targetIsMlx
+ ? servedWindow(memoryEstimate.estimate?.contextFitted)
+ : null;
+ const mlxServedWindow = resolveMlxServedWindow(
+ targetIsMlx && isActiveModel ? servedWindow(loadedContextLength) : null,
+ mlxFittedWindow,
+ mlxProspectiveWindow,
+ );
+ const maxSeqLengthValue =
+ servedWindow(savedContextPin(config)) ??
+ mlxServedWindow ??
+ clampMaxSeqLength(DEFAULT_MAX_SEQ_LENGTH, nativeMaxSeqLength);
+ const maxSeqLengthMax = Math.min(
+ MAX_SEQ_LENGTH_MAX,
+ Math.max(nativeMaxSeqLength, maxSeqLengthValue),
+ );
// No resident credit without a reported context; pending settings cannot price it.
- const residentContext = servedWindow(activeLoadedContext);
+ // activeLoadedContext is GGUF-only; an MLX resident reports its served window here.
+ const residentContext = servedWindow(
+ targetIsMlx && isActiveModel ? loadedContextLength : activeLoadedContext,
+ );
const residentEstimateRequest = resolveResidentEstimateRequest(
isActiveModel ? memoryEstimateRequest : null,
residentEstimateSettings,
residentContext,
+ targetIsMlx && !residentModelLoading
+ ? { kvQuant: loadedMlxKvQuantRequested ?? null }
+ : undefined,
);
const residentEstimate = useMemoryEstimate(residentEstimateRequest, {
refreshMemory: true,
@@ -3242,30 +3273,30 @@ export function ModelConfigPage({
)}
<div className="space-y-5">
+ {memoryEstimateRequest != null && (
+ <MemoryEstimateRow
+ estimate={memoryEstimate.estimate}
+ loading={memoryEstimate.loading}
+ stale={memoryEstimate.stale}
+ gpuCapacityGb={memoryGpuCapacityGb}
+ totalCapacityGb={memoryTotalCapacityGb}
+ systemRamCapacityGb={inferenceGpu.systemRamTotalGb}
+ freeGpuCapacityGb={memoryFreeGpuCapacityGb}
+ freeGpuCapacityKnown={memoryFreeGpuCapacityKnown}
+ freeGpuReserveDeficitGb={memoryFreeGpuReserveDeficitGb}
+ usableSystemRamGb={memoryUsableSystemRamGb}
+ usableSystemRamKnown={inferenceGpu.systemRamAvailableKnown}
+ systemRamReserveDeficitGb={memorySystemRamReserveDeficitGb}
+ isUnifiedMemory={isAppleUnifiedMemory}
+ singleMemoryPool={singleMemoryPool}
+ reclaimableTotalBytes={reclaimableCredit.totalBytes}
+ reclaimableGpuBytes={reclaimableCredit.gpuBytes}
+ expanded={memoryBreakdownOpen}
+ onExpandedChange={setMemoryBreakdownOpen}
+ />
+ )}
{target.isGguf && (
<>
- {/* Above Context Length on purpose: that is the control moving this number most, and a readout
- below it is one you go looking for. */}
- <MemoryEstimateRow
- estimate={memoryEstimate.estimate}
- loading={memoryEstimate.loading}
- stale={memoryEstimate.stale}
- gpuCapacityGb={memoryGpuCapacityGb}
- totalCapacityGb={memoryTotalCapacityGb}
- systemRamCapacityGb={inferenceGpu.systemRamTotalGb}
- freeGpuCapacityGb={memoryFreeGpuCapacityGb}
- freeGpuCapacityKnown={memoryFreeGpuCapacityKnown}
- freeGpuReserveDeficitGb={memoryFreeGpuReserveDeficitGb}
- usableSystemRamGb={memoryUsableSystemRamGb}
- usableSystemRamKnown={inferenceGpu.systemRamAvailableKnown}
- systemRamReserveDeficitGb={memorySystemRamReserveDeficitGb}
- isUnifiedMemory={isAppleUnifiedMemory}
- singleMemoryPool={singleMemoryPool}
- reclaimableTotalBytes={reclaimableCredit.totalBytes}
- reclaimableGpuBytes={reclaimableCredit.gpuBytes}
- expanded={memoryBreakdownOpen}
- onExpandedChange={setMemoryBreakdownOpen}
- />
<div className="space-y-2">
<div className={ROW_CLASS}>
<div className="flex min-w-0 items-center gap-1.5">
@@ -3383,6 +3414,9 @@ export function ModelConfigPage({
inputRef={maxSeqLengthInputRef}
isMlx={targetIsMlx}
pinned={savedContextPin(config) != null}
+ fittedToMemory={
+ savedContextPin(config) == null && mlxFittedWindow != null
+ }
windowUnknown={
savedContextPin(config) == null && mlxServedWindow == null
}
diff --git a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts
index 0d7e8486557..532dfb94f95 100644
--- a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts
+++ b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts
@@ -36,6 +36,8 @@ function estimateKey(request: MemoryEstimateRequest | null): string | null {
request.nativePathToken ?? null,
request.nCtx ?? null,
request.cacheTypeKv ?? null,
+ request.maxSeqLength ?? null,
+ request.mlxKvQuant ?? null,
request.nParallel ?? null,
request.nBatch ?? null,
request.nUbatch ?? null,
diff --git a/studio/frontend/src/features/model-picker/model-config/estimate-context.ts b/studio/frontend/src/features/model-picker/model-config/estimate-context.ts
index b91a180072d..c52d89e33f2 100644
--- a/studio/frontend/src/features/model-picker/model-config/estimate-context.ts
+++ b/studio/frontend/src/features/model-picker/model-config/estimate-context.ts
@@ -24,6 +24,18 @@ export function resolveEstimateContext(
return customContextLength ?? activeLoadedContext ?? 0;
}
+export function resolveMlxEstimateContext(contextPin: number | null): number {
+ return contextPin && contextPin > 0 ? contextPin : 0;
+}
+
+export function resolveMlxServedWindow(
+ loadedContext: number | null,
+ fittedContext: number | null,
+ nativeWindow: number | null,
+): number | null {
+ return fittedContext ?? loadedContext ?? nativeWindow;
+}
+
/** Which MODEL an estimate belongs to, for deciding whether shown numbers still do. Not the same
* question as "did anything change" -- that is the full request key, and a slider step must
* keep the old figures up and mark them stale. A change here means the numbers describe a
@@ -60,3 +72,13 @@ export function resolveEstimateSourceIdentity(
nativePathToken ?? null,
]);
}
+
+export function shouldRequestMemoryEstimate(opts: {
+ isGguf: boolean;
+ isAppleUnifiedMemory: boolean;
+ classifiedIsDiffusion: boolean | undefined;
+}): boolean {
+ const { isGguf, isAppleUnifiedMemory, classifiedIsDiffusion } = opts;
+ if (isGguf) return classifiedIsDiffusion === false;
+ return isAppleUnifiedMemory && classifiedIsDiffusion !== true;
+}
diff --git a/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts b/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts
index 5f66bd46d1d..45bcf8a1e01 100644
--- a/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts
+++ b/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts
@@ -65,15 +65,17 @@ export function selectResidentEstimateSettings(state: ResidentState) {
};
}
-/** Copy only source identity from the pending request. */
+/** Copy only source identity from the pending request. `mlx`: the resident is an MLX load,
+ * which reads its window from maxSeqLength and its cache width from mlxKvQuant, not nCtx. */
export function resolveResidentEstimateRequest(
source: MemoryEstimateRequest | null,
settings: ReturnType<typeof selectResidentEstimateSettings>,
context: number | null,
+ mlx?: { kvQuant: string | null },
): MemoryEstimateRequest | null {
if (
!source ||
- !settings ||
+ (!settings && !mlx) ||
context == null ||
!Number.isFinite(context) ||
context <= 0
@@ -84,7 +86,11 @@ export function resolveResidentEstimateRequest(
ggufVariant: source.ggufVariant,
hfToken: source.hfToken,
nativePathToken: source.nativePathToken,
- ...settings,
+ // An MLX load records none of the llama.cpp settings the selector requires.
+ ...(settings ?? {}),
nCtx: Math.floor(context),
+ ...(mlx
+ ? { maxSeqLength: Math.floor(context), mlxKvQuant: mlx.kvQuant }
+ : {}),
};
}
diff --git a/studio/frontend/tests/memory-estimate-context.test.ts b/studio/frontend/tests/memory-estimate-context.test.ts
index 9e9c55405bb..01484944808 100644
--- a/studio/frontend/tests/memory-estimate-context.test.ts
+++ b/studio/frontend/tests/memory-estimate-context.test.ts
@@ -1,9 +1,14 @@
import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { register } from "node:module";
import { test } from "node:test";
import {
resolveEstimateContext,
resolveEstimateSourceIdentity,
+ resolveMlxEstimateContext,
+ resolveMlxServedWindow,
+ shouldRequestMemoryEstimate,
} from "../src/features/model-picker/model-config/estimate-context.ts";
// The regression this file exists for: the Context Length control needs a number to
@@ -78,15 +83,24 @@ test("two repositories are different sources", () => {
});
test("two credentials are different sources: they resolve different files", () => {
- assert.notEqual(sourceId("org/gated", "Q4_K_M", "aaa"), sourceId("org/gated", "Q4_K_M", "bbb"));
+ assert.notEqual(
+ sourceId("org/gated", "Q4_K_M", "aaa"),
+ sourceId("org/gated", "Q4_K_M", "bbb"),
+ );
});
test("two native picks of the same file name are different sources", () => {
- assert.notEqual(sourceId("model.gguf", null, "", "tok-1"), sourceId("model.gguf", null, "", "tok-2"));
+ assert.notEqual(
+ sourceId("model.gguf", null, "", "tok-1"),
+ sourceId("model.gguf", null, "", "tok-2"),
+ );
});
test("absent and null are the same, so an unset variant does not thrash the row", () => {
- assert.equal(sourceId("org/a", null), sourceId("org/a", undefined as unknown as null));
+ assert.equal(
+ sourceId("org/a", null),
+ sourceId("org/a", undefined as unknown as null),
+ );
});
// Manual memory mode with GPU Layers on Auto hands context sizing to llama.cpp --fit.
@@ -119,3 +133,128 @@ test("every other shape keeps the resident fallback", () => {
test("a builtin-default GGUF load prices the fit, not the resident context", () => {
assert.equal(resolveEstimateContext(null, 131072, true), 0);
});
+
+const gguf = (classifiedIsDiffusion: boolean | undefined) =>
+ shouldRequestMemoryEstimate({
+ isGguf: true,
+ isAppleUnifiedMemory: true,
+ classifiedIsDiffusion,
+ });
+
+const safetensors = (
+ isAppleUnifiedMemory: boolean,
+ classifiedIsDiffusion: boolean | undefined,
+) =>
+ shouldRequestMemoryEstimate({
+ isGguf: false,
+ isAppleUnifiedMemory,
+ classifiedIsDiffusion,
+ });
+
+test("a GGUF is priced only once its probe has cleared it of being diffusion", () => {
+ assert.equal(gguf(false), true);
+ assert.equal(gguf(undefined), false);
+});
+
+test("an MLX load is priced without waiting for a GGUF-only probe", () => {
+ assert.equal(safetensors(true, undefined), true);
+ assert.equal(safetensors(true, false), true);
+});
+
+test("a model already known to be diffusion is priced by neither planner", () => {
+ assert.equal(gguf(true), false);
+ assert.equal(safetensors(true, true), false);
+});
+
+test("safetensors is not priced where MLX is not what would load it", () => {
+ assert.equal(safetensors(false, false), false);
+ assert.equal(safetensors(false, undefined), false);
+});
+
+register("./helpers/memory-estimate-resolver.mjs", import.meta.url);
+
+const auth = await import("./helpers/store-stubs/auth.ts");
+const native = await import("./helpers/native-path-stub.ts");
+const { fetchMemoryEstimate, resetMemoryEstimateRouteMemo } = await import(
+ "../src/features/model-picker/api/memory-estimate.ts"
+);
+
+let sent: Record<string, unknown> | null = null;
+
+test.beforeEach(() => {
+ resetMemoryEstimateRouteMemo();
+ native.setNativePathHandler(null);
+ sent = null;
+ auth.setAuthFetchHandler((_url, init) => {
+ sent = init?.body ? JSON.parse(String(init.body)) : null;
+ return new Response("{}", {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ });
+ });
+});
+
+test.after(() => {
+ resetMemoryEstimateRouteMemo();
+ auth.setAuthFetchHandler(null);
+ native.setNativePathHandler(null);
+});
+
+test("the non-GGUF context and MLX cache width reach the backend", async () => {
+ await fetchMemoryEstimate({
+ modelPath: "mlx-community/Qwen3-8B-4bit",
+ maxSeqLength: 4096,
+ mlxKvQuant: "tq-4",
+ });
+ assert.equal(sent!.max_seq_length, 4096);
+ assert.equal(sent!.mlx_kv_quant, "tq-4");
+});
+
+test("every field sent is also a field the hook re-fetches for", () => {
+ const read = (path: string, fn: string): Set<string> => {
+ const source = readFileSync(new URL(path, import.meta.url), "utf8");
+ const start = source.indexOf(`function ${fn}(`);
+ assert.ok(start >= 0, `${fn} not found in ${path}; this test is stale`);
+ const body = source.slice(start, source.indexOf("\n}\n", start));
+ return new Set(
+ [...body.matchAll(/\b(?:payload|request)\.([A-Za-z0-9_]+)/g)].map(
+ (m) => m[1],
+ ),
+ );
+ };
+ const wire = read(
+ "../src/features/model-picker/api/memory-estimate.ts",
+ "estimateRequestBody",
+ );
+ const key = read(
+ "../src/features/model-picker/hooks/use-memory-estimate.ts",
+ "estimateKey",
+ );
+ assert.ok(wire.has("maxSeqLength") && wire.has("mlxKvQuant"), "test is stale");
+ const unwatched = [...wire].filter((field) => !key.has(field));
+ assert.deepEqual(
+ unwatched,
+ [],
+ `sent to the backend but absent from estimateKey: ${unwatched.join(", ")}`,
+ );
+});
+
+test("what an MLX estimate names, and what the control shows", () => {
+ assert.equal(resolveMlxEstimateContext(null), 0);
+ assert.equal(resolveMlxEstimateContext(0), 0);
+ assert.equal(resolveMlxEstimateContext(8192), 8192);
+ assert.equal(resolveMlxServedWindow(8192, 24576, 262144), 24576);
+ assert.equal(resolveMlxServedWindow(null, 24576, 262144), 24576);
+ assert.equal(resolveMlxServedWindow(20480, null, 262144), 20480);
+ assert.equal(resolveMlxServedWindow(null, null, 262144), 262144);
+ assert.equal(resolveMlxServedWindow(null, null, null), null);
+});
+
+test("the fitted window survives the wire, and its absence reads as null", async () => {
+ const fitted = async (body: unknown) => {
+ auth.setAuthFetchHandler(() => Response.json(body));
+ return (await fetchMemoryEstimate({ modelPath: "a" })).contextFitted;
+ };
+ assert.equal(await fitted({ available: true, n_ctx: 24576, context_fitted: 24576 }), 24576);
+ assert.equal(await fitted({ available: true, n_ctx: 262144 }), null);
+});
diff --git a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts b/studio/frontend/tests/memory-estimate-narrow-panel.test.ts
index 6cc19f16a26..c974c686fe5 100644
--- a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts
+++ b/studio/frontend/tests/memory-estimate-narrow-panel.test.ts
@@ -48,6 +48,7 @@ const props: Props = {
drafterKvUnsized: false,
adaptersUnsized: false,
totalBytes: 5.5 * GIB,
+ contextFitted: null,
gpuBytes: 4.75 * GIB,
kvEstimable: true,
kvOnGpu: true,
diff --git a/studio/frontend/tests/resident-memory-request.test.ts b/studio/frontend/tests/resident-memory-request.test.ts
index bc33520d3b5..79a87e332bf 100644
--- a/studio/frontend/tests/resident-memory-request.test.ts
+++ b/studio/frontend/tests/resident-memory-request.test.ts
@@ -3,6 +3,7 @@
import assert from "node:assert/strict";
import test from "node:test";
+import { readFileSync } from "node:fs";
import {
resolveResidentEstimateRequest,
selectResidentEstimateSettings,
@@ -134,3 +135,46 @@ test("fixed Manual layers and automatic layers retain their loaded meaning", ()
assert.equal(settings?.nCpuMoe, 4);
}
});
+
+test("an MLX resident is priced at its served window and loaded cache width", () => {
+ const pending = {
+ modelPath: "mlx-community/Qwen3-8B-4bit",
+ ggufVariant: null,
+ };
+ const settings = selectResidentEstimateSettings({ ...loaded });
+ const request = resolveResidentEstimateRequest(pending, settings, 8192, {
+ kvQuant: "4",
+ });
+ assert.equal(request?.maxSeqLength, 8192);
+ assert.equal(request?.mlxKvQuant, "4");
+ const auto = resolveResidentEstimateRequest(pending, settings, 8192, {
+ kvQuant: null,
+ });
+ assert.equal(auto?.maxSeqLength, 8192);
+ assert.equal(auto?.mlxKvQuant, null);
+ // A GGUF resident keeps its old shape: nothing MLX reads is added.
+ const gguf = resolveResidentEstimateRequest(pending, settings, 8192);
+ assert.ok(gguf && !("maxSeqLength" in gguf) && !("mlxKvQuant" in gguf));
+});
+
+test("an MLX resident is credited without the llama.cpp settings it never records", () => {
+ const pending = { modelPath: "mlx-community/Qwen3-8B-4bit", ggufVariant: null };
+ const request = resolveResidentEstimateRequest(pending, null, 20736, {
+ kvQuant: null,
+ });
+ assert.equal(request?.maxSeqLength, 20736);
+ assert.equal(resolveResidentEstimateRequest(pending, null, 20736), null);
+ // The page reads an MLX resident's window from loadedContextLength: activeLoadedContext
+ // is GGUF-only, and reading it left the MLX branch unreachable.
+ const page = readFileSync(
+ new URL(
+ "../src/features/model-picker/components/model-config-page.tsx",
+ import.meta.url,
+ ),
+ "utf8",
+ );
+ assert.match(
+ page,
+ /targetIsMlx && isActiveModel \? loadedContextLength : activeLoadedContext/,
+ );
+});
diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py
index d3bb35fc03b..73bf24437b5 100644
--- a/tests/studio/test_model_picker_contracts.py
+++ b/tests/studio/test_model_picker_contracts.py
@@ -1318,8 +1318,8 @@ def test_an_mlx_target_is_offered_a_context_length_not_a_sequence_length():
# A number, not a word: the placeholder is only for a window nobody has read.
assert 'displayValue={isMlx && windowUnknown ? "—" : undefined}' in page
assert "savedContextPin(config) == null && mlxServedWindow == null\n" in page
- # The resident model's window, else this model's; request bounds would shorten it.
- assert "(targetIsMlx && isActiveModel ? servedWindow(loadedContextLength) : null) ??" in page
+ assert "const mlxServedWindow = resolveMlxServedWindow(" in page
+ assert "targetIsMlx && isActiveModel ? servedWindow(loadedContextLength) : null,\n" in page
assert "? servedWindow(modelMaxPosition.maxPositionEmbeddings)" in page
assert re.search(r"const servedWindow = [^;]*Math\.floor\(value\)\n\s*: null;", page), page
numeric = _read("features/model-picker/components/numeric-value-input.tsx")