Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: price MLX loads in the memory panel and fit an unpinned context to available memory (#10287)
Long Yixing98284907b802
diff --git a/studio/backend/core/inference/memory_contract.py b/studio/backend/core/inference/memory_contract.pyindex 1e9c6c04520..e567a8dfa49 100644--- a/studio/backend/core/inference/memory_contract.py+++ b/studio/backend/core/inference/memory_contract.py@@ -89,6 +89,7 @@ def build_memory_estimate(context_is_pinned: bool = True,inherited_device_pin: bool = False,spec_unpriced: bool = False,+ context_fitted: Optional[int] = None,moe_offload_unmodelled: bool = False,gpu_bytes: Any = _UNSET,compute_bytes: Any = _UNSET,@@ -163,6 +164,7 @@ def build_memory_estimate(else int(n_ctx or 0)),native_context = native_context,+ context_fitted = context_fitted,cache_type_kv = getattr(breakdown, "cache_type_kv", None),n_parallel = int(getattr(breakdown, "n_parallel", 1) or 1),layer_count = getattr(breakdown, "layer_count", None),@@ -198,6 +200,7 @@ def project_estimate_memory_response(estimate: MemoryEstimate) -> dict:"kv_estimable": estimate.kv_estimable,"kv_on_gpu": estimate.kv_on_gpu,"n_ctx": estimate.n_ctx,+ "context_fitted": estimate.context_fitted,"cache_type_kv": estimate.cache_type_kv,"n_parallel": estimate.n_parallel,"layer_count": estimate.layer_count,diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 63b2c828a6b..d52bcc0a81b 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -1595,11 +1595,81 @@ MLX_KV_BITS_CHOICES = (8, 6, 5, 4, 3, 2)MLX_TURBOQUANT_BITS_CHOICES = (4, 3.5, 3, 2)# Quantization group size; a head dim that is not a multiple makes mx.quantize raise.MLX_KV_GROUP_SIZE = 64+MLX_PREFILL_CHUNK_FALLBACK = 2048+++def _generation_step(*, vision: bool, drafted: bool):+ """The function that would run this generation's prefill."""+ if vision:+ from mlx_vlm.generate.ar import generate_step as vlm_generate_step+ return vlm_generate_step+ if drafted:+ from mlx_lm.generate import speculative_generate_step+ return speculative_generate_step+ from mlx_lm.generate import generate_step++ return generate_step+++def _generation_default(setting: str, fallback: int, *, vision: bool, drafted: bool) -> int:+ """A prefill setting as the runtime that would run generation defaults to it."""+ import inspect+ try:+ found = (+ inspect.signature(_generation_step(vision = vision, drafted = drafted))+ .parameters[setting]+ .default+ )+ if isinstance(found, bool) or not isinstance(found, int) or found <= 0:+ raise ValueError(f"the runtime states no usable {setting}")+ return found+ except Exception as exc:+ logger.debug("Cannot read the runtime's %s: %s", setting, exc)+ return fallback+++def mlx_vlm_snapshot_store_available() -> bool:+ """Whether a vision load can build the snapshot store, which decides two things about it."""+ if _prompt_cache_max_bytes() <= 0:+ return False+ try:+ from mlx_vlm.generate import GenerationResult+ except Exception as exc:+ logger.debug("Cannot tell whether mlx-vlm carries the snapshot grid: %s", exc)+ return False+ return hasattr(GenerationResult, "cached_tokens")+++def mlx_vlm_pins_the_prefill_step(config) -> bool:+ """Whether EVERY request to this vision load prefills on the pinned grid."""+ if not mlx_vlm_snapshot_store_available():+ return False+ return bool(MLXInferenceBackend._vlm_media_token_ids(config))+++def mlx_prefill_chunk(+ *,+ vision: bool = False,+ drafted: bool = False,+ config = None,+) -> int:+ """Most tokens one prefill step takes; the last step of a prompt may be shorter."""+ if vision and mlx_vlm_pins_the_prefill_step(config):+ return vlm_prefill_step()+ return _generation_default(+ "prefill_step_size", MLX_PREFILL_CHUNK_FALLBACK, vision = vision, drafted = drafted+ )++# Surfaced with the resolved setting so an API client sees the reuse cost too.MLX_KV_QUANT_NO_REUSE = ("The installed mlx-lm cannot measure a quantized cache entry, so prompt-cache ""reuse across turns is disabled while this is on.")+# The cap _configure_memory_limits installs, so a fitted context names a load MLX would allocate.+MLX_MEMORY_LIMIT_FRACTION = 0.85++MLX_TURBOQUANT_TEXT_LOAD = ("TurboQuant is not available for this model: it could not be loaded through the runtime ""TurboQuant requires. The model is served without it."@@ -1695,17 +1765,16 @@ def _mlx_rng_key_words():what lets the rewind below stay unconditional. A key that reads but is not two words is notthe same as an unreadable one: the rewind no longer works on the installed mlx, so say sobefore declining."""- import mlx.core as mx-try:+ import mlx.core as mxwords = mx.random.state[0].tolist()except Exception:return Noneif len(words) != 2:logger.warning("MLX exposes a %d-word random key; Unsloth can only rewind the "- "two-word form, so the KV quantization probe will not restore the "- "PRNG and sampling after a load may differ from an unprobed run.",+ "two-word form, so sizing a load will not restore the PRNG and "+ "sampling after a load may differ from an unsized run.",len(words),)return None@@ -1715,24 +1784,26 @@ def _mlx_rng_key_words():def _as_uint32_pair(high, low):"""Both words as uint32, or None if either is not a 32-bit word at all.- mx.random.seed takes a uint64 and raises outside [0, 2**64); that raise would land in the- probe's finally and replace the probe's own outcome. So the words are range-checked here rather- than passed through, which is what lets the rewind below stay unguarded.-- Range-checked and not simply masked, though. A negative reads as the two's complement of the- uint32 mlx stores, so reinterpreting it loses nothing. A value at or above 2**32 is not a 32-bit- word under any reading, and masking it would turn a key we cannot represent into a plausible- wrong one: (2**32, 0) would restore as (0, 0), the probe would report success, and sampling- would silently diverge from an unprobed run. Decline instead, which is the outcome the caller- already handles.+ mx.random.seed takes a uint64 and raises outside [0, 2**64); that raise would+ land in the rewind and replace the outcome of whatever it was guarding. So the+ words are range-checked here rather than passed through, which is what lets the+ rewind below stay unguarded.++ Range-checked and not simply masked, though. A negative reads as the two's+ complement of the uint32 mlx stores, so reinterpreting it loses nothing. A+ value at or above 2**32 is not a 32-bit word under any reading, and masking+ it would turn a key we cannot represent into a plausible wrong one: (2**32, 0)+ would restore as (0, 0), the rewind would report success, and sampling would+ silently diverge from an unsized run. Decline instead, which is the outcome+ the caller already handles."""converted = []for word in (high, low):if not -(2**31) <= word < 2**32:logger.warning("MLX exposed a random key word of %d, which is not a 32-bit "- "word; the KV quantization probe will not restore the PRNG and "- "sampling after a load may differ from an unprobed run.",+ "word; sizing a load will not restore the PRNG and sampling "+ "after a load may differ from an unsized run.",word,)return None@@ -1747,16 +1818,26 @@ def _restore_mlx_rng_key(words):unsigned 64-bit range. Unguarded on purpose: the range check in _as_uint32_pair removes theonly way this can raise, so there is no failure to swallow, and a blanket except here wouldbe a failure indistinguishable from an intentional no-op."""- import mlx.core as mx-if words is None:return+ import mlx.core as mx+pair = _as_uint32_pair(int(words[0]), int(words[1]))if pair is None:returnmx.random.seed((pair[0] << 32) | pair[1])+@contextmanager+def mlx_rng_preserved():+ """Leave the PRNG where it was found around weightless builds."""+ rng_key = _mlx_rng_key_words()+ try:+ yield+ finally:+ _restore_mlx_rng_key(rng_key)++def _kv_entry_windowed(entry):return any(getattr(entry, name, None) is not None for name in ("max_size", "window_size"))@@ -1812,11 +1893,19 @@ def _quantize_kv_entries(entries, bits):return entries-def _kv_quant_probe(language_model, entries, bits):+def _kv_quant_probe(+ language_model,+ entries,+ bits,+ evaluate = True,+):"""``(converted, skipped, failure, retainable)``: runs a token through the converted cache,- since static proxies (declared head_dim, window spelling) proved wrong both ways."""+ since static proxies (declared head_dim, window spelling) proved wrong both ways. Unevaluated,+ the same graph is only built, which is what a weightless tower affords."""import mlx.core as mx+ settle = mx.eval if evaluate else (lambda _arrays: None)+targets, held = _kv_quant_targets(entries)skipped = len(entries) - len(targets) - heldif not targets:@@ -1828,21 +1917,21 @@ def _kv_quant_probe(language_model, entries, bits):try:try:language_model(mx.array([[0]]), cache = entries)- mx.eval([getattr(entry, "state", None) for entry in entries])+ settle([getattr(entry, "state", None) for entry in entries])except Exception as exc:if is_metal_queue_dead(exc):raisereturn 0, 0, f"its cache could not be exercised ({type(exc).__name__})", Truetry:_quantize_kv_entries(entries, bits)- mx.eval([entries[index].state for index in targets])+ settle([entries[index].state for index in targets])except Exception as exc:if is_metal_queue_dead(exc):raisereturn 0, 0, f"MLX cannot quantize it ({type(exc).__name__})", Truetry:language_model(mx.array([[0]]), cache = entries)- mx.eval([getattr(entry, "state", None) for entry in entries])+ settle([getattr(entry, "state", None) for entry in entries])except Exception as exc:if is_metal_queue_dead(exc):raise@@ -2273,6 +2362,91 @@ def _kv_entry_is_bounded(entry, window):return cap > getattr(entry, "keep", 0)+def mlx_memory_budget(*, retains_history: bool = True) -> Optional[int]:+ """Bytes a load's own footprint may occupy here, or None where Metal cannot say."""+ import mlx.core as mx++ if not mx.metal.is_available():+ return None+ recommended = mx.device_info().get("max_recommended_working_set_size")+ if not recommended or recommended <= 0:+ return None+ budget = int(recommended * MLX_MEMORY_LIMIT_FRACTION)+ if retains_history:+ budget -= _prompt_cache_max_bytes(recommended / 1e9)+ return budget if budget > 0 else None+++def _snapshot_dir(model, model_name: str) -> Optional[str]:+ """Where the files this load actually opened are, or None if none are local."""+ resolved = getattr(model, "_src_path", None)+ if resolved and os.path.isdir(str(resolved)):+ return str(resolved)+ if os.path.isdir(model_name):+ return model_name+ from utils.utils import hf_cache_snapshot_dir++ snapshot = hf_cache_snapshot_dir(model_name)+ return str(snapshot) if snapshot is not None else None+++def mlx_fit_to_memory(+ model_dir,+ ceiling,+ *,+ load_in_4bit: bool,+ retains_history: bool,+ kv_bits = None,+) -> Optional[int]:+ """The context a load of *model_dir* is held to, priced at the KV width its cache takes."""+ budget = mlx_memory_budget(retains_history = retains_history)+ if budget is None or not ceiling or not model_dir:+ return None+ with mlx_rng_preserved():+ try:+ from core.inference.mlx_memory import mlx_fit_context++ fitted = mlx_fit_context(+ model_dir,+ budget_bytes = budget,+ max_ctx = int(ceiling),+ kv_bits = kv_bits,+ load_in_4bit = load_in_4bit,+ )+ logger.debug("MLX fit for %s: %s tokens under %.1f GB", model_dir, fitted, budget / 1e9)+ return fitted+ except Exception as exc:+ logger.debug("MLX context fit unavailable for %s: %s", model_dir, exc)+ return None+++def _fitted_context(+ model,+ model_name: str,+ ceiling,+ *,+ load_in_4bit: bool,+ retains_history: bool,+ kv_bits = None,+ is_vlm = False,+):+ """The fit for a resident model and the eligibility verdict it was priced under."""+ try:+ model_dir = _snapshot_dir(model, model_name)+ except Exception as exc:+ logger.debug("MLX snapshot unavailable for %s: %s", model_name, exc)+ return None, None+ verdict = None if kv_bits is None else _kv_quant_eligibility(model, is_vlm, kv_bits)+ fitted = mlx_fit_to_memory(+ model_dir,+ ceiling,+ load_in_4bit = load_in_4bit,+ retains_history = retains_history,+ kv_bits = kv_bits if verdict is None or verdict[0] in ("full", "partial") else None,+ )+ return fitted, verdict++def _kv_window_enforced(model, is_vlm, window):"""Whether a cache built at *window* is bounded in every layer, None where none could be builtto judge: both runtimes defer to a model's own make_cache and ignore max_kv_size there."""@@ -2286,6 +2460,69 @@ def _kv_window_enforced(model, is_vlm, window):return None+def _asking_the_architecture(model_dir, question):+ """Put *question* to the tower the sizing would price, built weightless from the checkpoint."""+ with mlx_rng_preserved():+ try:+ import mlx.core as mx++ from core.inference.mlx_memory import (+ _PROBE_SHORT,+ _probe_models,+ _runtime_dtype,+ _snapshot_config,+ )++ config = _snapshot_config(model_dir)+ if config is None:+ return None+ for build, make_prompt_cache, _ in _probe_models(config, _runtime_dtype()):+ model = build()+ try:+ model(+ mx.zeros((1, _PROBE_SHORT), dtype = mx.int32),+ cache = make_prompt_cache(model),+ )+ except Exception:+ continue+ return question(model, make_prompt_cache)+ except Exception as exc:+ logger.debug("MLX architecture probe unavailable for %s: %s", model_dir, exc)+ return None+++# Slider steps resend the width, and each answer is a weightless build.+_kv_refusal_cache: dict = {}+_kv_refusal_lock = threading.Lock()+++def mlx_kv_quant_is_refused(model_dir, bits) -> bool:+ """Whether a load would keep this checkpoint's cache at full width despite *bits*, by the+ eligibility probe's own rule (nothing to convert, or a converted cache the model cannot+ attend over)."""++ def refused(model, make_prompt_cache):+ converted, _skipped, failure, _retainable = _kv_quant_probe(+ model, make_prompt_cache(model), bits, evaluate = False+ )+ return failure is not None or not converted++ try:+ from core.inference.mlx_memory import _checkpoint_fingerprint+ key = (os.path.realpath(model_dir), bits, _checkpoint_fingerprint(model_dir))+ except Exception:+ return _asking_the_architecture(model_dir, refused) is True+ with _kv_refusal_lock:+ if key in _kv_refusal_cache:+ return _kv_refusal_cache[key]+ answer = _asking_the_architecture(model_dir, refused) is True+ with _kv_refusal_lock:+ _kv_refusal_cache[key] = answer+ while len(_kv_refusal_cache) > 32:+ _kv_refusal_cache.pop(next(iter(_kv_refusal_cache)))+ return answer++def _turboquant_reserved_positions(count, bits):"""Entry positions mlx-vlm reserves, learned from a control cache rather than a fixed number."""@@ -2377,7 +2614,12 @@ def _turboquant_status(mx.clear_cache()-def _kv_quant_status(requested_bits, model, is_vlm):+def _kv_quant_status(+ requested_bits,+ model,+ is_vlm,+ eligibility = None,+):"""Resolve a requested bit width against this model into a status dict."""status = {"requested_kv_bits": requested_bits,@@ -2388,7 +2630,9 @@ def _kv_quant_status(requested_bits, model, is_vlm):}if requested_bits is None:return status- verdict, reason, retainable = _kv_quant_eligibility(model, is_vlm, requested_bits)+ verdict, reason, retainable = eligibility or _kv_quant_eligibility(+ model, is_vlm, requested_bits+ )status["eligibility"] = verdictstatus["reason"] = reasonif verdict in ("full", "partial"):@@ -3941,7 +4185,7 @@ class MLXInferenceBackend:if not rec_bytes or rec_bytes <= 0:returnrec_gb = rec_bytes / 1e9- memory_limit_gb = rec_gb * 0.85+ memory_limit_gb = rec_gb * MLX_MEMORY_LIMIT_FRACTIONwired_limit_gb = min(rec_gb, memory_limit_gb)mx.set_memory_limit(int(memory_limit_gb * 1e9))mx.set_wired_limit(int(wired_limit_gb * 1e9))@@ -3997,11 +4241,20 @@ class MLXInferenceBackend:)return enforced- def _resolve_kv_policy(self, is_vlm, kv_bits, max_seq_length, served):+ def _resolve_kv_policy(+ self,+ is_vlm,+ kv_bits,+ max_seq_length,+ served,+ fitted = False,+ eligibility = None,+ ):"""The quantization status, cache window and per-request budget this load will run with.- A rotating cache cannot be quantized, so with kv_bits an enforceable pin becomes a budget."""- pinned = _positive_int(max_seq_length) is not None+ A rotating cache cannot be quantized, so with kv_bits an enforceable pin or fitted window+ becomes a budget."""+ pinned = _positive_int(max_seq_length) is not None or fitted# Tri-state: True bounded, False confirmed unbounded, None unjudgeable. Only True installs a bound; the other# two stay apart so a client can tell them apart.confirmed = self._kv_cache_window_enforceable(served)@@ -4011,7 +4264,12 @@ class MLXInferenceBackend:self._model, kv_bits, getattr(self, "_turboquant_refusal", ""))else:- quant = _kv_quant_status(_normalize_mlx_kv_bits(kv_bits), self._model, is_vlm)+ quant = _kv_quant_status(+ _normalize_mlx_kv_bits(kv_bits),+ self._model,+ is_vlm,+ eligibility,+ )if quant["kv_bits"] is None and enforceable:logger.info("MLX KV cache limited to %d tokens", int(served))return quant, int(served), True, None@@ -4180,6 +4438,26 @@ class MLXInferenceBackend:_served_ctx, _native_ctx, _max_ctx = self._resolve_context_lengths(self._model, max_seq_length)+ _priceable = not (is_distributed or is_lora or dtype is not None)+ # TurboQuant's layout is not one the planner prices, and a refused one stays full width:+ # fit either at full width, which only ever under-promises.+ _requested_bits = None if self._turboquant else _normalize_mlx_kv_bits(kv_bits)+ _fitted_ctx, _eligibility = (+ (None, None)+ if not _priceable or _positive_int(max_seq_length) is not None+ else _fitted_context(+ self._model,+ model_name,+ _served_ctx,+ load_in_4bit = load_in_4bit,+ retains_history = not use_vlm or mlx_vlm_snapshot_store_available(),+ kv_bits = _requested_bits,+ is_vlm = use_vlm,+ )+ )+ if _fitted_ctx:+ logger.info("MLX context fitted to %d tokens from %d", _fitted_ctx, _served_ctx or 0)+ _served_ctx = _fitted_ctxself._served_context = _served_ctx# Classified now so a cache the model cannot attend over is refused here, not on a token.(@@ -4187,7 +4465,14 @@ class MLXInferenceBackend:self._kv_cache_window,_ctx_enforced,self._kv_context_budget,- ) = self._resolve_kv_policy(use_vlm, kv_bits, max_seq_length, _served_ctx)+ ) = self._resolve_kv_policy(+ use_vlm,+ kv_bits,+ max_seq_length,+ _served_ctx,+ fitted = _fitted_ctx is not None,+ eligibility = _eligibility,+ )if self._kv_quant["kv_bits"] is not None:logger.info("MLX KV cache quantization: %s-bit (%s eligibility)",@@ -4260,13 +4545,13 @@ class MLXInferenceBackend:# Parity with llama.cpp's requested_n_ctx: the served window cannot say whether anything was asked for,# and that decides reuse and "pinned"."requested_context_length": _positive_int(max_seq_length) or 0,- # Nothing measures the machine, so window and ceiling coincide."native_context_length": _native_ctx,"max_context_length": _max_ctx,# Whether the served window actually bounds the cache: True confirmed on a real cache, False confirmed# unbounded, None nothing could be built to judge. Without it the API reports a limit a client cannot tell# from an enforced one."context_length_enforced": _ctx_enforced,+ "context_length_fitted": _fitted_ctx,"context_unbounded_when_batched": self._vision_batch_is_available(),"mlx_context_budget": self._kv_context_budget,"mlx_kv_bits": self._kv_quant["kv_bits"],diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.pynew file mode 100644index 00000000000..35665079252--- /dev/null+++ b/studio/backend/core/inference/mlx_memory.py@@ -0,0 +1,1165 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""What an MLX load would occupy, priced before anything is loaded."""++from __future__ import annotations++import glob+import functools+import json+import os+import posixpath+import threading+from collections import OrderedDict+from dataclasses import dataclass+from typing import Optional++from loggers import get_logger++logger = get_logger(__name__)++__all__ = [+ "MLX_FIT_MIN_CONTEXT",+ "MLX_KV_BLOCK",+ "MLX_PREFILL_CHUNK",+ "MlxMemoryBreakdown",+ "mlx_fit_context",+ "mlx_memory_breakdown",+ "mlx_shard_files",+ "mlx_weight_bytes",+]++MLX_KV_BLOCK = 256++_PROBE_SHORT = 8+_PROBE_LONG = 40++MLX_PREFILL_CHUNK = 2048++MLX_FIT_MIN_CONTEXT = 4096+_KV_GROUP_SIZE = 64++_ATTENTION_LIVE = 3.5+_COMPUTE_BASE_BYTES = int(0.64 * 1024**3)++_RECURRENT_DTYPE_SIZE = 4++_QUANT_SCORE_DTYPE_SIZE = 4++_QUANT_SCORE_LIVE = 1.0++_WIDTH_NAMES = {"float64": "f64", "float32": "f32", "bfloat16": "bf16", "float16": "f16"}++_CACHE_BOUND_ATTRS = ("max_size", "chunk_size")++_MAX_SAFETENSORS_HEADER = 100_000_000+++@dataclass+class MlxMemoryBreakdown:+ weights_bytes: int+ kv_bytes: int+ compute_bytes: int+ total_bytes: int+ gpu_bytes: int+ n_ctx: int+ layer_count: Optional[int] = None+ cache_type_kv: Optional[str] = None+ kv_estimable: bool = True+ kv_on_gpu: bool = True+ n_parallel: int = 1+ gpu_layers: Optional[int] = None+++def _snapshot_config(model_dir: str) -> Optional[dict]:+ path = os.path.join(model_dir, "config.json")+ try:+ with open(path, "r", encoding = "utf-8") as handle:+ config = json.load(handle)+ except Exception as exc:+ logger.debug("MLX estimate could not read %s: %s", path, exc)+ return None+ if not isinstance(config, dict):+ return None+ return _as_the_loader_reads_it(config)+++def _as_the_loader_reads_it(config: dict) -> dict:+ import contextlib+ import io++ try:+ from unsloth_zoo.mlx.loader import _mlx_vlm_config_override_data+ except Exception:+ return config+ try:+ with contextlib.redirect_stdout(io.StringIO()):+ patched = _mlx_vlm_config_override_data(config)+ except Exception:+ return config+ return patched if isinstance(patched, dict) else config+++def _escapes_snapshot(shard: str) -> bool:+ # Lexical, not realpath: hub snapshot files are symlinks into ../../blobs by design.+ norm = posixpath.normpath(shard.replace("\\", "/"))+ return norm.startswith("/") or norm.split("/")[0] == ".." or norm[1:2] == ":"+++def _indexed_shards(model_dir: str) -> list:+ index = os.path.join(model_dir, "model.safetensors.index.json")+ if not os.path.isfile(index):+ return []+ try:+ with open(index, encoding = "utf-8") as handle:+ weight_map = json.load(handle).get("weight_map", {})+ except (ValueError, OSError):+ return []+ if not isinstance(weight_map, dict):+ raise ValueError("model.safetensors.index.json has no weight map")+ if any(not isinstance(shard, str) for shard in weight_map.values()):+ raise ValueError("model.safetensors.index.json names a non-string shard")+ if any(_escapes_snapshot(shard) for shard in weight_map.values()):+ raise ValueError("model.safetensors.index.json names a shard outside the snapshot")+ named = sorted(set(weight_map.values()))+ return [+ os.path.join(model_dir, shard)+ for shard in named+ if os.path.isfile(os.path.join(model_dir, shard))+ ]+++def mlx_shard_files(model_dir: str, config: Optional[dict] = None) -> list:+ """The shards the package that would load this model actually reads."""+ names = [+ (os.path.basename(path), path)+ for path in glob.glob(os.path.join(model_dir, "*.safetensors"))+ ]+ if config is None:+ return [path for _, path in names]+ if _loads_as_vision(config):+ return _indexed_shards(model_dir) or [+ path for name, path in names if name != "consolidated.safetensors"+ ]+ return [path for name, path in names if name.startswith("model")]+++def _shard_bytes(model_dir: str, config: Optional[dict] = None) -> int:+ total = 0+ for shard in mlx_shard_files(model_dir, config):+ try:+ total += os.path.getsize(os.path.realpath(shard))+ except OSError:+ continue+ return total+++_SAFETENSORS_DTYPES = {+ "F64": "float32",+ "F32": "float32",+ "F16": "float16",+ "BF16": "bfloat16",+ "I64": "int64",+ "I32": "int32",+ "I16": "int16",+ "I8": "int8",+ "U64": "uint64",+ "U32": "uint32",+ "U16": "uint16",+ "U8": "uint8",+ "BOOL": "bool_",+}+++def _checkpoint_tensors(model_dir: str, config: Optional[dict], dtype):+ import mlx.core as mx++ tensors = {}+ for shard in mlx_shard_files(model_dir, config):+ with open(shard, "rb") as handle:+ length = int.from_bytes(handle.read(8), "little")+ limit = min(_MAX_SAFETENSORS_HEADER, os.fstat(handle.fileno()).st_size - 8)+ if not 0 < length <= limit:+ raise ValueError(f"{shard} declares a {length}-byte safetensors header")+ header = json.loads(handle.read(length))+ for name, meta in header.items():+ if name == "__metadata__":+ continue+ stored = getattr(mx, _SAFETENSORS_DTYPES.get(meta["dtype"], "float32"))+ if stored in (mx.float32, mx.float16, mx.bfloat16):+ stored = dtype+ tensors[name] = mx.zeros(tuple(meta["shape"]), dtype = stored)+ return tensors+++def _resident_bytes(model_dir: str, config: dict, quantize: bool) -> int:+ """Parameter bytes the load leaves resident, at the width it holds them."""+ import contextlib+ import io++ from mlx.utils import tree_flatten+ from unsloth_zoo.mlx.loader import (+ _apply_mlx_quantization,+ _resolve_mlx_quantization_spec,+ )++ spec = (+ None+ if not quantize+ else _resolve_mlx_quantization_spec(+ load_in_4bit = True,+ load_in_8bit = False,+ load_in_16bit = False,+ load_in_fp8 = False,+ load_in_mxfp4 = False,+ load_in_nvfp4 = False,+ full_finetuning = False,+ q_bits = None,+ q_group_size = None,+ q_mode = None,+ mlx_quantization_config = None,+ quantization_config = None,+ quant_predicate = None,+ quantize_modules = None,+ force_requantize = False,+ )+ )+ if quantize and not spec.enabled:+ raise ValueError("the loader would quantize nothing")++ dtype = _runtime_dtype()+ model = _whole_model(config, dtype)+ tensors = _checkpoint_tensors(model_dir, config, dtype)+ if not tensors:+ raise ValueError("this checkpoint ships no tensors")+ if hasattr(model, "sanitize"):+ tensors = model.sanitize(tensors)+ model.load_weights(list(tensors.items()), strict = False)+ orphaned = [name for name, _ in tree_flatten(model.parameters()) if name not in tensors]+ if orphaned:+ raise ValueError(+ f"{len(orphaned)} parameters this checkpoint does not supply, "+ f"beginning {orphaned[0]}"+ )+ if quantize:+ with contextlib.redirect_stdout(io.StringIO()):+ model, _ = _apply_mlx_quantization(+ model,+ config,+ spec,+ is_vlm = _loads_as_vision(config),+ )+ return sum(+ value.nbytes for _, value in tree_flatten(model.parameters()) if hasattr(value, "nbytes")+ )+++def mlx_weight_bytes(+ model_dir: str,+ config: Optional[dict] = None,+ load_in_4bit: bool = False,+) -> int:+ """Weight bytes an MLX load would leave resident."""+ shards = _shard_bytes(model_dir, config)+ if not config:+ return shards+ if config.get("quantization") or config.get("quantization_config"):+ return shards+ try:+ return _resident_bytes(model_dir, config, quantize = load_in_4bit)+ except Exception as exc:+ logger.debug("MLX estimate priced %s as stored: %s", model_dir, exc)+ return shards+++def _runtime_dtype():+ import mlx.core as mx+ chip = mx.device_info().get("device_name", "") or ""+ return mx.float16 if chip.startswith(("Apple M1", "Apple M2")) else mx.bfloat16+++def _generation_settings(config: dict) -> tuple:+ """``(prefill chunk, kv group size)`` a load of this model runs at."""+ try:+ from core.inference.mlx_inference import MLX_KV_GROUP_SIZE, mlx_prefill_chunk+ except Exception as exc:+ logger.debug("MLX estimate cannot reach the loader's generation settings: %s", exc)+ return MLX_PREFILL_CHUNK, _KV_GROUP_SIZE+ vision = _loads_as_vision(config)+ if vision and _routes_to_diffusion(config):+ raise ValueError("mlx-vlm would divert this to a diffusion generator")+ return (+ mlx_prefill_chunk(vision = vision, config = config if vision else None),+ MLX_KV_GROUP_SIZE,+ )+++def _loads_as_vision(config: dict) -> bool:+ from types import SimpleNamespace+ try:+ from utils.models.model_config import _is_vlm+ except Exception:+ return any(+ key in config+ for key in ("vision_config", "img_processor", "image_token_index", "projector_config")+ )+ return bool(_is_vlm(SimpleNamespace(**config)))+++def _routes_to_diffusion(config: dict) -> bool:+ """Whether mlx-vlm would divert this load to a diffusion generator."""+ try:+ from mlx_vlm.generate.diffusion import is_diffusion_model+ from mlx_vlm.utils import get_model_and_args+ except ImportError:+ return False+ arch = get_model_and_args(config)[0]+ loader_config = _loader_config(arch, config)+ if (+ getattr(loader_config, "canvas_length", None) is None+ and getattr(loader_config, "mask_token_id", None) is None+ ):+ return False+ return bool(is_diffusion_model(arch.Model(loader_config), {}))+++def _declines_to_chunk(model_class, model) -> bool:+ """Whether this model's runtime feeds the whole prompt to a single step."""+ return bool(+ getattr(model_class, "no_chunked_prefill", False)+ or getattr(model, "no_chunked_prefill", False)+ )+++def _tower_needs_parent(arch) -> bool:+ import inspect++ tower = getattr(arch, "LanguageModel", None)+ if tower is None or tower.__init__ is object.__init__:+ return True+ try:+ parameters = inspect.signature(tower.__init__).parameters+ except (TypeError, ValueError):+ return False+ required = [+ name+ for name, value in parameters.items()+ if name != "self"+ and value.default is inspect.Parameter.empty+ and value.kind+ in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)+ ]+ return len(required) > 1+++def _loader_config(arch, config: dict):+ """Config resolved as the loader does (``from_dict`` leaves nested sub-configs as dicts)."""+ from mlx_vlm.utils import update_module_configs++ config = dict(config)+ config.setdefault("text_config", config.pop("llm_config", {}))+ config.setdefault("vision_config", {})+ config.setdefault("audio_config", {})++ model_config = arch.ModelConfig.from_dict(config)+ model_config = update_module_configs(+ model_config,+ arch,+ config,+ ["text", "vision", "perceiver", "projector", "audio"],+ )+ return model_config+++def _whole_model(config: dict, dtype):+ if config.get("model_file"):+ raise ValueError("this checkpoint carries its own model module")+ if _loads_as_vision(config):+ from mlx_vlm.utils import get_model_and_args+ arch = get_model_and_args(config)[0]+ model = arch.Model(_loader_config(arch, config))+ else:+ from mlx_lm.utils import _get_classes+ model_class, args_class = _get_classes(config)+ model = model_class(args_class.from_dict(config))+ model.set_dtype(dtype)+ model.eval()+ return model+++def _probe_models(config: dict, dtype):+ """Language towers to try, in order, with the cache builder and class owning each."""+ if not config.get("model_type"):+ raise ValueError("config.json declares no model_type")+ if config.get("model_file"):+ raise ValueError("this checkpoint carries its own model module")++ def prepared(build):+ def _build():+ model = build()+ model.set_dtype(dtype)+ model.eval()+ return model++ return _build++ if not _loads_as_vision(config):+ from mlx_lm.models.cache import make_prompt_cache+ from mlx_lm.utils import _get_classes++ try:+ model_class, args_class = _get_classes(config)+ except Exception as exc:+ raise ValueError(f"mlx-lm cannot resolve this architecture: {exc}")+ yield (+ prepared(lambda: model_class(args_class.from_dict(config))),+ make_prompt_cache,+ model_class,+ )+ return++ from mlx_vlm.models.cache import make_prompt_cache+ from mlx_vlm.utils import get_model_and_args++ try:+ arch = get_model_and_args(config)[0]+ except Exception as exc:+ raise ValueError(f"mlx-vlm cannot resolve this architecture: {exc}")++ if not _tower_needs_parent(arch):++ def _tower():+ text_config = getattr(_loader_config(arch, config), "text_config", None)+ if text_config is None:+ raise ValueError("this architecture declares no text config")+ return arch.LanguageModel(text_config)++ yield prepared(_tower), make_prompt_cache, getattr(arch, "Model", None)++ def _whole_tower():+ whole = arch.Model(_loader_config(arch, config))+ return getattr(whole, "language_model", whole)++ yield prepared(_whole_tower), make_prompt_cache, getattr(arch, "Model", None)+++def _quantize_like_runtime(cache, kv_bits: Optional[int]):+ """The converted cache a load ends up with, or None if nothing in it converts."""+ if not kv_bits:+ return None+ from core.inference.mlx_inference import _quantize_kv_entries++ try:+ converted = _quantize_kv_entries(list(cache), kv_bits)+ except Exception:+ return None+ return converted if any(new is not old for new, old in zip(converted, cache)) else None+++def _sub_caches(entry):+ inner = getattr(entry, "caches", None)+ if not isinstance(inner, (list, tuple)):+ return (entry,)+ return tuple(sub for cache in inner for sub in _sub_caches(cache))+++def _block(entry) -> int:+ """Allocation granularity of the storage behind ``entry``, in tokens."""+ step = getattr(entry, "step", None)+ return step if isinstance(step, int) and step > 0 else 1+++def _bound_spec(entry):+ for name in _CACHE_BOUND_ATTRS:+ value = getattr(entry, name, None)+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:+ return type(entry), name, value+ return None+++def _entry_bytes(entry) -> int:+ import mlx.core as mx++ state = entry.state+ if isinstance(state, mx.array):+ state = (state,)+ elif not isinstance(state, (list, tuple)):+ return 0+ total = 0+ for item in state:+ if isinstance(item, mx.array):+ total += item.size * item.dtype.size+ elif isinstance(item, (list, tuple)):+ total += sum(sub.size * sub.dtype.size for sub in item if isinstance(sub, mx.array))+ return total+++def _width_name(dtype) -> str:+ name = str(dtype).rsplit(".", 1)[-1]+ return _WIDTH_NAMES.get(name, name)+++def _cache_width_name(+ plan, kv_bits, converted: bool, full_width: str, n_ctx: int, prefill_chunk: int+) -> str:+ """How to caption the cache, from the width each growing entry ends up at."""+ charged: dict = {}+ for slot in plan:+ if slot["slope"] <= 0:+ continue+ quantized = converted and slot["converts"]+ name = f"{kv_bits}-bit" if quantized else full_width+ const, slope = ("quant_const", "quant_slope") if quantized else ("const", "slope")+ charged[name] = (+ charged.get(name, 0.0)+ + slot[const]+ + slot[slope] * _held_tokens(slot, n_ctx, prefill_chunk, True)+ )+ if not charged:+ return full_width+ return "/".join(sorted(charged, key = lambda name: charged[name], reverse = True))+++def _conv_width(entry) -> int:+ """Total channel width of a linear-attention layer's convolution states."""+ import mlx.core as mx++ state = entry.state+ if not isinstance(state, (list, tuple)):+ return 0+ return sum(+ int(item.shape[-1]) for item in state if isinstance(item, mx.array) and item.ndim == 3+ )+++def _tower_layers(model) -> Optional[int]:+ seen = model+ for _ in range(3):+ layers = getattr(seen, "layers", None)+ if isinstance(layers, list) and layers:+ return len(layers)+ seen = getattr(seen, "model", None) or getattr(seen, "language_model", None)+ if seen is None:+ return None+ return None+++def _tower_widths(model):+ """The widths in ``_config_widths``' order as the tower was BUILT, or zeros."""+ node, depth = model, 0+ while node is not None and depth < 3:+ for name in ("args", "config"):+ resolved = getattr(node, name, None)+ hidden = (+ 0 if resolved is None else _config_width(getattr(resolved, "hidden_size", None))+ )+ if hidden > 0:+ heads = _config_width(getattr(resolved, "num_attention_heads", None))+ return (+ hidden,+ _config_width(getattr(resolved, "intermediate_size", None))+ or _config_width(getattr(resolved, "moe_intermediate_size", None)),+ heads,+ _config_width(getattr(resolved, "num_key_value_heads", None)) or heads,+ _config_width(getattr(resolved, "head_dim", None))+ or (hidden // heads if heads else 0),+ _config_width(getattr(resolved, "v_head_dim", None)),+ )+ node = getattr(node, "model", None) or getattr(node, "language_model", None)+ depth += 1+ return (0,) * len(_config_widths({}))+++def _config_widths(config: dict):+ text = config.get("text_config") or config+ hidden = _config_width(text.get("hidden_size"))+ heads = _config_width(text.get("num_attention_heads"))+ return (+ hidden,+ _config_width(text.get("intermediate_size"))+ or _config_width(text.get("moe_intermediate_size")),+ heads,+ _config_width(text.get("num_key_value_heads")) or heads,+ _config_width(text.get("head_dim")) or (hidden // heads if heads else 0),+ _config_width(text.get("v_head_dim")),+ )+++def _probe(config: dict, dtype, n_tokens: int, kv_bits):+ import mlx.core as mx++ cache = None+ whole_prompt = False+ layers = None+ widths = (0,) * len(_config_widths({}))+ failure = ValueError("no architecture module could build this config")+ for build, make_prompt_cache, model_class in _probe_models(config, dtype):+ model = build()+ cache = make_prompt_cache(model)+ try:+ model(mx.zeros((1, n_tokens), dtype = mx.int32), cache = cache)+ except Exception as exc:+ failure, cache = exc, None+ continue+ whole_prompt = _declines_to_chunk(model_class, model)+ layers = _tower_layers(model)+ widths = _tower_widths(model)+ break+ if cache is None:+ raise failure+ quantized = _quantize_like_runtime(cache, kv_bits)+ entries = []+ for index, entry in enumerate(cache):+ converts = quantized is not None and quantized[index] is not entry+ for leaf in _sub_caches(entry):+ entries.append(+ {+ "bytes": _entry_bytes(leaf),+ "quant_bytes": _entry_bytes(leaf),+ "bound_spec": _bound_spec(leaf),+ "conv_width": _conv_width(leaf),+ "block": _block(leaf),+ "converts": converts,+ }+ )+ if quantized is not None:+ converted = [leaf for entry in quantized for leaf in _sub_caches(entry)]+ if len(converted) == len(entries):+ for slot, leaf in zip(entries, converted):+ slot["quant_bytes"] = _entry_bytes(leaf)+ return {+ "entries": entries,+ "quantized": quantized is not None,+ "whole_prompt": whole_prompt,+ "layers": layers,+ "widths": widths,+ }+++def _cache_plan(config: dict, dtype, kv_bits):+ """Per-entry ``const + slope * T``, solved from two lazy probes."""+ near = _probe(config, dtype, _PROBE_SHORT, kv_bits)+ far = _probe(config, dtype, _PROBE_LONG, kv_bits)+ if not near["entries"] or len(near["entries"]) != len(far["entries"]):+ raise ValueError("cache probe returned no comparable entries")+ span = _PROBE_LONG - _PROBE_SHORT++ def solve(key):+ return [+ (n[key] - (f[key] - n[key]) / span * _PROBE_SHORT, (f[key] - n[key]) / span)+ for n, f in zip(near["entries"], far["entries"])+ ]++ plan = []+ for slot, (const, slope), (q_const, q_slope) in zip(+ near["entries"], solve("bytes"), solve("quant_bytes")+ ):+ plan.append(+ {+ "const": const,+ "slope": slope,+ "quant_const": q_const,+ "quant_slope": q_slope,+ "bound_spec": slot["bound_spec"],+ "conv_width": slot["conv_width"],+ "block": slot["block"],+ "converts": slot["converts"] and near["quantized"],+ }+ )+ return (+ plan,+ 0 if near["quantized"] else None,+ {+ "whole_prompt": near["whole_prompt"],+ "layers": near["layers"],+ "widths": near["widths"],+ },+ )++[email protected]_cache(maxsize = 256)+def _bounded_peak(cache_type, attribute: str, value: int, n_ctx: int, prefill_chunk: int) -> int:+ """Largest slot count a bounded cache reaches, by driving the real class."""+ import mlx.core as mx++ cache = cache_type(**{attribute: value})+ settled = value + 4 * (prefill_chunk + (getattr(cache, "step", 0) or MLX_KV_BLOCK))+ n_ctx = min(n_ctx, settled)+ peak, processed = 0, 0++ def call(width):+ nonlocal peak+ trim = getattr(cache, "maybe_trim_front", None)+ if callable(trim):+ trim()+ cache.update_and_fetch(+ mx.zeros((1, 1, width, 1), dtype = mx.bfloat16),+ mx.zeros((1, 1, width, 1), dtype = mx.bfloat16),+ )+ keys = getattr(cache, "keys", None)+ peak = max(peak, 0 if keys is None else int(keys.shape[2]))++ while n_ctx - processed > 1:+ width = min(prefill_chunk, n_ctx - processed - 1)+ call(width)+ processed += width+ call(1)+ call(1)+ return peak+++def _held_tokens(+ entry,+ n_ctx: int,+ prefill_chunk: int,+ decoding: bool = True,+) -> int:+ tokens = n_ctx+ block = entry.get("block") or MLX_KV_BLOCK+ if entry["slope"] > 0:+ tokens += 1 if decoding else 0+ if block > 1:+ tokens = -(-tokens // block) * block+ spec = entry.get("bound_spec")+ if spec:+ tokens = _bounded_peak(*spec, n_ctx, prefill_chunk)+ return tokens+++def _line_bytes(+ plan,+ n_ctx: int,+ prefill_chunk: int,+ quantized: bool,+ decoding: bool = True,+) -> int:+ const, slope = ("quant_const", "quant_slope") if quantized else ("const", "slope")+ return int(+ sum(+ entry[const] + entry[slope] * _held_tokens(entry, n_ctx, prefill_chunk, decoding)+ for entry in plan+ )+ )+++def _kv_bytes(plan, n_ctx: int, quant_start, prefill_chunk: int):+ """Peak cache bytes across the run, and where the cache is quantized from."""+ if quant_start is None:+ return _line_bytes(plan, n_ctx, prefill_chunk, quantized = False), None+ # The load converts the cache before prefill, so no full-width copy is ever held.+ return _line_bytes(plan, n_ctx, prefill_chunk, quantized = True), quant_start+++def _config_width(value) -> int:+ """A width from the config, which is not always a single number."""+ if isinstance(value, bool):+ return 0+ if isinstance(value, (int, float)):+ return int(value)+ if isinstance(value, (list, tuple)):+ widths = [int(v) for v in value if isinstance(v, (int, float)) and not isinstance(v, bool)]+ return max(widths) if widths else 0+ return 0+++def _widest_quantized_scores(n_ctx: int, boundary: int, prefill_chunk: int) -> int:+ widest = 0+ remaining = n_ctx - boundary - 1+ if remaining > 0:+ whole, rest = divmod(remaining, prefill_chunk)+ if whole:+ widest = prefill_chunk * (boundary + whole * prefill_chunk)+ if rest:+ widest = max(widest, rest * (n_ctx - 1))+ return max(widest, n_ctx + 1)+++def _dequantized_row_bytes(widths, dtype_size: int, kv_bits, kv_group_size, prefill_chunk):+ """Bytes per cached token one attention call holds where the runtime dequantizes the cache."""+ _hidden, _intermediate, heads, kv_heads, head_dim, value_dim = widths+ if not (heads and kv_heads and head_dim and kv_bits):+ return None+ try:+ import mlx.core as mx++ from unsloth_zoo.mlx.attention import (+ _row_bytes,+ dequantizing_is_smaller,+ fused_kernel_exists,+ )+ except Exception as exc:+ logger.debug("MLX estimate cannot reach the quantized-attention wrapper: %s", exc)+ return None+ try:+ dtype = mx.float32 if dtype_size == 4 else mx.bfloat16++ def rows(width):+ return mx.quantize(+ mx.zeros((1, kv_heads, 1, width), dtype = dtype),+ group_size = kv_group_size,+ bits = kv_bits,+ )++ keys, values = rows(head_dim), rows(value_dim or head_dim)+ queries = mx.zeros((1, heads, prefill_chunk, head_dim), dtype = dtype)+ if not fused_kernel_exists(queries, keys, values, kv_group_size, "causal"):+ return None+ if not dequantizing_is_smaller(queries, keys, values, kv_group_size):+ return None+ return kv_heads * sum(_row_bytes(cache, kv_group_size, dtype) for cache in (keys, values))+ except Exception as exc:+ logger.debug("MLX estimate cannot route the quantized attention: %s", exc)+ return None+++def _compute_bytes(+ widths,+ dtype_size: int,+ prefill_chunk: int,+ plan,+ n_ctx: int,+ quant_boundary,+ kv_bits = None,+ kv_group_size: int = _KV_GROUP_SIZE,+) -> int:+ """Transient buffers a prefill step holds, on top of weights and cache."""+ hidden, intermediate, heads = widths[:3]+ if hidden <= 0:+ return 0+ intermediate = intermediate or 4 * hidden+ attention_width = (hidden + intermediate) * dtype_size * _ATTENTION_LIVE+ recurrent_width = sum(int(entry["conv_width"]) for entry in plan) * _RECURRENT_DTYPE_SIZE+ total = _COMPUTE_BASE_BYTES + prefill_chunk * (attention_width + recurrent_width)+ if quant_boundary is not None:+ score = heads * _QUANT_SCORE_DTYPE_SIZE * _QUANT_SCORE_LIVE+ row = _dequantized_row_bytes(widths, dtype_size, kv_bits, kv_group_size, prefill_chunk)+ if row is None:+ total += _widest_quantized_scores(n_ctx, quant_boundary, prefill_chunk) * score+ else:+ total += max(row * n_ctx, (n_ctx + 1) * score)+ return int(total)+++def _load_is_refused(model_dir: str, config: dict, load_in_4bit: bool) -> Optional[str]:+ try:+ from unsloth_zoo.mlx.loader import (+ _ensure_quantization_compatible,+ _get_existing_mlx_quantization,+ _resolve_mlx_quantization_spec,+ )+ except Exception:+ return None++ import contextlib+ import io++ existing = _get_existing_mlx_quantization(config)+ if isinstance(existing, dict) and existing.get("quant_method") == "bitsandbytes":+ return "bitsandbytes weights mlx-lm cannot read"++ try:+ with contextlib.redirect_stdout(io.StringIO()):+ spec = _resolve_mlx_quantization_spec(+ load_in_4bit = bool(load_in_4bit),+ load_in_8bit = False,+ load_in_16bit = False,+ load_in_fp8 = False,+ load_in_mxfp4 = False,+ load_in_nvfp4 = False,+ full_finetuning = False,+ q_bits = None,+ q_group_size = None,+ q_mode = None,+ mlx_quantization_config = None,+ quantization_config = None,+ quant_predicate = None,+ quantize_modules = None,+ force_requantize = False,+ )+ _ensure_quantization_compatible(config, spec, "this checkpoint")+ except ValueError as exc:+ return str(exc)+ except Exception:+ return None++ return _refused_by_its_own_tensors(model_dir, config)+++def _extra_tensors_refused(model, config: dict, extras: list) -> Optional[str]:+ try:+ from unsloth_zoo.mlx.loader import (+ _KNOWN_MLX_LM_STRICT_FALLBACKS,+ _KNOWN_VLM_EXTRA_WEIGHT_FILTERS,+ _gemma4_unused_shared_kv_weight,+ _message_matches_known_fallback,+ _raise_if_qk_norm_version_gap,+ )+ except Exception:+ return None++ model_type = config.get("model_type")+ message = f"Received {len(extras)} parameters not in model: \n{extras}."+ try:+ _raise_if_qk_norm_version_gap(model_type, message, ValueError(message))+ except ValueError as refusal:+ return str(refusal)++ vision = _loads_as_vision(config)+ table = _KNOWN_VLM_EXTRA_WEIGHT_FILTERS if vision else _KNOWN_MLX_LM_STRICT_FALLBACKS+ rule = table.get(model_type)+ if rule is not None and _message_matches_known_fallback(message, rule):+ allowed = rule.get("allowed_extra", frozenset())+ shared_kv = rule.get("allow_shared_kv", False)+ extras = [+ name+ for name in extras+ if name not in allowed+ and not (shared_kv and _gemma4_unused_shared_kv_weight(model, name))+ ]+ if not extras:+ return None++ return f"{len(extras)} tensors this architecture has no home for, " f"beginning {extras[0]}"+++def _refused_by_its_own_tensors(model_dir: str, config: dict) -> Optional[str]:+ """Refusals only the checkpoint's own tensors reveal."""+ try:+ from mlx.utils import tree_flatten+ from unsloth_zoo.mlx.loader import _raise_if_qk_norm_version_gap+ except Exception:+ return None++ import contextlib+ import io++ try:+ with contextlib.redirect_stdout(io.StringIO()):+ dtype = _runtime_dtype()+ model = _whole_model(config, dtype)+ tensors = _checkpoint_tensors(model_dir, config, dtype)+ if not tensors:+ return None+ if hasattr(model, "sanitize"):+ tensors = model.sanitize(tensors)+ built = {name for name, _ in tree_flatten(model.parameters())}+ except Exception:+ return None++ unhoused = sorted(+ name for name in set(tensors) - built if not name.endswith((".scales", ".biases"))+ )+ if unhoused:+ refusal = _extra_tensors_refused(model, config, unhoused)+ if refusal:+ return refusal++ try:+ with contextlib.redirect_stdout(io.StringIO()):+ model.load_weights(list(tensors.items()), strict = True)+ except Exception as exc:+ try:+ _raise_if_qk_norm_version_gap(config.get("model_type"), str(exc), exc)+ except ValueError as refusal:+ return str(refusal)+ except Exception:+ return None+ return None+++@dataclass+class _MlxSizing:+ """What a load costs before a context is named, so a search over contexts probes once."""++ weights: int+ dtype: object+ plan: list+ quant_start: Optional[int]+ facts: dict+ widths: tuple+ chunk: int+ kv_bits: Optional[int]+ kv_group_size: int+++_SIZING_CACHE_MAX = 16+_sizing_cache: "OrderedDict[tuple, Optional[_MlxSizing]]" = OrderedDict()+_sizing_lock = threading.Lock()+++def _checkpoint_fingerprint(model_dir: str) -> tuple:+ """Size and mtime of every config, index and shard file, so a changed download re-sizes."""+ root = os.path.realpath(model_dir)+ found = []+ for pattern in ("*.json", "**/*.safetensors"):+ for path in sorted(glob.glob(os.path.join(root, pattern), recursive = True)):+ try:+ stat = os.stat(path)+ except OSError:+ continue+ found.append((os.path.relpath(path, root), stat.st_size, stat.st_mtime_ns))+ return tuple(found)+++def _size_load(+ model_dir: str, kv_bits: Optional[int], prefill_chunk: Optional[int], load_in_4bit: bool+) -> Optional[_MlxSizing]:+ """Cached ``_size_load_uncached``: every estimate prices the fit and the breakdown, and every+ slider step prices again, each otherwise rebuilding the same weightless architecture."""+ try:+ key = (+ os.path.realpath(model_dir),+ kv_bits,+ prefill_chunk,+ bool(load_in_4bit),+ os.environ.get("UNSLOTH_MLX_PROMPT_CACHE_BYTES"),+ _checkpoint_fingerprint(model_dir),+ )+ except Exception:+ return _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit)+ with _sizing_lock:+ if key in _sizing_cache:+ _sizing_cache.move_to_end(key)+ return _sizing_cache[key]+ sizing = _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit)+ with _sizing_lock:+ _sizing_cache[key] = sizing+ while len(_sizing_cache) > _SIZING_CACHE_MAX:+ _sizing_cache.popitem(last = False)+ return sizing+++def _clear_sizing_cache() -> None:+ with _sizing_lock:+ _sizing_cache.clear()+++def _size_load_uncached(+ model_dir: str, kv_bits: Optional[int], prefill_chunk: Optional[int], load_in_4bit: bool+) -> Optional[_MlxSizing]:+ """Everything about a load that does not move with the context, or None if it cannot be sized."""+ config = _snapshot_config(model_dir)+ if config is None:+ return None+ refusal = _load_is_refused(model_dir, config, load_in_4bit)+ if refusal is not None:+ logger.debug("MLX estimate refuses %s: %s", model_dir, refusal)+ return None+ try:+ weights = mlx_weight_bytes(model_dir, config, load_in_4bit)+ except Exception as exc:+ logger.debug("MLX estimate cannot read the shards of %s: %s", model_dir, exc)+ return None+ if weights <= 0:+ return None+ try:+ dtype = _runtime_dtype()+ loaded_chunk, loaded_group = _generation_settings(config)+ plan, quant_start, facts = _cache_plan(config, dtype, kv_bits)+ widths = tuple(+ tower or checkpoint+ for tower, checkpoint in zip(facts["widths"], _config_widths(config))+ )+ chunk = prefill_chunk or loaded_chunk+ except Exception as exc:+ logger.debug(+ "MLX estimate could not size %s (%s): %s", model_dir, config.get("model_type"), exc+ )+ return None+ return _MlxSizing(+ weights = weights,+ dtype = dtype,+ plan = plan,+ quant_start = quant_start,+ facts = facts,+ widths = widths,+ chunk = chunk,+ kv_bits = kv_bits,+ kv_group_size = loaded_group,+ )+++def _priced_at(sizing: _MlxSizing, n_ctx: int) -> Optional[MlxMemoryBreakdown]:+ try:+ context = max(int(n_ctx or 0), 1)+ whole_prompt = sizing.facts["whole_prompt"]+ chunk = context if whole_prompt else sizing.chunk+ kv, quant_boundary = _kv_bytes(sizing.plan, context, sizing.quant_start, chunk)+ compute = _compute_bytes(+ sizing.widths,+ sizing.dtype.size,+ chunk,+ sizing.plan,+ context,+ quant_boundary,+ sizing.kv_bits,+ sizing.kv_group_size,+ )+ except Exception as exc:+ logger.debug("MLX estimate could not price %s tokens: %s", n_ctx, exc)+ return None+ total = sizing.weights + kv + compute+ return MlxMemoryBreakdown(+ weights_bytes = sizing.weights,+ kv_bytes = kv,+ compute_bytes = compute,+ total_bytes = total,+ gpu_bytes = total,+ n_ctx = context,+ layer_count = sizing.facts["layers"],+ cache_type_kv = _cache_width_name(+ sizing.plan,+ sizing.kv_bits,+ quant_boundary is not None,+ _width_name(sizing.dtype),+ context,+ chunk,+ ),+ )+++def mlx_memory_breakdown(+ model_dir: str,+ *,+ n_ctx: int,+ kv_bits: Optional[int] = None,+ prefill_chunk: Optional[int] = None,+ load_in_4bit: bool = False,+) -> Optional[MlxMemoryBreakdown]:+ """Price an MLX load, or None when it cannot honestly be sized."""+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit)+ return None if sizing is None else _priced_at(sizing, n_ctx)+++def mlx_fit_context(+ model_dir: str,+ *,+ budget_bytes: int,+ max_ctx: int,+ min_ctx: int = MLX_FIT_MIN_CONTEXT,+ kv_bits: Optional[int] = None,+ prefill_chunk: Optional[int] = None,+ load_in_4bit: bool = False,+) -> Optional[int]:+ """Largest context whose estimated footprint stays inside ``budget_bytes``."""+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit)+ if sizing is None:+ return None+ ceiling = max(int(max_ctx or 0), MLX_KV_BLOCK)+ at_ceiling = _priced_at(sizing, ceiling)+ if at_ceiling is None or at_ceiling.total_bytes <= budget_bytes:+ return None+ floor = max(-(-int(min_ctx or 0) // MLX_KV_BLOCK) * MLX_KV_BLOCK, MLX_KV_BLOCK)+ low, high, best = floor, ceiling, None+ while low <= high:+ middle = (low + high) // 2+ priced = _priced_at(sizing, middle)+ if priced is None:+ return None+ if priced.total_bytes <= budget_bytes:+ best = middle+ low = middle + 1+ else:+ high = middle - 1+ return None if best is None else (best // MLX_KV_BLOCK) * MLX_KV_BLOCKdiff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 9137f128cef..ac1c715f2ba 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -377,6 +377,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:"max_context_length": model_info.get("max_context_length"),"requested_context_length": model_info.get("requested_context_length"),"context_length_enforced": model_info.get("context_length_enforced"),+ "context_length_fitted": model_info.get("context_length_fitted"),"context_unbounded_when_batched": model_info.get("context_unbounded_when_batched"),"mlx_context_budget": model_info.get("mlx_context_budget"),}diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 4c67f8433bf..6a3f6be5328 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -640,13 +640,12 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:_entry = (_bm.get(mc.identifier) or _bm.get(getattr(backend, "active_model_name", None)) or {})- # The whole group: the parent reports all four and can recompute none of- # them once the worker holds the model.for _ctx_field in ("context_length","native_context_length","max_context_length","requested_context_length",+ "context_length_fitted","mlx_context_budget",):try:diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 53123838982..68d58a2c5db 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -866,7 +866,9 @@ class EstimateMemoryRequest(BaseModel):"""Settings a Load-Model panel is about to submit, priced before it submits them.Every field mirrors the load request it previews, so the estimate answers for the- command that would actually run. Header-only: nothing is read, touched or loaded.+ command that would actually run. No model is loaded and no tensor data is read: a+ GGUF is priced from its header, an MLX repo from safetensors headers and the shapes+ of a graph that is built and never evaluated."""model_path: str = Field(..., description = "Model identifier or local path")@@ -884,6 +886,14 @@ class EstimateMemoryRequest(BaseModel):description = "Context length to price (--ctx-size). 0 or omitted prices the ""model's native context, which is what an Auto load asks for.",)+ load_in_4bit: bool = Field(+ True,+ description = "Quantize an unquantized checkpoint to 4 bits on load, as "+ "/load does by default. Priced because it is the difference between a "+ "checkpoint's shards and what those shards go resident as. A checkpoint "+ "carrying its own quantization is priced at its shards, and asking for 4 "+ "bits on top can make it unsizable, because the real load refuses that.",+ )cache_type_kv: Optional[str] = Field(None,description = "KV cache dtype to price. The biggest lever on this estimate at "@@ -940,8 +950,30 @@ class EstimateMemoryRequest(BaseModel):"-nkvo, --swa-full and the cache-type flags all move this estimate.",)+ max_seq_length: Optional[int] = Field(+ None,+ ge = 0,+ le = 1048576,+ description = "Context an MLX load would open at. Separate from n_ctx because "+ "the two backends take the length from different fields -- /load reads n_ctx "+ "for GGUF and max_seq_length for everything else -- and pricing one from the "+ "other quotes a context the load will not use.",+ )+ mlx_kv_quant: Optional[str] = Field(+ None,+ description = "MLX KV cache quantization in /load's vocabulary ('auto', '8', 'tq-4', ...). "+ "A TurboQuant choice is priced at full width, as the load fits it.",+ )+ mlx_kv_bits: Optional[int] = Field(+ None,+ description = "Superseded by mlx_kv_quant; read only when mlx_kv_quant is omitted. Not derived "+ "from cache_type_kv: that is llama.cpp's setting, it survives in the config of "+ "a model that never used it, and reading it here would quantize an estimate "+ "for a load whose cache stays full width.",+ )+_no_booleans = field_validator(- "n_batch", "n_ubatch", "ctx_checkpoints", "n_ctx", mode = "before"+ "n_batch", "n_ubatch", "ctx_checkpoints", "n_ctx", "max_seq_length", mode = "before")(LoadRequest._no_booleans.__func__)# Unresolved, the reference reads as a Hub id and the panel is told the estimate is# unavailable for a row it was offered.@@ -957,7 +989,12 @@ class EstimateMemoryResponse(BaseModel):description = "Cause when available is false: 'not_gguf', 'not_downloaded', ""'unsupported_source' or 'unsizable'.",)- weights_bytes: int = Field(0, description = "Resident model files: weights, projector, drafter")+ weights_bytes: int = Field(+ 0,+ description = "Resident weights, projector and drafter. The size of the files "+ "for a GGUF load; for MLX, what those files go resident AS, which is smaller "+ "wherever the load casts or quantizes them.",+ )kv_bytes: int = Field(0, description = "KV cache at the requested context and slots")kv_checkpoint_bytes: int = Field(0,@@ -1013,6 +1050,13 @@ class EstimateMemoryResponse(BaseModel):True, description = "False under --no-kv-offload, which moves the cache to host RAM")n_ctx: int = Field(0, description = "Context length the estimate actually priced")+ context_fitted: Optional[int] = Field(+ None,+ description = "The window MLX would fit to this machine's memory for a load that "+ "names no Context Length, which is then what n_ctx prices. Null when the load names "+ "one, when the machine holds the model's own window, or when the footprint of the "+ "load cannot be described.",+ )cache_type_kv: Optional[str] = Field(None, description = "KV dtype the estimate priced, after flags and fallbacks resolve")@@ -1112,13 +1156,28 @@ class MemoryEstimate(BaseModel):native_context: Optional[int] = Field(None, description = "The model's own trained context length, when readable")- cache_type_kv: Optional[str] = Field(None, description = "KV cache dtype the estimate priced")+ cache_type_kv: Optional[str] = Field(+ None,+ description = (+ "KV cache width the estimate priced, in the vocabulary of whichever backend "+ "would load it: llama.cpp spellings such as f16 or q8_0 for a GGUF load, and "+ "a dtype abbreviation or an N-bit width for an MLX one. An MLX cache whose "+ "entries do not all end up at one width names each of them, the one carrying "+ "the most of the cache first, as in bf16/4-bit"+ ),+ )n_parallel: int = Field(1, description = "Slots the estimate priced, after the launch clamps")layer_count: Optional[int] = Field(None, description = "GGUF block_count, when readable")gpu_layers: Optional[int] = Field(None, description = "Layers placed on the GPU, when known")moe_offload_unmodelled: bool = Field(False, description = "--n-cpu-moe is set, so the GPU figure reads high")+ context_fitted: Optional[int] = Field(+ None,+ description = "The window MLX would fit to this machine's memory for a load that "+ "names no Context Length, and which n_ctx then prices. Null wherever the served "+ "window was not chosen for this machine.",+ )context_is_pinned: bool = Field(True,description = "False only when the loader is free to shrink the context. A caller "@@ -1259,6 +1318,14 @@ class _InferenceRuntimeFields(BaseModel):"bounded while still reporting true -- so read it as the weaker claim it is."),)+ context_length_fitted: Optional[int] = Field(+ None,+ description = (+ "The window MLX fitted to this machine's memory, where that is shorter than the "+ "model's own and no Context Length was requested. Null means context_length was not "+ "chosen for this machine: a pin, or the model's own window."+ ),+ )mlx_context_budget: Optional[int] = Field(None,description = (diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 62f1b9ae9a6..365640be0ca 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -7607,6 +7607,7 @@ def _unsloth_serving_fields(model_info: dict) -> dict:"requested_parallel_slots": slots,"parallel_slots": effective,"context_length_enforced": model_info.get("context_length_enforced"),+ "context_length_fitted": _positive_int_or_none(model_info.get("context_length_fitted")),"context_unbounded_when_batched": bool(model_info.get("context_unbounded_when_batched")),}@@ -7632,6 +7633,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:mlx_context_budget = None,chat_template_override_reason = None,context_length_enforced = True,+ context_length_fitted = None,context_unbounded_when_batched = False,# Older/custom backend doubles predate this additive runtime field.preserve_thinking_default = bool(getattr(llama_backend, "preserve_thinking_default", False)),@@ -13821,6 +13823,194 @@ def _local_gguf_main_path(config: ModelConfig) -> Optional[str]:return None+def _mlx_estimate_ceiling(model_dir: str) -> Optional[int]:+ """The window this checkpoint declares, held to what a load may be asked for."""+ from types import SimpleNamespace++ from core.inference.mlx_inference import mlx_native_context_length+ from core.inference.mlx_memory import _snapshot_config+ from core.inference.runtime_context import MAX_REQUESTABLE_CONTEXT++ try:+ config = _snapshot_config(model_dir)+ except Exception:+ return None+ if config is None:+ return None+ native = mlx_native_context_length(SimpleNamespace(config = config))+ return None if native is None else min(int(native), MAX_REQUESTABLE_CONTEXT)+++def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_bits):+ """The window a load naming no Context Length would be fitted to."""+ from core.inference.mlx_inference import (+ mlx_fit_to_memory,+ mlx_vlm_snapshot_store_available,+ )++ ceiling = _mlx_estimate_ceiling(model_dir)+ if not ceiling:+ return None+ return mlx_fit_to_memory(+ model_dir,+ ceiling,+ load_in_4bit = load_in_4bit,+ retains_history = (+ not getattr(config, "is_vision", False) or mlx_vlm_snapshot_store_available()+ ),+ kv_bits = kv_bits,+ )+++def _mlx_estimate_available() -> bool:+ """Whether this host would actually run the load through MLX."""+ try:+ from utils.mlx_repair import is_apple_silicon, mlx_stack_blockers+ except Exception:+ return False+ try:+ return bool(is_apple_silicon() and not mlx_stack_blockers())+ except Exception:+ return False+++def _the_revision_that_loads(snapshots: list) -> list:+ """Just the revision `main` names, where it names one that is here."""+ from hub.utils.hf_cache_state import ref_snapshot_dir++ paths = [Path(snapshot) for snapshot in snapshots]+ if not paths:+ return paths+ pinned = ref_snapshot_dir(paths[0].parent.parent)+ return [pinned] if pinned is not None else paths+++def _local_mlx_model_dir(config: ModelConfig) -> Optional[str]:+ def _complete(directory: Path, shards) -> bool:+ named = [+ (Path(shard).parent, match)+ for shard in shards+ if (match := _re.fullmatch(r"(.+)-(\d+)-of-(\d+)\.safetensors", Path(shard).name))+ ]+ if not named:+ return _index_agrees(directory)+ for parent, match in named:+ stem, total = match.group(1), match.group(3)+ width, count = len(match.group(2)), int(total)+ if any(+ not (parent / f"{stem}-{i:0{width}d}-of-{total}.safetensors").is_file()+ for i in range(1, count + 1)+ ):+ return False+ return _index_agrees(directory)++ def _index_agrees(directory: Path) -> bool:+ """Whether an index that describes THIS directory finds all of it here."""+ index = directory / "model.safetensors.index.json"+ if not index.is_file():+ return True+ try:+ with open(index, encoding = "utf-8") as handle:+ weight_map = json.load(handle).get("weight_map", {})+ named = {shard for shard in weight_map.values() if isinstance(shard, str)}+ except (ValueError, OSError, AttributeError):+ return True+ here = {+ path.relative_to(directory).as_posix() for path in directory.glob("**/*.safetensors")+ }+ return not (named & here) or named <= here++ def _usable(path) -> bool:+ directory = Path(path)+ if not directory.is_dir() or not (directory / "config.json").is_file():+ return False+ from core.inference.mlx_memory import mlx_shard_files++ try:+ snapshot = json.loads((directory / "config.json").read_text(encoding = "utf-8"))+ except Exception:+ snapshot = None+ try:+ shards = mlx_shard_files(+ str(directory), snapshot if isinstance(snapshot, dict) else None+ )+ except Exception:+ return False+ if not shards:+ return False+ return _complete(directory, shards)++ candidate = getattr(config, "path", None)+ if candidate and _usable(candidate):+ return str(candidate)+ identifier = candidate or getattr(config, "identifier", None)+ if not identifier or getattr(config, "is_local", False):+ return None+ names = [_mlx_load_identifier(str(identifier))]+ if "/" not in names[0]:+ names.append(f"unsloth/{names[0]}")+ try:+ from utils.models.model_config import _iter_hf_cache_snapshots+ roots = _estimate_hf_cache_roots()+ except Exception:+ return None+ for root in roots or [None]:+ for name in names:+ try:+ snapshots = _iter_hf_cache_snapshots(name, cache_dir = root)+ except Exception:+ continue+ for snapshot in _the_revision_that_loads(list(snapshots)):+ if _usable(snapshot):+ return str(snapshot)+ return None+++def _mlx_load_identifier(identifier: str) -> str:+ """The repository the MLX load would actually open, not the one selected."""+ try:+ from unsloth_zoo.mlx.loader import _remap_unsloth_bnb_hub_id_for_mlx+ except Exception:+ return identifier+ try:+ return _remap_unsloth_bnb_hub_id_for_mlx(identifier, None)[0] or identifier+ except Exception:+ return identifier+++def _mlx_estimate_load_in_4bit(config, request) -> bool:+ """The 4-bit setting the load resolves, not the one the panel sent."""+ # Same order as the load route: a full-finetune output loads 16-bit before any tier check.+ if not _effective_load_in_4bit(config, request.load_in_4bit):+ return False+ from utils.transformers_version import latest_tier_active_for++ identifier = getattr(config, "identifier", None)+ return not _offline_guarded(+ (identifier, getattr(config, "base_model", None)),+ latest_tier_active_for,+ identifier,+ request.hf_token,+ )+++def _mlx_estimate_kv_bits(mlx_kv_quant, mlx_kv_bits = None) -> Optional[int]:+ """The mx.quantize width the load would price, or None for a full-width cache."""+ try:+ from core.inference.mlx_inference import MLX_KV_BITS_CHOICES, parse_mlx_kv_quant+ except Exception:+ return None+ if mlx_kv_quant is None:+ if isinstance(mlx_kv_bits, bool) or not isinstance(mlx_kv_bits, int):+ return None+ return mlx_kv_bits if mlx_kv_bits in MLX_KV_BITS_CHOICES else None+ bits, turboquant = parse_mlx_kv_quant(mlx_kv_quant)+ # TurboQuant's layout is not one the planner prices; the load fits it at full width too.+ if turboquant or bits not in MLX_KV_BITS_CHOICES:+ return None+ return bits++def _is_embedding_gguf(config: ModelConfig) -> bool:"""Whether this GGUF's pooling type makes llama-server launch with --embedding.@@ -16221,6 +16411,7 @@ def _npu_load_response(resident, status: str) -> LoadResponse:max_context_length = model.max_context_length,native_context_length = model.max_context_length,context_length_enforced = True,+ context_length_fitted = None,supports_reasoning = model.reasoning,reasoning_style = "enable_thinking",supports_tools = model.tools,@@ -18749,17 +18940,16 @@ async def estimate_memory(fastapi_request: Request = None,current_subject: str = Depends(get_current_subject),):- """Price a prospective GGUF load from its header, before anything is allocated.+ """Price a prospective load from its metadata, before the load allocates anything.- Reads only GGUF metadata and file sizes, so it is safe to call on every settings- change: no model is loaded, no device is touched, nothing is downloaded. A model- that is not on this disk answers ``not_downloaded``.+ Reads GGUF metadata and file sizes -- or, for an MLX repo, safetensors headers and the+ shapes of an architecture built and never evaluated -- so it is safe to call on every+ settings change: no model is loaded, no tensor data is read, nothing is downloaded.- The arithmetic is the loader's own KV, compute-buffer and companion sizing, which- is the point. Weights times a constant is fine until the KV cache stops being a- rounding error: at 262k tokens the cache can outweigh the weights, and it swings- fourfold on the cache dtype alone. Where the header cannot supply the dims this- answers ``kv_estimable = false`` rather than quoting an assumed total.+ The arithmetic is the loader's own KV, compute-buffer and companion sizing. Weights+ times a constant is fine until the KV cache stops being a rounding error: at 262k+ tokens the cache can outweigh the weights. Where the metadata cannot supply the dims+ this answers ``kv_estimable = false`` rather than quoting an assumed total."""native_access_deferred = _defers_access_to_native_grant(request)if account_access.managed_account() and not native_access_deferred:@@ -18798,10 +18988,6 @@ async def estimate_memory(requested_slots = _resolve_parallel_slots(request, fastapi_request)def _estimate() -> EstimateMemoryResponse:- resolved_slots = _effective_parallel_slots(- requested_slots,- diffusion_kind = False,- )config = _cached_estimate_config(model_identifier,request.gguf_variant,@@ -18813,12 +18999,58 @@ async def estimate_memory(if config is None:return EstimateMemoryResponse(available = False, reason = "unsizable")if not getattr(config, "is_gguf", False):- # Safetensors / MLX allocate on a different plan; the GGUF arithmetic- # would be a made-up number in a confident box.- return EstimateMemoryResponse(available = False, reason = "not_gguf")+ if not _mlx_estimate_available():+ return EstimateMemoryResponse(available = False, reason = "not_gguf")+ from core.inference.native_audio import is_native_audio_model++ if is_native_audio_model(model_identifier):+ return EstimateMemoryResponse(available = False, reason = "not_gguf")+ if getattr(config, "is_lora", False):+ return EstimateMemoryResponse(available = False, reason = "unsizable")+ model_dir = _local_mlx_model_dir(config)+ if not model_dir:+ return EstimateMemoryResponse(available = False, reason = "not_downloaded")+ from core.inference.mlx_inference import mlx_kv_quant_is_refused+ from core.inference.mlx_memory import mlx_memory_breakdown++ mlx_load_in_4bit = _mlx_estimate_load_in_4bit(config, request)+ mlx_kv_bits = _mlx_estimate_kv_bits(request.mlx_kv_quant, request.mlx_kv_bits)+ # A width the cache takes applies under a pin or a fit alike: the window is then a budget.+ if mlx_kv_bits is not None and mlx_kv_quant_is_refused(model_dir, mlx_kv_bits):+ mlx_kv_bits = None+ mlx_named_ctx = request.max_seq_length or 0+ mlx_fitted_ctx = None+ if not mlx_named_ctx:+ mlx_fitted_ctx = _mlx_estimate_fitted_context(+ config, model_dir, mlx_load_in_4bit, mlx_kv_bits+ )+ mlx_priced_ctx = mlx_named_ctx or mlx_fitted_ctx or _mlx_estimate_ceiling(model_dir)+ if not mlx_priced_ctx:+ return EstimateMemoryResponse(available = False, reason = "unsizable")+ mlx_breakdown = mlx_memory_breakdown(+ model_dir,+ n_ctx = mlx_priced_ctx,+ kv_bits = mlx_kv_bits,+ load_in_4bit = mlx_load_in_4bit,+ )+ if mlx_breakdown is None:+ return EstimateMemoryResponse(available = False, reason = "unsizable")+ return EstimateMemoryResponse(+ **project_estimate_memory_response(+ build_memory_estimate(+ mlx_breakdown,+ quant_file_bytes = 0,+ context_fitted = mlx_fitted_ctx,+ )+ )+ )gguf_path = _local_gguf_main_path(config)if not gguf_path:return EstimateMemoryResponse(available = False, reason = "not_downloaded")+ resolved_slots = _effective_parallel_slots(+ requested_slots,+ diffusion_kind = False,+ )# Price the files on this disk, not the repository they came from.config = _localized_estimate_config(config, gguf_path)@@ -19791,6 +20023,7 @@ async def get_status(current_subject: str):max_context_length = _npu_model.max_context_length,native_context_length = _npu_model.max_context_length,context_length_enforced = True,+ context_length_fitted = None,supports_reasoning = _npu_model.reasoning,reasoning_style = "enable_thinking",supports_tools = _npu_model.tools,@@ -30655,7 +30888,7 @@ def _openai_model_objects() -> list[dict]:):entry["task"] = _TTS_MODEL_TASK- for _field in ("native_context_length", "max_context_length"):+ for _field in ("native_context_length", "max_context_length", "context_length_fitted"):_value = _positive_int_or_none(model_info.get(_field))if _value is not None:entry[_field] = _valuediff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.pyindex 553eda6dfca..f6359e0181d 100644--- a/studio/backend/tests/test_gguf_load_cache_reuse.py+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py@@ -1064,6 +1064,7 @@ class TestLoadHubDownloadExclusion:"chat_template_override_reason",# Constant True: llama.cpp allocates the window it reports."context_length_enforced",+ "context_length_fitted",# Constant False: each slot decodes against its own window."context_unbounded_when_batched",# Read from requested_extra_args, which is what the load was invokeddiff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex 336c51678f8..b99b5154507 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -26,6 +26,7 @@ No GPU, no network, no model load: every GGUF here is a synthetic header on tmp_"""import inspect+import jsonimport osimport sysimport types as _types@@ -911,18 +912,390 @@ class TestEstimateMemoryRoute:assert resp.available is Falseassert resp.reason == "unsupported_source"- def test_non_gguf_model_is_not_priced(self, monkeypatch):- # Safetensors / MLX size their memory through a different allocator;- # quoting the GGUF arithmetic for them is a made-up number in a box.+ def test_the_estimate_asks_the_question_the_worker_asks_of_the_stack(self, monkeypatch):+ import utils.mlx_repair as repair+ monkeypatch.setattr(repair, "is_apple_silicon", lambda: True)+ for blockers, available in (([], True), (["mlx-vlm 0.0.1 is too old"], False)):+ monkeypatch.setattr(repair, "mlx_stack_blockers", lambda b = blockers: b)+ assert ri._mlx_estimate_available() is available++ def test_non_gguf_model_is_not_priced_without_mlx(self, monkeypatch):monkeypatch.setattr(ri,"_cached_estimate_config",lambda *a, **kw: SimpleNamespace(is_gguf = False, identifier = "org/model"),)+ monkeypatch.setattr(ri, "_mlx_estimate_available", lambda: False)resp = _estimate(model_path = "org/model")assert resp.available is Falseassert resp.reason == "not_gguf"+ def test_a_native_audio_load_is_not_priced_as_an_mlx_one(self, monkeypatch, tmp_path):+ import core.inference.native_audio as native_audio++ self._mlx_target(monkeypatch, str(tmp_path))+ priced = []+ monkeypatch.setattr(+ ri, "_mlx_estimate_fitted_context", lambda *a, **kw: priced.append(a) or None+ )+ monkeypatch.setattr(native_audio, "is_native_audio_model", lambda name: name == "org/tts")+ resp = _estimate(model_path = "org/tts")+ assert (resp.available, resp.reason, priced) == (False, "not_gguf", [])+ assert _estimate(model_path = "org/model").reason != "not_gguf"++ @staticmethod+ def _mlx_target(monkeypatch, model_dir, **config):+ """A resolved non-GGUF config on an MLX host, with its weights at *model_dir*."""+ monkeypatch.setattr(+ ri,+ "_cached_estimate_config",+ lambda *a, **kw: SimpleNamespace(is_gguf = False, identifier = "org/model", **config),+ )+ monkeypatch.setattr(ri, "_mlx_estimate_available", lambda: True)+ monkeypatch.setattr(ri, "_local_mlx_model_dir", lambda config: model_dir)++ @staticmethod+ def _record_breakdown(monkeypatch, **fields):+ """Install a breakdown stub, returning the dict its call is recorded into."""+ import core.inference.mlx_memory as mlx_memory++ seen = {}++ def _breakdown(+ model_dir,+ *,+ n_ctx,+ kv_bits = None,+ load_in_4bit = False,+ **kw,+ ):+ seen.update(+ model_dir = model_dir, n_ctx = n_ctx, kv_bits = kv_bits, load_in_4bit = load_in_4bit+ )+ return mlx_memory.MlxMemoryBreakdown(n_ctx = n_ctx, **fields)++ monkeypatch.setattr(mlx_memory, "mlx_memory_breakdown", _breakdown)+ return seen++ def test_mlx_model_not_on_disk_is_not_downloaded(self, monkeypatch):+ self._mlx_target(monkeypatch, None)+ resp = _estimate(model_path = "org/model")+ assert resp.available is False+ assert resp.reason == "not_downloaded"++ def test_mlx_model_whose_cache_cannot_be_read_is_unsizable(self, monkeypatch):+ self._mlx_target(monkeypatch, "/models/thing")+ import core.inference.mlx_memory as mlx_memory++ monkeypatch.setattr(mlx_memory, "mlx_memory_breakdown", lambda *a, **kw: None)+ resp = _estimate(model_path = "org/model")+ assert resp.available is False+ assert resp.reason == "unsizable"+ assert resp.total_bytes == 0++ def test_mlx_model_is_priced_and_itemized(self, monkeypatch, tmp_path):+ self._mlx_target(monkeypatch, "/models/thing")+ seen = self._record_breakdown(+ monkeypatch,+ weights_bytes = 8_000_000_000,+ kv_bytes = 600_000_000,+ compute_bytes = 400_000_000,+ total_bytes = 9_000_000_000,+ gpu_bytes = 9_000_000_000,+ layer_count = 36,+ )+ resp = _estimate(+ model_path = "org/model",+ max_seq_length = 16384,+ mlx_kv_bits = 4,+ cache_type_kv = "q8_0",+ )+ assert (resp.available, resp.reason, resp.kv_estimable) == (True, None, True)+ assert (resp.weights_bytes, resp.kv_bytes, resp.compute_bytes) == (+ 8_000_000_000,+ 600_000_000,+ 400_000_000,+ )+ assert (resp.total_bytes, resp.gpu_bytes, resp.n_ctx, resp.layer_count) == (+ 9_000_000_000,+ 9_000_000_000,+ 16384,+ 36,+ )+ assert seen == {+ "model_dir": "/models/thing",+ "n_ctx": 16384,+ "kv_bits": 4,+ "load_in_4bit": True,+ }+ _estimate(+ model_path = "org/model",+ max_seq_length = 16384,+ mlx_kv_bits = 4,+ load_in_4bit = False,+ )+ assert seen["load_in_4bit"] is False++ import utils.transformers_version as tv++ asked, guarded, real_guard = [], [], ri._offline_guarded+ monkeypatch.setattr(+ tv,+ "latest_tier_active_for",+ lambda name, token = None: asked.append((name, token)) or True,+ )+ monkeypatch.setattr(+ ri,+ "_offline_guarded",+ lambda t, fn, *a, **kw: guarded.append(fn) or real_guard(t, fn, *a, **kw),+ )+ _estimate(model_path = "org/model", max_seq_length = 16384, load_in_4bit = True, hf_token = "tok")+ assert seen["load_in_4bit"] is False+ assert asked == [("org/model", "tok")] and guarded == [tv.latest_tier_active_for]+ monkeypatch.setattr(tv, "latest_tier_active_for", lambda *a, **kw: False)++ monkeypatch.setattr(+ ri,+ "_cached_estimate_config",+ lambda *a, **kw: SimpleNamespace(+ is_gguf = False,+ identifier = "org/model",+ is_lora = True,+ path = str(tmp_path),+ base_model = "org/base",+ ),+ )+ seen.clear()+ resp = _estimate(model_path = "org/model", max_seq_length = 16384, load_in_4bit = True)+ assert (resp.available, resp.reason, resp.total_bytes) == (False, "unsizable", 0)+ assert seen == {}++ def test_a_full_finetune_output_is_priced_at_the_16_bit_it_loads_at(+ self, monkeypatch, tmp_path+ ):+ # The load route resolves a Studio full-finetune output to 16-bit; pricing it 4-bit+ # advertised a fitted window the real load could not hold.+ self._mlx_target(monkeypatch, "/models/thing", is_lora = False, path = str(tmp_path))+ seen = self._record_breakdown(+ monkeypatch,+ weights_bytes = 1,+ kv_bytes = 1,+ compute_bytes = 1,+ total_bytes = 3,+ gpu_bytes = 3,+ layer_count = 2,+ )+ import utils.transformers_version as tv++ monkeypatch.setattr(tv, "latest_tier_active_for", lambda *a, **kw: False)+ monkeypatch.setattr(ri, "is_full_finetune_output", lambda path: path == str(tmp_path))+ _estimate(model_path = "org/model", max_seq_length = 4096, load_in_4bit = True)+ assert seen["load_in_4bit"] is False+ monkeypatch.setattr(ri, "is_full_finetune_output", lambda path: False)+ _estimate(model_path = "org/model", max_seq_length = 4096, load_in_4bit = True)+ assert seen["load_in_4bit"] is True++ def test_the_snapshot_priced_is_the_one_the_load_would_open(self, monkeypatch, tmp_path):+ import utils.models.model_config as mc++ asked, found = [], []+ snaps, ref = tmp_path / "snapshots", tmp_path / "refs" / "main"+ monkeypatch.setattr(ri, "_estimate_hf_cache_roots", lambda: [None])+ monkeypatch.setattr(+ mc, "_iter_hf_cache_snapshots", lambda name, cache_dir = None: asked.append(name) or found+ )+ if ri._mlx_estimate_available():+ for alias in ("unsloth/Qwen3-4B-bnb-4bit", "unsloth/Qwen3-4B-unsloth-bnb-4bit"):+ assert (+ ri._local_mlx_model_dir(+ SimpleNamespace(path = None, is_local = False, identifier = alias)+ )+ is None+ )+ assert asked == ["unsloth/Qwen3-4B", "unsloth/Qwen3-4B"]+ for made in (+ "stub/config.json",+ "new/config.json",+ "new/model.safetensors",+ "old/config.json",+ "old/model.safetensors",+ ):+ (snaps / made).parent.mkdir(parents = True, exist_ok = True)+ (snaps / made).write_text("{}")+ ref.parent.mkdir()+ found[:], new_, old = (+ [snaps / "stub", snaps / "new", snaps / "old"],+ str(snaps / "new"),+ str(snaps / "old"),+ )+ config = SimpleNamespace(path = None, is_local = False, is_lora = False, identifier = "org/model")+ for named, expected in (+ ("old", old),+ ("stub", None),+ ("collected", new_),+ ("../snapshots/old", new_),+ (None, new_),+ ):+ ref.write_text(named) if named else ref.unlink()+ assert ri._local_mlx_model_dir(config) == expected++ def test_a_part_finished_download_is_not_priced_as_a_smaller_model(self, tmp_path):+ def index(weight_map):+ (tmp_path / "model.safetensors.index.json").write_text(+ json.dumps({"weight_map": weight_map})+ )++ (tmp_path / "config.json").write_text("{}")+ config = SimpleNamespace(path = str(tmp_path), is_local = True, is_lora = False)+ five = [f"model-0000{i}-of-00005.safetensors" for i in range(1, 6)]+ for name in five[:2]:+ (tmp_path / name).write_bytes(b"")+ assert ri._local_mlx_model_dir(config) is None+ for name in five[2:]:+ (tmp_path / name).write_bytes(b"")+ assert ri._local_mlx_model_dir(config) == str(tmp_path)+ (tmp_path / "adapter-00001-of-00002.safetensors").write_bytes(b"")+ assert ri._local_mlx_model_dir(config) == str(tmp_path)+ (tmp_path / "config.json").write_text(json.dumps({"vision_config": {}}))+ (tmp_path / "weights").mkdir()+ nested = [f"model-0000{i}-of-00002.safetensors" for i in (1, 2)]+ for name in nested:+ (tmp_path / "weights" / name).write_bytes(b"")+ index({n: f"weights/{n}" for n in nested})+ assert ri._local_mlx_model_dir(config) == str(tmp_path)+ (tmp_path / "weights" / nested[1]).unlink()+ assert ri._local_mlx_model_dir(config) is None++ def test_shard_names_that_count_nothing_are_counted_by_the_index(self, tmp_path):+ (tmp_path / "config.json").write_text("{}")+ config = SimpleNamespace(path = str(tmp_path), is_local = True, is_lora = False)+ (tmp_path / "model.safetensors").write_bytes(b"")+ assert ri._local_mlx_model_dir(config) == str(tmp_path)+ (tmp_path / "model.safetensors").unlink()+ (tmp_path / "weights").mkdir()+ named = [f"model-{i:05d}.safetensors" for i in range(1, 45)] + [+ "mtp.safetensors",+ "weights/extra.safetensors",+ ]+ (tmp_path / "model.safetensors.index.json").write_text(+ json.dumps({"weight_map": {n: n for n in named}})+ )+ (tmp_path / named[0]).write_bytes(b"")+ assert ri._local_mlx_model_dir(config) is None+ for name in named[1:]:+ (tmp_path / name).write_bytes(b"")+ assert ri._local_mlx_model_dir(config) == str(tmp_path)+ (tmp_path / "model.safetensors.index.json").write_text(+ json.dumps(+ {+ "weight_map": {+ str(i): f"model-{i:05d}-of-00099.safetensors" for i in range(1, 100)+ }+ }+ )+ )+ assert ri._local_mlx_model_dir(config) == str(tmp_path)++ def test_an_mlx_estimate_prices_max_seq_length_not_n_ctx(self, monkeypatch):+ self._mlx_target(monkeypatch, "/models/thing")+ seen = self._record_breakdown(+ monkeypatch,+ weights_bytes = 1,+ kv_bytes = 1,+ compute_bytes = 1,+ total_bytes = 3,+ gpu_bytes = 3,+ )+ _estimate(model_path = "org/model", max_seq_length = 8192, n_ctx = 32768)+ assert seen["n_ctx"] == 8192++ def test_an_unpinned_mlx_estimate_prices_the_fit_and_reports_it(self, monkeypatch, tmp_path):+ from core.inference import mlx_inference+ from core.inference.runtime_context import MAX_REQUESTABLE_CONTEXT++ write = (tmp_path / "config.json").write_text+ write(json.dumps({"max_position_embeddings": 262_144}))+ self._mlx_target(monkeypatch, str(tmp_path))+ asked, fit = {}, {"answer": 24_576}+ monkeypatch.setattr(+ mlx_inference,+ "mlx_fit_to_memory",+ lambda model_dir, ceiling, **kw: (+ asked.update(kw, ceiling = ceiling, dir = model_dir) or fit["answer"]+ ),+ )+ seen = self._record_breakdown(+ monkeypatch,+ weights_bytes = 1,+ kv_bytes = 1,+ compute_bytes = 1,+ total_bytes = 3,+ gpu_bytes = 3,+ )+ widths, refuses = [], {"answer": False}+ monkeypatch.setattr(+ mlx_inference,+ "mlx_kv_quant_is_refused",+ lambda *a: widths.append(a) or refuses["answer"],+ )++ # The width the cache takes is priced into the fit, and the fitted window is a budget.+ resp = _estimate(model_path = "org/model", n_ctx = 32_768, mlx_kv_bits = 4)+ assert (seen["n_ctx"], seen["kv_bits"], resp.context_fitted) == (24_576, 4, 24_576)+ assert widths == [(str(tmp_path), 4)]+ assert asked == {+ "ceiling": 262_144,+ "retains_history": True,+ "dir": str(tmp_path),+ "kv_bits": 4,+ "load_in_4bit": seen["load_in_4bit"],+ }++ # /load's own vocabulary prices the same width; TurboQuant is priced at full width,+ # the width the load fits it at, and never asks the mx.quantize probe.+ widths.clear()+ resp = _estimate(model_path = "org/model", n_ctx = 32_768, mlx_kv_quant = "4")+ assert (seen["kv_bits"], resp.context_fitted, widths) == (4, 24_576, [(str(tmp_path), 4)])+ widths.clear()+ resp = _estimate(model_path = "org/model", mlx_kv_quant = "tq-4", mlx_kv_bits = 4)+ assert (asked["kv_bits"], seen["kv_bits"], widths) == (None, None, [])++ # A width the load would refuse is priced at full width, fitted or pinned.+ refuses["answer"] = True+ resp = _estimate(model_path = "org/model", mlx_kv_bits = 4)+ assert (asked["kv_bits"], seen["kv_bits"], resp.context_fitted) == (None, None, 24_576)+ resp = _estimate(model_path = "org/model", max_seq_length = 8192, mlx_kv_bits = 4)+ assert (seen["n_ctx"], seen["kv_bits"], resp.context_fitted) == (8192, None, None)++ # A pin never displaces an accepted width; it is not fitted either.+ refuses["answer"] = False+ asked.clear()+ resp = _estimate(model_path = "org/model", max_seq_length = 8192, mlx_kv_bits = 4)+ assert (seen["n_ctx"], seen["kv_bits"], resp.context_fitted, asked) == (8192, 4, None, {})++ widths.clear()+ _estimate(model_path = "org/model")+ assert (widths, asked["kv_bits"], seen["kv_bits"]) == ([], None, None)++ fit["answer"] = None+ write(json.dumps({"max_position_embeddings": MAX_REQUESTABLE_CONTEXT * 4}))+ resp = _estimate(model_path = "org/model", mlx_kv_bits = 4)+ assert resp.context_fitted is None+ assert (seen["n_ctx"], seen["kv_bits"]) == (MAX_REQUESTABLE_CONTEXT, 4)+ write(json.dumps({"model_type": "llama"}))+ seen.clear()+ assert _estimate(model_path = "org/model").reason == "unsizable"+ assert seen == {}++ write(json.dumps({"max_position_embeddings": 262_144}))+ self._mlx_target(monkeypatch, str(tmp_path), is_vision = True)+ for available in (True, False):+ monkeypatch.setattr(+ mlx_inference, "mlx_vlm_snapshot_store_available", lambda a = available: a+ )+ _estimate(model_path = "org/model")+ assert asked["retains_history"] is available+def test_gguf_not_on_disk_is_not_downloaded(self, monkeypatch):# No header to read and no reaching for the network on a slider drag.monkeypatch.setattr(@@ -3308,3 +3681,1091 @@ class TestFourMoreLaunchNormalizations:)out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 8192, n_parallel = 4)assert out.n_parallel == 4+++class TestMlxEstimateKvBits:+ """Which requested KV widths reach the MLX planner, and which are dropped."""++ @pytest.mark.parametrize(+ "quant, bits, expected",+ [+ (None, 4, 4),+ (None, 8, 8),+ (None, 7, None),+ (None, 0, None),+ (None, None, None),+ (None, True, None),+ ("4", None, 4),+ (" 8 ", 4, 8),+ ("auto", 8, None),+ ("tq-4", 4, None),+ ("bogus", 4, None),+ ],+ )+ def test_kv_bits_resolution(self, quant, bits, expected):+ # An explicit mlx_kv_quant wins over the superseded width, as on /load.+ assert ri._mlx_estimate_kv_bits(quant, bits) == expected+++import glob # noqa: E402+from dataclasses import replace # noqa: E402++from core.inference import mlx_memory as mm # noqa: E402++[email protected](autouse = True)+def _fresh_sizing_cache():+ # Tests re-price one checkpoint under different patched internals.+ from core.inference import mlx_inference++ mm._clear_sizing_cache()+ mlx_inference._kv_refusal_cache.clear()+ yield+ mm._clear_sizing_cache()+ mlx_inference._kv_refusal_cache.clear()+++def test_a_load_is_sized_once_until_its_files_change(monkeypatch, tmp_path):+ (tmp_path / "config.json").write_text("{}")+ shard = tmp_path / "model.safetensors"+ shard.write_bytes(b"x")+ built = []+ monkeypatch.setattr(+ mm, "_size_load_uncached", lambda *a: built.append(a) or f"sizing{len(built)}"+ )+ first = mm._size_load(str(tmp_path), 4, None, True)+ assert mm._size_load(str(tmp_path), 4, None, True) is first and len(built) == 1+ assert mm._size_load(str(tmp_path), None, None, True) != first and len(built) == 2+ shard.write_bytes(b"xy")+ assert mm._size_load(str(tmp_path), 4, None, True) != first and len(built) == 3+++try:+ import importlib++ importlib.import_module("mlx.core")+ importlib.import_module("mlx_lm")+ _HAVE_MLX = True+except Exception:+ _HAVE_MLX = False++_NEEDS_MLX = pytest.mark.skipif(not _HAVE_MLX, reason = "needs a working MLX install")+++def _mlx_raise():+ raise NotImplementedError+++def _plan_entry(**over):+ """One cache plan entry: a growing key/value cache unless overridden."""+ return {+ "const": 0.0,+ "slope": 4096.0,+ "quant_const": 0.0,+ "quant_slope": 1152.0,+ "bound_spec": None,+ "conv_width": 0,+ "block": 256,+ "converts": True,+ **over,+ }+++_GROWING = _plan_entry()+_RECURRENT = _plan_entry(+ const = 2_195_456.0,+ slope = 0.0,+ quant_const = 2_195_456.0,+ quant_slope = 0.0,+ conv_width = 8192,+ block = 1,+ converts = False,+)+_UNQUANTIZED = {"quant_start": None, "prefill_chunk": mm.MLX_PREFILL_CHUNK}+++def _caption(+ plan,+ bits = None,+ converted = False,+ n_ctx = 32768,+):+ return mm._cache_width_name(plan, bits, converted, "bf16", n_ctx, mm.MLX_PREFILL_CHUNK)+++def _local_snapshot(repo):+ """This repo's snapshot directory, skipping the test when it is not here."""+ hub = os.path.expanduser("~/.cache/huggingface/hub")+ hits = glob.glob(+ os.path.join(hub, "models--" + repo.replace("/", "--"), "snapshots", "*", "config.json")+ )+ if not hits:+ pytest.skip(f"{repo} not on this disk")+ return os.path.dirname(hits[0])+++def _on_bfloat16_chip():+ import mlx.core as mx+ if mm._runtime_dtype() is not mx.bfloat16:+ pytest.skip("measured on a chip the loader gives bfloat16")+++class TestCacheWidthName:+ def test_only_the_entries_that_grow_name_the_cache(self):+ hybrid = [_RECURRENT] * 30 + [_GROWING] * 10+ assert _caption(hybrid) == "bf16"+ assert _caption(hybrid, 4, True) == "4-bit"+ assert _caption([_RECURRENT]) == "bf16"++ def test_a_converted_run_is_named_as_its_own_control_is_labelled(self):+ assert [_caption([_GROWING] * 10, b, True) for b in (8, 6, 5, 4, 3, 2)] == [+ "8-bit",+ "6-bit",+ "5-bit",+ "4-bit",+ "3-bit",+ "2-bit",+ ]+ assert _caption([_GROWING] * 10, 4, False) == "bf16"++ def test_an_entry_that_cannot_convert_keeps_its_width_through_a_converting_run(self):+ staying = _plan_entry(converts = False, quant_slope = _GROWING["slope"])+ assert _caption([staying] * 57 + [_GROWING] * 3, 4, True) == "bf16/4-bit"+ heavy, light = (+ _plan_entry(quant_slope = 100_000.0),+ _plan_entry(converts = False, slope = 1.0, quant_slope = 1.0),+ )+ assert _caption([light] * 50 + [heavy] * 2, 4, True) == "4-bit/bf16"++ @_NEEDS_MLX+ def test_a_bounded_entry_is_weighed_by_what_it_holds_and_not_by_its_slope(self):+ from mlx_lm.models.cache import ChunkedKVCache++ bounded = _plan_entry(converts = False, bound_spec = (ChunkedKVCache, "chunk_size", 8192))+ plan = [bounded] * 3 + [_plan_entry(quant_slope = 1152.0)]+ assert _caption(plan, 4, True, 8192).startswith("bf16")+ assert _caption(plan, 4, True, 131_072).startswith("4-bit")++ @_NEEDS_MLX+ def test_every_width_a_cache_is_built_at_has_a_caption(self):+ import mlx.core as mx+ assert [mm._width_name(d) for d in (mx.bfloat16, mx.float16, mx.float32)] == [+ "bf16",+ "f16",+ "f32",+ ]+++class TestKvBytes:+ def test_a_hybrid_plan_is_far_cheaper_than_charging_every_layer_as_attention(self):+ hybrid = [_RECURRENT] * 30 + [_GROWING] * 10+ assert (+ mm._kv_bytes(hybrid, 131_072, **_UNQUANTIZED)[0] * 3+ < mm._kv_bytes([_GROWING] * 40, 131_072, **_UNQUANTIZED)[0]+ )++ def test_a_runtime_that_declines_to_chunk_is_recognised_on_either_half(self):+ assert mm._declines_to_chunk(type("M", (), {"no_chunked_prefill": True}), object())+ assert mm._declines_to_chunk(object, type("T", (), {"no_chunked_prefill": True})())+ assert not mm._declines_to_chunk(object, object())++ def test_a_cache_converted_before_prefill_is_never_charged_at_full_width(self):+ plan = [_GROWING, _RECURRENT]+ quantized = mm._line_bytes(plan, 32768, 2048, quantized = True)+ assert mm._kv_bytes(plan, 32768, 0, 2048) == (quantized, 0)+ assert mm._kv_bytes(plan, 2048, 0, 2048)[0] == mm._line_bytes(plan, 2048, 2048, True)++ def test_one_failed_conversion_refuses_the_whole_request(self):+ def _entry(converts):+ return type(+ "Entry",+ (),+ {+ "max_size": None,+ "window_size": None,+ "to_quantized": lambda self, **kw: "converted" if converts else _mlx_raise(),+ },+ )()++ assert mm._quantize_like_runtime([_entry(True), _entry(False)], 4) is None++ def test_a_windowed_entry_keeps_its_width_while_the_rest_convert(self):+ from core.inference.mlx_inference import MLX_KV_GROUP_SIZE++ def _entry(**window):+ attrs = {"max_size": None, "window_size": None, **window}+ attrs["to_quantized"] = lambda self, **kw: ("converted", kw)+ return type("Entry", (), attrs)()++ full, ring, window = _entry(), _entry(max_size = 512), _entry(window_size = 512)+ quantized = ("converted", {"group_size": MLX_KV_GROUP_SIZE, "bits": 4})+ assert mm._quantize_like_runtime([full, ring], 4) == [quantized, ring]+ # Past two entries the last full-attention one stays float, as the load keeps it.+ last = _entry()+ assert mm._quantize_like_runtime([full, last, ring, window], 4) == [+ quantized,+ last,+ ring,+ window,+ ]+ assert mm._quantize_like_runtime([full, ring, window], 4) is None+ assert mm._quantize_like_runtime([ring, window], 4) is None+ assert mm._quantize_like_runtime([full], None) is None++ @pytest.mark.skipif(not _HAVE_MLX, reason = "drives a real cache class")+ def test_a_bounded_window_leaves_the_request_full_width(self):+ from mlx_vlm.models.unlimited_ocr.language import RingSlidingKVCache++ entry = RingSlidingKVCache(512)+ assert entry.to_quantized(group_size = 64, bits = 4) is not None+ assert mm._quantize_like_runtime([entry], 4) is None+++class TestComputeBytes:+ def test_an_unreadable_width_is_not_guessed(self):+ assert mm._compute_bytes(mm._config_widths({}), 2, 2048, [_GROWING], 8192, None) == 0++ def test_the_calibrated_term_matches_the_measurement_it_came_from(self):+ measured = 610 * 1024**2+ priced = mm._compute_bytes(+ (1024, 3072, 16), 2, mm.MLX_PREFILL_CHUNK, [_GROWING] * 28, 8192, None+ )+ assert 0.7 * measured < priced < 1.5 * measured, (+ f"prices {priced / 1024**2:.0f} MiB where the load measured "+ f"{measured / 1024**2:.0f} MiB"+ )+++@_NEEDS_MLX+class TestTheQuantizedAttentionRouteIsTheOneZooTakes:+ """unsloth-zoo routes each call between a score matrix and a dequantized copy of the cache."""++ GQA = (3584, 18944, 32, 8, 128, 0)+ MHA = (3584, 18944, 32, 32, 128, 0)+ WIDE = (3584, 18944, 16, 2, 256, 0)++ @pytest.mark.parametrize(+ "widths, chunk, copies",+ [+ (GQA, 2048, True),+ (GQA, 256, True),+ (MHA, 2048, True),+ (MHA, 256, False),+ (WIDE, 2048, False),+ (WIDE, 256, False),+ ],+ )+ def test_both_halves_of_the_wrappers_choice_are_asked(self, widths, chunk, copies):+ assert (mm._dequantized_row_bytes(widths, 2, 4, 64, chunk) is not None) is copies++ def test_the_copy_charged_is_the_cache_the_wrapper_would_make(self):+ import mlx.core as mx++ from unsloth_zoo.mlx.attention import _row_bytes++ cache = mx.quantize(mx.zeros((1, 8, 1, 128), dtype = mx.bfloat16), group_size = 64, bits = 4)+ assert mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048) == (+ 2 * 8 * _row_bytes(cache, 64, mx.bfloat16)+ )++ def test_a_copied_route_is_charged_the_copy_and_not_the_scores(self):+ row = mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048)+ floor = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, None)+ priced = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, 2048, 4, 64)+ scores = mm._widest_quantized_scores(32768, 2048, 2048) * 32 * mm._QUANT_SCORE_DTYPE_SIZE+ assert priced - floor == row * 32768 < scores++ def test_the_decode_step_still_scores_where_that_is_the_wider_of_the_two(self):+ widths = (3584, 18944, 128, 1, 64, 0)+ row = mm._dequantized_row_bytes(widths, 2, 8, 64, 2048)+ floor = mm._compute_bytes(widths, 2, 2048, [], 32768, None)+ charged = mm._compute_bytes(widths, 2, 2048, [], 32768, 2048, 8, 64) - floor+ assert row is not None and charged == (32769) * 128 * mm._QUANT_SCORE_DTYPE_SIZE+ assert charged > row * 32768++ def test_a_zoo_that_cannot_route_it_keeps_the_scores_whole(self, monkeypatch):+ import builtins++ real = builtins.__import__++ def _no_wrapper(name, *a, **kw):+ if name == "unsloth_zoo.mlx.attention":+ raise ImportError("this zoo predates the wrapper")+ return real(name, *a, **kw)++ monkeypatch.setattr(builtins, "__import__", _no_wrapper)+ assert mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048) is None+ floor = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, None)+ priced = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, 2048, 4, 64)+ assert priced - floor == (+ mm._widest_quantized_scores(32768, 2048, 2048) * 32 * mm._QUANT_SCORE_DTYPE_SIZE+ )++ @pytest.mark.parametrize(+ "stated, expected",+ [+ ({"num_key_value_heads": 8, "head_dim": 128}, (8, 128, 0)),+ ({}, (32, 112, 0)),+ ({"num_key_value_heads": 4}, (4, 112, 0)),+ ({"head_dim": 64}, (32, 64, 0)),+ ({"head_dim": 192, "v_head_dim": 128}, (32, 192, 128)),+ ],+ )+ def test_the_geometry_the_route_reads_comes_off_the_checkpoint(self, stated, expected):+ from types import SimpleNamespace++ config = {"hidden_size": 3584, "num_attention_heads": 32, **stated}+ assert mm._config_widths(config)[3:] == expected+ assert mm._tower_widths(SimpleNamespace(args = SimpleNamespace(**config)))[3:] == expected++ def test_a_tower_whose_values_are_narrower_keeps_the_scores(self):+ assert mm._dequantized_row_bytes((3584, 18944, 32, 8, 128, 64), 2, 4, 64, 2048) is None+ assert mm._dequantized_row_bytes((3584, 18944, 32, 8, 192, 128), 2, 4, 64, 2048) is None+ assert mm._dequantized_row_bytes((3584, 18944, 32, 8, 128, 128), 2, 4, 64, 2048) == (+ mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048)+ )++ def test_a_tower_that_states_nothing_is_still_priced_from_its_checkpoint(self):+ assert len(mm._tower_widths(object())) == len(mm._config_widths({}))+ config = {+ "hidden_size": 3584,+ "intermediate_size": 18944,+ "num_attention_heads": 32,+ "num_key_value_heads": 8,+ "head_dim": 128,+ }+ merged = tuple(+ tower or checkpoint+ for tower, checkpoint in zip(mm._tower_widths(object()), mm._config_widths(config))+ )+ assert mm._dequantized_row_bytes(merged, 2, 4, 64, 2048) == (+ mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048)+ )++ def test_an_unquantized_load_never_reaches_the_route(self):+ assert mm._dequantized_row_bytes(self.GQA, 2, None, 64, 2048) is None+ assert mm._dequantized_row_bytes((3584, 18944, 32, 0, 0, 0), 2, 4, 64, 2048) is None++[email protected](not _HAVE_MLX, reason = "imports architecture modules")+class TestProbeFollowsTheLoadersRoute:+ """The estimator must not price a load path that cannot be taken."""++ def test_a_vision_model_is_never_priced_from_mlx_lms_copy_of_it(self):+ import importlib++ config = {"model_type": "kimi_vl", "vision_config": {"depth": 2}}+ assert importlib.import_module("mlx_lm.models.kimi_vl")+ assert {+ model_class.__module__.split(".")[0]+ for _, _, model_class in mm._probe_models(config, None)+ } == {"mlx_vlm"}++ @pytest.mark.parametrize(+ "architecture, resolved",+ [("DeepseekOCRForCausalLM", "deepseekocr"), ("DeepseekOCR2ForCausalLM", "deepseekocr_2")],+ )+ def test_a_config_is_read_as_the_loader_resolves_it_not_as_stated(+ self, tmp_path, architecture, resolved+ ):+ (tmp_path / "config.json").write_text(+ json.dumps({"model_type": "deepseek_vl_v2", "architectures": [architecture]})+ )+ assert mm._snapshot_config(str(tmp_path))["model_type"] == resolved++ def test_a_checkpoint_supplying_its_own_module_is_refused_and_not_imported(self):+ with pytest.raises(ValueError, match = "its own model module"):+ list(mm._probe_models({"model_type": "qwen3", "model_file": "modeling.py"}, None))++ def test_the_probe_hands_the_forward_pass_a_model_in_eval_mode(self):+ import mlx.core as mx+ config = json.loads(+ (Path(_local_snapshot("unsloth/Qwen3-4B-Thinking-2507")) / "config.json").read_text()+ )+ assert next(iter(mm._probe_models(config, mx.bfloat16)))[0]().training is False++ @pytest.mark.parametrize(+ "chip, expected", [("Apple M1 Max", "float16"), ("Apple M3 Max", "bfloat16")]+ )+ def test_the_chip_decides_the_width_the_loader_installs(self, monkeypatch, chip, expected):+ import mlx.core as mx+ monkeypatch.setattr(mx, "device_info", lambda: {"device_name": chip})+ assert mm._runtime_dtype() is getattr(mx, expected)++[email protected](not _HAVE_MLX, reason = "asks the installed runtime")+class TestGenerationSettingsComeFromTheLoader:+ """What a load prefills at is read off the runtime, except where Studio pins it."""++ def test_each_path_is_read_from_the_function_that_would_run_it(self, monkeypatch):+ import inspect++ from core.inference import mlx_inference as mi++ monkeypatch.setattr(mi, "mlx_vlm_snapshot_store_available", lambda: False)+ for vision, drafted in ((False, False), (True, False), (False, True), (True, True)):+ step = mi._generation_step(vision = vision, drafted = drafted)+ assert mi.mlx_prefill_chunk(vision = vision, drafted = drafted) == (+ inspect.signature(step).parameters["prefill_step_size"].default+ )+ assert mi._generation_step(vision = True, drafted = False).__module__.startswith("mlx_vlm")+ assert mi._generation_step(vision = False, drafted = False).__module__.startswith("mlx_lm")+ assert mi.mlx_prefill_chunk(drafted = True) != mi.mlx_prefill_chunk()+ assert mi.mlx_prefill_chunk(vision = True, drafted = True) == mi.mlx_prefill_chunk(vision = True)++ def test_a_pinned_vision_step_outranks_the_runtimes_own(self, monkeypatch):+ from core.inference import mlx_inference as mi++ vlm_default = mi._generation_default("prefill_step_size", 0, vision = True, drafted = False)+ monkeypatch.setattr(mi, "vlm_prefill_step", lambda: vlm_default + 256)+ media, bare = {"image_token_id": 151655}, {"model_type": "vlm"}+ for store, config, expected in (+ (True, media, vlm_default + 256),+ (True, bare, vlm_default),+ (False, media, vlm_default),+ (True, None, vlm_default),+ ):+ monkeypatch.setattr(mi, "mlx_vlm_snapshot_store_available", lambda p = store: p)+ assert mi.mlx_prefill_chunk(vision = True, config = config) == expected+ assert mi.mlx_prefill_chunk(config = media) == mi.MLX_PREFILL_CHUNK_FALLBACK++ def test_the_step_priced_and_the_store_that_pins_it_answer_together(self, monkeypatch):+ """The estimate is only right while these two read the same capability."""+ import importlib++ from core.inference.mlx_inference import (+ MLXInferenceBackend,+ mlx_vlm_snapshot_store_available,+ )++ vlm_generate = importlib.import_module("mlx_vlm.generate")+ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)+ backend._memory_limits_applied = {}+ for cached_tokens, built in ((True, True), (False, False)):+ backend._vlm_snapshot_store = None+ backend._vlm_snapshot_store_unavailable = False+ monkeypatch.setattr(+ vlm_generate,+ "GenerationResult",+ type("R", (), {"cached_tokens": 0} if cached_tokens else {}),+ )+ assert mlx_vlm_snapshot_store_available() is built+ assert (backend._vlm_prompt_cache_store() is not None) is built++ @pytest.mark.parametrize("chunk", [None, 0, True, "2048"])+ def test_a_runtime_that_states_no_usable_value_falls_back(self, monkeypatch, chunk):+ from core.inference import mlx_inference as mi++ def _stated(**kw):+ return lambda prefill_step_size = chunk: None++ monkeypatch.setattr(mi, "_generation_step", _stated)+ assert mi.mlx_prefill_chunk() == mi.MLX_PREFILL_CHUNK_FALLBACK++ def test_the_estimator_tells_the_loader_which_package_would_load_it(self, monkeypatch):+ from core.inference import mlx_inference as mi++ seen = []+ monkeypatch.setattr(mi, "mlx_prefill_chunk", lambda **kw: seen.append(kw["vision"]) or 2048)+ vision = mm._generation_settings({"model_type": "kimi_vl", "vision_config": {"depth": 2}})+ assert mm._generation_settings({"model_type": "qwen3"}) == vision == (2048, 64)+ assert seen == [True, False]++ def test_every_loader_quantizes_a_converted_cache_from_the_first_token(self, monkeypatch):+ entry = {+ "bytes": 1,+ "quant_bytes": 1,+ "bound_spec": None,+ "conv_width": 0,+ "block": 1,+ "converts": True,+ }+ probe = {+ "entries": [entry],+ "quantized": True,+ "whole_prompt": False,+ "layers": 1,+ "widths": (1, 1, 1),+ }+ monkeypatch.setattr(mm, "_probe", lambda *a, **kw: probe)+ for vision in (True, False):+ monkeypatch.setattr(mm, "_loads_as_vision", lambda _c, v = vision: v)+ assert mm._cache_plan({}, None, 4)[1] == 0+ probe["quantized"] = False+ assert mm._cache_plan({}, None, 4)[1] is None++ def test_a_host_without_the_loader_prices_the_fallback(self, monkeypatch):+ import builtins++ real = builtins.__import__++ def _no_loader(name, *a, **kw):+ if name == "core.inference.mlx_inference":+ raise ImportError("no loader here")+ return real(name, *a, **kw)++ monkeypatch.setattr(builtins, "__import__", _no_loader)+ assert mm._generation_settings({"model_type": "llama"}) == (mm.MLX_PREFILL_CHUNK, 64)+++@_NEEDS_MLX[email protected]("reported, compute", [(None, 1_035_321_999), (512, 903_201_423)])+def test_the_estimate_prices_the_chunk_the_loader_reports(monkeypatch, reported, compute):+ _on_bfloat16_chip()+ if reported is not None:+ from core.inference import mlx_inference as mi+ monkeypatch.setattr(mi, "mlx_prefill_chunk", lambda **kw: reported)+ breakdown = mm.mlx_memory_breakdown(+ _local_snapshot("unsloth/Qwen3-4B-Thinking-2507"),+ n_ctx = 32768,+ load_in_4bit = True,+ kv_bits = 4,+ )+ assert breakdown is not None and breakdown.compute_bytes == compute+ override = mm.mlx_memory_breakdown(+ _local_snapshot("unsloth/Qwen3-4B-Thinking-2507"),+ n_ctx = 32768,+ load_in_4bit = True,+ kv_bits = 4,+ prefill_chunk = 512,+ )+ assert override.compute_bytes == 903_201_423+++@_NEEDS_MLX+def test_a_model_mlx_vlm_would_diffuse_is_refused_rather_than_priced():+ snapshot = _local_snapshot("mlx-community/diffusiongemma-26B-A4B-it-4bit")+ assert mm._routes_to_diffusion(mm._snapshot_config(snapshot)) is True+ assert mm.mlx_memory_breakdown(snapshot, n_ctx = 32768, load_in_4bit = True) is None+++@_NEEDS_MLX+def test_an_ordinary_vision_model_is_not_mistaken_for_a_diffusion_one():+ config = {"model_type": "kimi_vl", "vision_config": {"depth": 2}}+ assert mm._routes_to_diffusion(config) is False+ assert mm._generation_settings(config)[0] > 0+++@_NEEDS_MLX+class TestDiffusionRouting:+ """The markers gate a build; mlx-vlm's own predicate is the verdict."""++ @staticmethod+ def _resolving(+ monkeypatch,+ *,+ canvas,+ mask,+ build = lambda config: object(),+ ):+ monkeypatch.setattr(+ mm,+ "_loader_config",+ lambda arch, config: SimpleNamespace(canvas_length = canvas, mask_token_id = mask),+ )+ monkeypatch.setattr(+ "mlx_vlm.utils.get_model_and_args",+ lambda config: (SimpleNamespace(Model = build), None),+ )++ @pytest.mark.parametrize("verdict", [True, False])+ def test_the_predicate_outranks_the_marker(self, monkeypatch, verdict):+ from mlx_vlm.generate import diffusion as vlm_diffusion++ self._resolving(monkeypatch, canvas = None, mask = 100)+ monkeypatch.setattr(vlm_diffusion, "is_diffusion_model", lambda model, kw: verdict)+ assert mm._routes_to_diffusion({"model_type": "whatever"}) is verdict++ def test_a_diverted_load_refuses_without_needing_a_checkpoint(self, monkeypatch):+ from mlx_vlm.generate import diffusion as vlm_diffusion++ self._resolving(monkeypatch, canvas = 512, mask = None)+ monkeypatch.setattr(vlm_diffusion, "is_diffusion_model", lambda model, kw: True)+ monkeypatch.setattr(mm, "_loads_as_vision", lambda config: True)+ with pytest.raises(ValueError, match = "diffusion generator"):+ mm._generation_settings({"model_type": "whatever"})++ def test_an_architecture_carrying_no_marker_is_never_built(self, monkeypatch):+ def _explode(config):+ raise AssertionError("built a wrapper for a config carrying no marker")++ self._resolving(monkeypatch, canvas = None, mask = None, build = _explode)+ assert mm._routes_to_diffusion({"model_type": "whatever"}) is False++ def test_a_marked_model_that_cannot_be_placed_is_refused_not_assumed(self, monkeypatch):+ def _explode(config):+ raise RuntimeError("this wrapper cannot be built")++ self._resolving(monkeypatch, canvas = 512, mask = None, build = _explode)+ with pytest.raises(RuntimeError):+ mm._routes_to_diffusion({"model_type": "whatever"})+ with pytest.raises(RuntimeError):+ mm._generation_settings({"model_type": "whatever", "vision_config": {"depth": 2}})++[email protected](not _HAVE_MLX, reason = "drives real cache classes")+def test_the_peak_of_a_bounded_cache_is_measured_not_derived():+ from mlx_lm.models import cache as C++ for bound, attribute, value, n_ctx, chunk, expected in (+ (C.RotatingKVCache, "max_size", 512, 100, 2048, 355),+ (C.RotatingKVCache, "max_size", 512, 255, 2048, 510),+ (C.RotatingKVCache, "max_size", 512, 1024, 2048, 1023),+ (C.ChunkedKVCache, "chunk_size", 512, 4096, 2048, 2560),+ (C.ChunkedKVCache, "chunk_size", 512, 2560, 2048, 2048),+ (C.RotatingKVCache, "max_size", 512, 256, 1, 512),+ (C.RotatingKVCache, "max_size", 512, 10**12, 2048, 2559),+ ):+ assert (+ mm._bounded_peak(bound, attribute, value, n_ctx, chunk) == expected+ ), f"{bound.__name__}({value}) at {n_ctx}"+ assert mm._bound_spec(type("E", (), {"max_size": 512})())+ assert mm._bound_spec(type("E", (), {"chunk_size": 512})())+ spec = (C.RotatingKVCache, "max_size", 512)+ entry = {"slope": 1.0, "block": mm.MLX_KV_BLOCK, "bound_spec": spec}+ for n_ctx, charged in ((253, 508), (100, 355), (1024, 1023), (10**12, 2559)):+ assert mm._held_tokens(entry, n_ctx, 2048) == charged++[email protected](not _HAVE_MLX, reason = "builds a real architecture")+class TestPricingALoad:+ _CONFIG = {+ "model_type": "llama",+ "hidden_size": 256,+ "num_hidden_layers": 2,+ "intermediate_size": 512,+ "num_attention_heads": 4,+ "num_key_value_heads": 4,+ "rms_norm_eps": 1e-5,+ "vocab_size": 512,+ "rope_theta": 10000.0,+ }++ def _checkpoint(+ self,+ tmp_path,+ *,+ prefix = "",+ ):+ import mlx.core as mx+ from mlx.utils import tree_flatten++ config = dict(self._CONFIG, tie_word_embeddings = True)+ (tmp_path / "config.json").write_text(json.dumps(config))+ built = mm._whole_model(config, mx.bfloat16)+ mx.save_safetensors(+ str(tmp_path / "model.safetensors"),+ {+ prefix + name: mx.zeros(value.shape, dtype = mx.bfloat16)+ for name, value in tree_flatten(built.parameters())+ },+ )+ return config++ def test_a_float32_checkpoint_goes_resident_at_the_width_the_chip_gives_it(self):+ if mm._runtime_dtype().size != 2:+ pytest.skip("measured on a chip the loader gives a 16-bit width")+ snapshot = _local_snapshot("hf-internal-testing/tiny-random-LlamaForCausalLM")+ config = json.loads((Path(snapshot) / "config.json").read_text())+ assert mm.mlx_weight_bytes(snapshot, config, load_in_4bit = False) == 2_064_544++ def test_a_quantizing_load_is_priced_below_the_shards(self, tmp_path):+ config = self._checkpoint(tmp_path)+ wide = mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = False)+ quantized = mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = True)+ assert (wide, quantized) == (2_886_144, 1_001_984)+ assert quantized < wide < mm._shard_bytes(str(tmp_path), config) == 2_888_265++ def test_shards_that_name_nothing_the_architecture_has_are_not_priced_from_it(self, tmp_path):+ config = self._checkpoint(tmp_path, prefix = "draft.")+ with pytest.raises(ValueError, match = "does not supply"):+ mm._resident_bytes(str(tmp_path), config, quantize = True)+ for flag in (True, False):+ assert mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = flag) == (+ mm._shard_bytes(str(tmp_path), config)+ )++[email protected](not _HAVE_MLX, reason = "asks the installed loader")+class TestALoadWithNoFootprintToQuote:+ def test_quantization_metadata_no_load_can_honour_is_refused_not_priced(self, tmp_path):+ per_module = {"model.layers.0.self_attn.q_proj": {"bits": 4, "group_size": 64}}+ for spelling in ("quantization", "quantization_config"):+ stated = {"model_type": "llama", spelling: per_module}+ assert mm._load_is_refused(str(tmp_path), stated, True)+ assert mm._load_is_refused(str(tmp_path), stated, False) is None+ bnb = {"model_type": "llama", spelling: {"quant_method": "bitsandbytes"}}+ for flag in (True, False):+ assert "bitsandbytes" in (mm._load_is_refused(str(tmp_path), bnb, flag) or "")++ @pytest.mark.parametrize(+ "repo, names",+ [+ ("OsaurusAI/ZAYA1-VL-8B-MXFP4", "no home for"),+ ("Qwen/Qwen3-Embedding-0.6B", "QK-norm"),+ ("z-lab/Qwen3.6-27B-DFlash", "QK-norm"),+ ("z-lab/gemma-4-31B-it-DFlash", "QK-norm"),+ ("yuyijiong/Qwen3.5-4B-Eagle3", "no home for"),+ ],+ )+ def test_a_checkpoint_no_load_starts_from_is_refused(self, repo, names):+ snapshot = _local_snapshot(repo)+ config = mm._snapshot_config(snapshot)+ for flag in (True, False):+ assert names in (mm._load_is_refused(snapshot, config, flag) or "")+ assert mm.mlx_memory_breakdown(snapshot, n_ctx = 2048, load_in_4bit = flag) is None++ def test_no_checkpoint_on_this_disk_is_refused_for_an_invented_reason(self):+ hub = os.path.expanduser("~/.cache/huggingface/hub")+ seen = accepted = 0+ for path in glob.glob(os.path.join(hub, "models--*", "snapshots", "*", "config.json")):+ directory = os.path.dirname(path)+ config = mm._snapshot_config(directory)+ if not isinstance(config, dict) or not config.get("model_type"):+ continue+ try:+ if not mm.mlx_shard_files(directory, config):+ continue+ except Exception:+ continue+ seen += 1+ refusals = [mm._load_is_refused(directory, config, flag) for flag in (True, False)]+ for refusal in refusals:+ assert refusal is None or any(+ reason in refusal for reason in ("QK-norm", "no home for", "bitsandbytes")+ ), f"{path}: {refusal}"+ accepted += refusals[0] is None+ if not seen:+ pytest.skip("no checkpoints on this disk")+ assert accepted > seen * 0.9, "the gate has started refusing wholesale"++ def test_extras_a_registered_rule_filters_are_not_refused(self):+ snapshot = _local_snapshot("google/gemma-4-E2B")+ config = mm._snapshot_config(snapshot)+ assert mm._load_is_refused(snapshot, config, True) is None+ assert mm.mlx_weight_bytes(snapshot, config, load_in_4bit = True) == 7_510_668_230++ _FILTERED = [+ "language_model.model.per_layer_model_projection.biases",+ "language_model.model.per_layer_model_projection.scales",+ ]+ _GEMMA4 = {"model_type": "gemma4", "vision_config": {"model_type": "gemma4"}}++ @pytest.mark.parametrize(+ "config, extras, names",+ [+ (_GEMMA4, _FILTERED, None),+ (_GEMMA4, _FILTERED + ["mystery.weight"], "mystery.weight"),+ ({"model_type": "an-architecture-with-no-rule"}, ["d2t", "fc.weight"], "d2t"),+ ],+ )+ def test_a_rule_exempts_only_the_keys_it_names(self, config, extras, names):+ refusal = mm._extra_tensors_refused(object(), config, extras)+ assert refusal is None if names is None else (names in refusal and "no home for" in refusal)+++class TestShardsTheLoaderReads:+ """Which safetensors beside a config actually become weights."""++ _VISION = {"model_type": "llava", "vision_config": {"hidden_size": 8}}++ def _spread(self, tmp_path, config):+ for name in (+ "model-00001.safetensors",+ "adapter_model.safetensors",+ "consolidated.safetensors",+ ):+ (tmp_path / name).write_bytes(b"x" * 100)+ return sorted(os.path.basename(p) for p in mm.mlx_shard_files(str(tmp_path), config))++ def test_the_text_loader_reads_model_shards_only(self, tmp_path):+ assert self._spread(tmp_path, {"model_type": "llama"}) == ["model-00001.safetensors"]+ adapter_only = tmp_path / "lora"+ adapter_only.mkdir()+ (adapter_only / "adapter_model.safetensors").write_bytes(b"x" * 100)+ assert mm.mlx_shard_files(str(adapter_only), {"model_type": "llama"}) == []++ def test_the_vision_loader_reads_the_index_before_the_directory(self, tmp_path):+ assert self._spread(tmp_path, self._VISION) == [+ "adapter_model.safetensors",+ "model-00001.safetensors",+ ]+ (tmp_path / "model.safetensors.index.json").write_text(+ json.dumps({"weight_map": {"a.weight": "model-00001.safetensors"}})+ )+ assert self._spread(tmp_path, self._VISION) == ["model-00001.safetensors"]++ @_NEEDS_MLX+ @pytest.mark.parametrize("declared", [0, 1_000_000, 1 << 40])+ def test_a_shard_is_not_read_past_the_header_it_declares(self, tmp_path, declared):+ import mlx.core as mx++ shard = tmp_path / "model.safetensors"+ body = json.dumps({"a.weight": {"dtype": "F16", "shape": [2], "data_offsets": [0, 4]}})+ shard.write_bytes(declared.to_bytes(8, "little") + body.encode() + b"\0" * 4)+ with pytest.raises(ValueError, match = "safetensors header"):+ mm._checkpoint_tensors(str(tmp_path), {"model_type": "llama"}, mx.bfloat16)+ shard.write_bytes(len(body).to_bytes(8, "little") + body.encode() + b"\0" * 4)+ assert list(mm._checkpoint_tensors(str(tmp_path), {"model_type": "llama"}, mx.bfloat16))++ @_NEEDS_MLX+ def test_a_header_is_capped_below_the_shard_it_sits_in(self, tmp_path, monkeypatch):+ import mlx.core as mx++ body = json.dumps({"a.weight": {"dtype": "F16", "shape": [2], "data_offsets": [0, 4]}})+ (tmp_path / "model.safetensors").write_bytes(+ len(body).to_bytes(8, "little") + body.encode() + b"\0" * 4+ )+ read = lambda: mm._checkpoint_tensors(str(tmp_path), {"model_type": "llama"}, mx.bfloat16)+ assert list(read())+ monkeypatch.setattr(mm, "_MAX_SAFETENSORS_HEADER", len(body) - 1)+ with pytest.raises(ValueError, match = "safetensors header"):+ read()++ @pytest.mark.parametrize(+ "shard",+ [+ "../secret.safetensors",+ "a/../../secret.safetensors",+ "/etc/passwd",+ "..\\secret.safetensors",+ "C:/secret.safetensors",+ ],+ )+ def test_an_index_naming_a_shard_outside_the_snapshot_is_refused(self, tmp_path, shard):+ # A cached repo's index must not steer sizing at files beside or above its snapshot.+ (tmp_path / "secret.safetensors").write_bytes(b"x" * 100)+ snapshot = tmp_path / "snap"+ snapshot.mkdir()+ (snapshot / "model-00001.safetensors").write_bytes(b"x" * 100)+ (snapshot / "model.safetensors.index.json").write_text(+ json.dumps({"weight_map": {"a.weight": "model-00001.safetensors", "b.weight": shard}})+ )+ with pytest.raises(ValueError, match = "outside the snapshot"):+ mm.mlx_shard_files(str(snapshot), self._VISION)++ def test_a_nested_shard_inside_the_snapshot_is_still_read(self, tmp_path):+ (tmp_path / "sub").mkdir()+ (tmp_path / "sub" / "model-00001.safetensors").write_bytes(b"x" * 100)+ (tmp_path / "model.safetensors.index.json").write_text(+ json.dumps({"weight_map": {"a.weight": "sub/./model-00001.safetensors"}})+ )+ assert [os.path.basename(p) for p in mm.mlx_shard_files(str(tmp_path), self._VISION)] == [+ "model-00001.safetensors"+ ]++ @pytest.mark.parametrize(+ "weight_map", [["model-00001.safetensors"], "model-00001.safetensors", 7, True, None]+ )+ def test_an_index_whose_weight_map_is_not_a_mapping_is_fatal(self, tmp_path, weight_map):+ (tmp_path / "model.safetensors.index.json").write_text(+ json.dumps({"weight_map": weight_map})+ )+ with pytest.raises(ValueError, match = "no weight map"):+ mm.mlx_shard_files(str(tmp_path), self._VISION)+++@_NEEDS_MLX+ "repo, n_ctx, kv_bits, weights, kv, compute, layers, caption",+ [+ (+ "unsloth/Qwen3-4B-Thinking-2507",+ 32768,+ None,+ 2_822_044_672,+ 4_869_586_944,+ 863_355_535,+ 36,+ "bf16",+ ),+ (+ "unsloth/Qwen3-4B-Thinking-2507",+ 32768,+ 4,+ 2_822_044_672,+ 1_369_571_328,+ 1_035_321_999,+ 36,+ "4-bit",+ ),+ (+ "unsloth/Qwen3.6-35B-A3B-UD-MLX-4bit",+ 32768,+ None,+ 21_634_993_114,+ 740_720_640,+ 943_440_527,+ 40,+ "bf16",+ ),+ ("unsloth/gemma-3-270m-it", 8192, None, 392_058_112, 65_258_496, 725_729_935, 18, "bf16"),+ (+ "mlx-community/deepseek-vl2-tiny-4bit",+ 4096,+ None,+ 2_099_712_075,+ 267_386_880,+ 803_717_775,+ 12,+ "bf16",+ ),+ (+ "Qwen/Qwen2.5-VL-3B-Instruct",+ 32768,+ 4,+ 3_520_856_064,+ 342_392_832,+ 753_582_735,+ 36,+ "4-bit",+ ),+ (+ "Qwen/Qwen2.5-VL-3B-Instruct",+ 5001,+ 4,+ 3_520_856_064,+ 56_852_480,+ 717_152_431,+ 36,+ "4-bit",+ ),+ (+ "Qwen/Qwen2.5-VL-3B-Instruct",+ 5000,+ 4,+ 3_520_856_064,+ 56_852_480,+ 717_151_119,+ 36,+ "4-bit",+ ),+ (+ "mlx-community/gemma-3n-E2B-it-4bit",+ 4096,+ None,+ 4_462_976_625,+ 60_784_640,+ 705_544_847,+ 30,+ "bf16",+ ),+ (+ "mlx-community/Molmo-7B-D-0924-4bit",+ 4096,+ None,+ 5_298_715_127,+ 249_561_088,+ 1_281_737_359,+ 28,+ "bf16",+ ),+ (+ "hf-internal-testing/tiny-random-LlamaForCausalLM",+ 512,+ None,+ 2_061_600,+ 98_304,+ 688_341_647,+ 2,+ "bf16",+ ),+ ],+)+def test_a_real_checkpoint_is_priced_to_the_byte(+ repo, n_ctx, kv_bits, weights, kv, compute, layers, caption+):+ _on_bfloat16_chip()+ breakdown = mm.mlx_memory_breakdown(+ _local_snapshot(repo), n_ctx = n_ctx, load_in_4bit = True, kv_bits = kv_bits+ )+ assert breakdown is not None+ assert (breakdown.weights_bytes, breakdown.kv_bytes) == (weights, kv)+ assert breakdown.compute_bytes == compute+ assert breakdown.layer_count == layers+ assert breakdown.cache_type_kv == caption+++@_NEEDS_MLX[email protected]("kv_bits, caption", [(6, "6-bit"), (8, "8-bit")])+def test_the_caption_names_a_width_the_real_conversion_reaches(kv_bits, caption):+ _on_bfloat16_chip()+ breakdown = mm.mlx_memory_breakdown(+ _local_snapshot("unsloth/Qwen3-4B-Thinking-2507"),+ n_ctx = 32768,+ load_in_4bit = True,+ kv_bits = kv_bits,+ )+ assert breakdown is not None and breakdown.cache_type_kv == caption+++class TestTheContextSearch:+ """The search itself, priced by a stub so it runs where MLX and the checkpoints do not."""++ @staticmethod+ def _linear(monkeypatch, unpriceable = ()):+ """A byte per token, so every expected answer below is readable off the budget."""++ def _priced(sizing, n_ctx):+ return None if n_ctx in unpriceable else SimpleNamespace(total_bytes = n_ctx)++ monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: object())+ monkeypatch.setattr(mm, "_priced_at", _priced)++ def test_the_answer_is_the_largest_whole_block_under_budget(self, monkeypatch):+ self._linear(monkeypatch)+ assert mm.mlx_fit_context("x", budget_bytes = 7000, max_ctx = 8192) == 6912++ def test_a_floor_between_blocks_rounds_up_rather_than_under_the_minimum(self, monkeypatch):+ self._linear(monkeypatch)+ assert mm.mlx_fit_context("x", budget_bytes = 4200, max_ctx = 8192, min_ctx = 4100) is None+ assert mm.mlx_fit_context("x", budget_bytes = 5000, max_ctx = 8192, min_ctx = 4100) == 4864++ def test_a_context_that_cannot_be_priced_abandons_the_fit(self, monkeypatch):+ self._linear(monkeypatch, unpriceable = (6144,))+ assert mm.mlx_fit_context("x", budget_bytes = 7000, max_ctx = 8192) is None++ def test_a_ceiling_that_already_fits_is_left_alone(self, monkeypatch):+ self._linear(monkeypatch)+ assert mm.mlx_fit_context("x", budget_bytes = 9000, max_ctx = 8192) is None++ def test_a_load_that_cannot_be_sized_is_not_fitted(self, monkeypatch):+ monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: None)+ assert mm.mlx_fit_context("x", budget_bytes = 7000, max_ctx = 8192) is None+++@_NEEDS_MLX[email protected]("budget_gib, fitted", [(6, 18_432), (12, 61_952), (24, 149_504)])+def test_a_real_checkpoint_is_fitted_to_the_byte(budget_gib, fitted):+ _on_bfloat16_chip()+ snapshot = _local_snapshot("unsloth/Qwen3-4B-Thinking-2507")+ budget = budget_gib * 1024**3+ assert (+ mm.mlx_fit_context(snapshot, budget_bytes = budget, max_ctx = 262_144, load_in_4bit = True)+ == fitted+ )+ priced = mm.mlx_memory_breakdown(snapshot, n_ctx = fitted, load_in_4bit = True)+ over = mm.mlx_memory_breakdown(snapshot, n_ctx = fitted + mm.MLX_KV_BLOCK, load_in_4bit = True)+ assert priced.total_bytes <= budget < over.total_bytes+ assert mm.mlx_memory_breakdown(snapshot, n_ctx = float("nan"), load_in_4bit = True) is None+++@_NEEDS_MLX+ "repo, kv_bits, whole_prompt",+ [+ ("unsloth/Qwen3-4B-Thinking-2507", 4, False),+ ("unsloth/Qwen3-4B-Thinking-2507", None, True),+ ("unsloth/gemma-3-270m-it", None, False),+ ("unsloth/Qwen3.6-35B-A3B-UD-MLX-4bit", None, False),+ ],+)+def test_the_total_never_falls_as_the_context_grows(repo, kv_bits, whole_prompt):+ sizing = mm._size_load(_local_snapshot(repo), kv_bits, None, True)+ if whole_prompt:+ forced = replace(sizing, facts = {**sizing.facts, "whole_prompt": True})+ assert mm._priced_at(forced, 32_768).total_bytes > mm._priced_at(sizing, 32_768).total_bytes+ sizing = forced+ steps = range(mm.MLX_KV_BLOCK, 262_145, mm.MLX_KV_BLOCK)+ totals = [mm._priced_at(sizing, n).total_bytes for n in steps]+ assert totals == sorted(totals)diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 10eb382295b..f543b369e19 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -4807,12 +4807,13 @@ def test_the_load_policy_bounds_a_pin_only_where_the_bound_can_be_enforced(monkekv_bits,requested,served = unset,+ fitted = False,):backend = MLXInferenceBackend()backend._model = modelbackend._is_vlm = Falsequant, window, enforced, budget = backend._resolve_kv_policy(- False, kv_bits, requested, requested if served is unset else served+ False, kv_bits, requested, requested if served is unset else served, fitted = fitted)return quant["kv_bits"], window, enforced, budget@@ -4821,6 +4822,11 @@ def test_the_load_policy_bounds_a_pin_only_where_the_bound_can_be_enforced(monkeassert policy(honours, None, 8192) == (None, 8192, True, None)assert policy(honours, None, 0, 262144) == (None, 262144, True, None) # nothing to yield to+ # A window fitted to the machine is held like a pin: a budget under a width, else a bound.+ assert policy(honours, 4, 0, 24576, fitted = True) == (4, None, False, 24576)+ assert policy(honours, None, 0, 24576, fitted = True) == (None, 24576, True, None)+ assert policy(owns, 4, 0, 24576, fitted = True) == (4, None, False, None)+assert policy(owns, 4, 8192) == (4, None, False, None)assert policy(owns, None, 8192) == (None, None, False, None)@@ -5383,6 +5389,366 @@ def test_the_mlx_mcp_snapshot_is_taken_under_the_same_guard_the_gguf_count_uses(assert guard < snapshot, "the guard must be held across the snapshot, not after it"[email protected](autouse = True)+def _fresh_mlx_estimate_caches():+ # Tests ask one path under different patched towers.+ from core.inference import mlx_inference, mlx_memory++ mlx_inference._kv_refusal_cache.clear()+ mlx_memory._clear_sizing_cache()+ yield+ mlx_inference._kv_refusal_cache.clear()+ mlx_memory._clear_sizing_cache()+++def test_a_width_refusal_is_asked_once_per_checkpoint_and_width(monkeypatch, tmp_path):+ from core.inference import mlx_inference++ (tmp_path / "config.json").write_text("{}")+ asked = []+ monkeypatch.setattr(+ mlx_inference, "_asking_the_architecture", lambda d, q: asked.append(d) or True+ )+ assert mlx_inference.mlx_kv_quant_is_refused(str(tmp_path), 4) is True+ assert mlx_inference.mlx_kv_quant_is_refused(str(tmp_path), 4) is True and len(asked) == 1+ assert mlx_inference.mlx_kv_quant_is_refused(str(tmp_path), 8) is True and len(asked) == 2+ (tmp_path / "config.json").write_text('{"changed": 1}')+ assert mlx_inference.mlx_kv_quant_is_refused(str(tmp_path), 4) is True and len(asked) == 3+++def test_what_the_fit_is_asked_and_when_it_is_asked_at_all(monkeypatch, tmp_path):+ from core.inference import mlx_inference++ priced, rewound = [], []+ monkeypatch.setitem(+ sys.modules,+ "core.inference.mlx_memory",+ types.SimpleNamespace(+ mlx_fit_context = lambda model_dir, **kw: (+ priced.append(kw | {"dir": model_dir})+ or rewound.append(f"priced@{kw['kv_bits']}")+ or 24_576+ )+ ),+ )+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: 8 * 1024**3)+ monkeypatch.setattr(+ mlx_inference, "_mlx_rng_key_words", lambda: rewound.append("held") or ("key",)+ )+ monkeypatch.setattr(mlx_inference, "_restore_mlx_rng_key", rewound.append)+ asked = []+ verdict = {"answer": ("full", "", True)}+ monkeypatch.setattr(+ mlx_inference,+ "_kv_quant_eligibility",+ lambda *a: asked.append(a) or verdict["answer"],+ )+ model = object()++ def fit(kv_bits = None, **kw):+ priced.clear(), asked.clear(), rewound.clear()+ return mlx_inference._fitted_context(+ model,+ str(tmp_path),+ 262_144,+ load_in_4bit = True,+ retains_history = True,+ kv_bits = kv_bits,+ **kw,+ )++ assert fit() == (24_576, None)+ assert asked == [] and rewound == ["held", "priced@None", ("key",)]+ assert priced == [+ {+ "budget_bytes": 8 * 1024**3,+ "max_ctx": 262_144,+ "kv_bits": None,+ "load_in_4bit": True,+ "dir": str(tmp_path),+ }+ ]+ # The width the cache will take is the width the fit is priced at.+ for answer, bits in ((("full", "", True), 4), (("partial", "w", True), 4)):+ verdict["answer"] = answer+ assert fit(4, is_vlm = True) == (24_576, answer)+ assert asked == [(model, True, 4)] and priced[0]["kv_bits"] == bits+ for answer in (("refused", "r", True), ("none", "n", True)):+ verdict["answer"] = answer+ assert fit(4) == (24_576, answer) and priced[0]["kv_bits"] is None++ monkeypatch.setitem(+ sys.modules,+ "core.inference.mlx_memory",+ types.SimpleNamespace(+ mlx_fit_context = lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError("no"))+ ),+ )+ assert fit() == (None, None) and rewound == ["held", ("key",)]+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: None)+ assert fit() == (None, None) and rewound == []+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: 8 * 1024**3)+ monkeypatch.setattr(mlx_inference, "_snapshot_dir", lambda model, name: None)+ assert fit() == (None, None) and (priced, rewound) == ([], [])+++def test_the_fit_budget_leaves_the_prompt_history_its_own_room(monkeypatch):+ from core.inference import mlx_inference++ fake = types.SimpleNamespace(+ metal = types.SimpleNamespace(is_available = lambda: True),+ device_info = lambda: {"max_recommended_working_set_size": 100_000_000_000},+ )+ monkeypatch.setitem(sys.modules, "mlx.core", fake)+ if "mlx" in sys.modules:+ monkeypatch.setattr(sys.modules["mlx"], "core", fake, raising = False)+ else:+ # The parent package is imported first, so it must resolve on hosts without MLX.+ monkeypatch.setitem(sys.modules, "mlx", types.SimpleNamespace(core = fake))+ monkeypatch.delenv("UNSLOTH_MLX_PROMPT_CACHE_BYTES", raising = False)++ assert mlx_inference.mlx_memory_budget() == 85_000_000_000 - 15_000_000_000+ assert mlx_inference.mlx_memory_budget(retains_history = False) == 85_000_000_000+ fake.metal.is_available = lambda: False+ assert mlx_inference.mlx_memory_budget() is None+++def test_the_fit_asks_about_the_repo_the_loader_actually_opens(monkeypatch, tmp_path):+ import utils.utils as backend_utils+ from core.inference import mlx_inference++ asked = []+ monkeypatch.setattr(+ backend_utils, "hf_cache_snapshot_dir", lambda name: asked.append(name) or tmp_path+ )+ assert mlx_inference._snapshot_dir(SimpleNamespace(_src_path = tmp_path), "org/alias") == str(+ tmp_path+ )+ assert asked == []+ assert mlx_inference._snapshot_dir(SimpleNamespace(), "org/alias") == str(tmp_path)+ assert asked == ["org/alias"]+++def test_only_a_load_that_asked_for_nothing_is_fitted_to_the_machine(monkeypatch):+ """A pin is the user's own answer about memory and is served verbatim."""+ _install_fake_mlx(monkeypatch)+ _install_fake_fast_mlx(monkeypatch, [])++ from core.inference import mlx_inference++ FULL = ("full", "", True)+ asked, verdicts, probes = [], [], []+ monkeypatch.setattr(+ mlx_inference, "_kv_quant_eligibility", lambda *a: verdicts.append(a) or FULL+ )++ def load(+ fitted = (24_576, FULL),+ enforceable = bool,+ model = "fake/text",+ **kwargs,+ ):+ del asked[:], verdicts[:], probes[:]+ monkeypatch.setattr(+ mlx_inference,+ "_fitted_context",+ lambda _m, name, ceiling, **kw: asked.append((name, ceiling, kw)) or fitted,+ )+ monkeypatch.setattr(+ mlx_inference.MLXInferenceBackend,+ "_kv_cache_window_enforceable",+ lambda _s, served: probes.append(served) or enforceable(served),+ )+ backend = mlx_inference.MLXInferenceBackend()+ assert backend.load_model(+ SimpleNamespace(+ identifier = model,+ is_vision = kwargs.pop("vision", False),+ is_lora = kwargs.pop("lora", False),+ ),+ **kwargs,+ )+ return backend, backend.models[model]++ backend, pinned = load(max_seq_length = 4096, kv_quant = "4")+ assert (asked, verdicts) == ([], [(backend._model, False, 4)])+ assert (pinned["context_length"], pinned["context_length_fitted"]) == (4096, None)+ assert (pinned["mlx_kv_bits"], pinned["mlx_context_budget"]) == (4, 4096)+ # A fitted window is held like a pin: the width takes the cache, the window a budget.+ backend, info = load(max_seq_length = 0, kv_quant = "4")+ assert (verdicts, probes) == ([], [24_576])+ assert asked == [+ ("fake/text", None, dict(load_in_4bit = True, retains_history = True, kv_bits = 4, is_vlm = False))+ ]+ assert (info["context_length"], info["context_length_fitted"]) == (24_576, 24_576)+ assert (info["mlx_kv_bits"], info["mlx_context_budget"]) == (4, 24_576)+ assert (info["context_length_enforced"], backend._kv_cache_window) == (False, None)++ backend, info = load((24_576, None), max_seq_length = 0)+ assert (info["mlx_kv_bits"], info["mlx_context_budget"]) == (None, None)+ assert (info["context_length_enforced"], backend._kv_cache_window) == (True, 24_576)+ refused = ("refused", "r", True)+ backend, info = load((24_576, refused), max_seq_length = 0, kv_quant = "4")+ assert (info["mlx_kv_bits"], backend._kv_cache_window, verdicts) == (None, 24_576, [])+ _, info = load(enforceable = lambda _s: False, max_seq_length = 0, kv_quant = "4")+ assert (info["mlx_kv_bits"], info["mlx_context_budget"], info["context_length"]) == (+ 4,+ None,+ 24_576,+ )+ backend, info = load(enforceable = lambda _s: False, max_seq_length = 4096, kv_quant = "4")+ assert (info["mlx_kv_bits"], verdicts) == (4, [(backend._model, False, 4)])++ # TurboQuant's layout is not one the planner prices: fitted at full width, no mx.quantize probe.+ monkeypatch.setitem(+ sys.modules, "mlx_vlm.turboquant", SimpleNamespace(TurboQuantKVCache = object)+ )+ monkeypatch.setattr(+ mlx_inference,+ "_turboquant_status",+ lambda _m, bits, refusal = "": dict(+ requested_kv_bits = bits, kv_bits = bits, eligibility = "full", reason = "", note = ""+ ),+ )+ load(max_seq_length = 0, kv_quant = "tq-4")+ assert verdicts == [] and [kw["kv_bits"] for _, _, kw in asked] == [None]++ for available, reserved in ((True, True), (False, False)):+ monkeypatch.setattr(+ mlx_inference, "mlx_vlm_snapshot_store_available", lambda a = available: a+ )+ load(model = "fake/vlm", vision = True, max_seq_length = 0)+ assert asked == [+ (+ "fake/vlm",+ None,+ dict(+ load_in_4bit = True,+ retains_history = reserved,+ kv_bits = None,+ is_vlm = True,+ ),+ )+ ]++ for kwargs in (+ dict(+ parallel_mode = "tensor",+ distributed_group = SimpleNamespace(rank = lambda: 0, size = lambda: 2),+ ),+ dict(dtype = "float32"),+ dict(lora = True),+ ):+ load(max_seq_length = 0, **kwargs)+ assert not asked+++def test_the_width_probe_judges_the_tower_the_sizing_would_price(monkeypatch):+ """The load's eligibility rule, asked of the first tower that runs, and never evaluated."""+ from core.inference import mlx_inference++ fake_mx = types.SimpleNamespace(+ array = lambda value: value,+ zeros = lambda *_a, **_k: None,+ int32 = None,+ eval = lambda *_a: pytest.fail("a weightless tower must never be evaluated"),+ )+ monkeypatch.setitem(sys.modules, "mlx.core", fake_mx)+ if "mlx" in sys.modules:+ monkeypatch.setattr(sys.modules["mlx"], "core", fake_mx, raising = False)+ else:+ monkeypatch.setitem(sys.modules, "mlx", types.SimpleNamespace(core = fake_mx))++ class Quantized:+ keys, state, nbytes = "k", (), 1++ class Entry:+ keys, state, window_size = "k", (), None++ def __init__(+ self,+ max_size = None,+ converts = True,+ ):+ self.max_size = max_size+ if converts:+ self.to_quantized = lambda **_kw: Quantized()++ class Tower:+ def __init__(+ self,+ *entries,+ runs = True,+ attends_quantized = True,+ ):+ self.entries, self.runs, self.attends_quantized = entries, runs, attends_quantized++ def __call__(+ self,+ *_a,+ cache = None,+ **_k,+ ):+ if not self.runs:+ raise ValueError("this tower cannot run")+ if not self.attends_quantized and any(isinstance(e, Quantized) for e in cache):+ raise TypeError("shared KV takes arrays only")++ rewound = []++ def refused(*towers):+ monkeypatch.setitem(+ sys.modules,+ "core.inference.mlx_memory",+ types.SimpleNamespace(+ _PROBE_SHORT = 8,+ _runtime_dtype = lambda: None,+ _snapshot_config = lambda _d: {"model_type": "x"},+ _probe_models = lambda *_a: iter(+ [(lambda t = t: t, lambda tower: list(tower.entries), None) for t in towers]+ ),+ ),+ )+ marks = len(rewound)+ mlx_inference._kv_refusal_cache.clear()+ answer = mlx_inference.mlx_kv_quant_is_refused("/d", 4)+ assert rewound[marks] == "held" and rewound[-1] == ("key",)+ return answer++ monkeypatch.setattr(+ mlx_inference, "_mlx_rng_key_words", lambda: rewound.append("held") or ("key",)+ )+ monkeypatch.setattr(mlx_inference, "_restore_mlx_rng_key", rewound.append)++ assert refused(Tower(Entry())) is False+ # Sliding-window layers keep their ring and the rest convert: partial, still quantized.+ assert refused(Tower(Entry(), Entry(max_size = 512))) is False+ assert refused(Tower(Entry(max_size = 512))) is True+ assert refused(Tower(Entry(converts = False))) is True+ assert refused(Tower()) is True+ # Gemma 4 E2B / E4B: the conversion succeeds, attending over it does not.+ assert refused(Tower(Entry(), Entry(max_size = 512), attends_quantized = False)) is True+ broken = Entry()+ broken.to_quantized = lambda **_kw: (_ for _ in ()).throw(ValueError("group size"))+ assert refused(Tower(broken)) is True+ assert refused(Tower(Entry(max_size = 512), runs = False), Tower(Entry())) is False+ assert refused(Tower(Entry(), runs = False), Tower(Entry(max_size = 512))) is True+ # Nothing that runs to ask: priced as requested, as the load would try it.+ assert refused(Tower(Entry(max_size = 512), runs = False)) is False++ monkeypatch.setitem(+ sys.modules,+ "core.inference.mlx_memory",+ types.SimpleNamespace(+ _snapshot_config = lambda _d: (_ for _ in ()).throw(RuntimeError("no")),+ ),+ )+ marks = len(rewound)+ mlx_inference._kv_refusal_cache.clear()+ assert mlx_inference.mlx_kv_quant_is_refused("/d", 4) is False+ assert rewound[marks:] == ["held", ("key",)]++class _TwoRowStream:"""A vision batch reporting two rows, recording whatever is taken back."""diff --git a/studio/backend/tests/test_non_gguf_reload_settings.py b/studio/backend/tests/test_non_gguf_reload_settings.pyindex ee6eb41433c..789e5e87b0f 100644--- a/studio/backend/tests/test_non_gguf_reload_settings.py+++ b/studio/backend/tests/test_non_gguf_reload_settings.py@@ -194,6 +194,43 @@ class TestNonGgufStatusReportsWhatTheLoadAskedFor:assert response.load_in_4bit is Falseassert response.requested_gpu_ids == [0, 1]+ def test_a_window_fitted_to_the_machine_reaches_the_client(self, monkeypatch):+ """Only the worker measured it, and nothing outside can recompute it."""+ response = self._status_for(monkeypatch, {"context_length_fitted": 14336})++ assert response.context_length_fitted == 14336++ def test_every_projection_that_reports_a_bound_reports_how_it_was_chosen(self):+ """Four hand-maintained projections, and a drop from any one of them is silent."""+ lines = (_BACKEND / "routes" / "inference.py").read_text(encoding = "utf-8").split("\n")+ orphaned = [+ index + 1+ for index, line in enumerate(lines)+ if "context_length_enforced" in line+ and not any(+ "context_length_fitted" in lines[near]+ for near in range(max(0, index - 6), min(len(lines), index + 7))+ )+ ]++ assert orphaned == [], f"context_length_enforced without a fitted window at {orphaned}"++ def test_the_parent_mirrors_every_context_field_the_worker_sends(self):+ """The hop the route test cannot see: a field the mirror drops is one no API reports."""+ from core.inference.orchestrator import _mirrored_model_entry++ sent = {+ "context_length": 14336,+ "native_context_length": 40960,+ "max_context_length": 40960,+ "requested_context_length": 0,+ "context_length_enforced": True,+ "context_length_fitted": 14336,+ }+ mirrored = _mirrored_model_entry(dict(sent, display_name = "m"), "m")++ assert {name: mirrored.get(name) for name in sent} == sent+def test_the_mlx_mirror_wins_over_the_stamped_spelling(self, monkeypatch):"""#8125: the MLX worker mirrors the real context back as requested_context_length."""response = self._status_for(diff --git a/studio/frontend/src/features/model-picker/api/memory-estimate.ts b/studio/frontend/src/features/model-picker/api/memory-estimate.tsindex 588d89af955..163de0dd26b 100644--- a/studio/frontend/src/features/model-picker/api/memory-estimate.ts+++ b/studio/frontend/src/features/model-picker/api/memory-estimate.ts@@ -18,7 +18,6 @@ export type MemoryEstimateReason =export interface MemoryEstimate {available: boolean;reason: MemoryEstimateReason | null;- /** Files that become resident: weights, projector, drafter. */weightsBytes: number;/** KV cache at the priced context and slot count. Meaningless unless `kvEstimable`. */kvBytes: number;@@ -50,6 +49,7 @@ export interface MemoryEstimate {kvOnGpu: boolean;/** What was actually priced, after overrides and clamps resolve. */nCtx: number;+ contextFitted: number | null;cacheTypeKv: string | null;nParallel: number;layerCount: number | null;@@ -67,6 +67,8 @@ export interface MemoryEstimateRequest {nativePathToken?: string | null;nCtx?: number | null;cacheTypeKv?: string | null;+ maxSeqLength?: number | null;+ mlxKvQuant?: string | null;nParallel?: number | null;nBatch?: number | null;nUbatch?: number | null;@@ -100,6 +102,7 @@ const UNAVAILABLE: MemoryEstimate = {kvEstimable: false,kvOnGpu: true,nCtx: 0,+ contextFitted: null,cacheTypeKv: null,nParallel: 1,layerCount: null,@@ -124,6 +127,7 @@ interface ApiEstimateResponse {kv_estimable: boolean;kv_on_gpu: boolean;n_ctx: number;+ context_fitted?: number | null;cache_type_kv: string | null;n_parallel: number;layer_count: number | null;@@ -142,6 +146,8 @@ function estimateRequestBody(native_path_lease: nativePathLease,n_ctx: payload.nCtx ?? null,cache_type_kv: payload.cacheTypeKv ?? null,+ max_seq_length: payload.maxSeqLength ?? null,+ mlx_kv_quant: payload.mlxKvQuant ?? null,n_parallel: payload.nParallel ?? null,n_batch: payload.nBatch ?? null,n_ubatch: payload.nUbatch ?? null,@@ -235,6 +241,7 @@ function toMemoryEstimate(body: ApiEstimateResponse): MemoryEstimate {kvEstimable: flag(body.kv_estimable, false),kvOnGpu: flag(body.kv_on_gpu, true),nCtx: finiteCount(body.n_ctx, 0),+ contextFitted: nullableCount(body.context_fitted),cacheTypeKv:typeof body.cache_type_kv === "string" ? body.cache_type_kv : null,nParallel: finiteCount(body.n_parallel, 1),@@ -266,10 +273,6 @@ export function resetMemoryEstimateRouteMemo(): void {routeAbsentAt = null;}-/** Price a prospective GGUF load from its header. Allocates nothing, loads nothing. Never- * throws for a backend answer: an absent route, an auth expiry, a 500, an HTML error page- * served as 200 or a truncated body all come back as an unavailable estimate, so the panel- * hides the row. The statuses are told apart only to decide whether the miss is memoable. */export async function fetchMemoryEstimate(payload: MemoryEstimateRequest,signal?: AbortSignal,diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex 3c8b56d3ca1..72058ee3f2c 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -81,7 +81,13 @@ import {loadManagedLlamaFlags,subscribeLlamaFlagCatalog,} from "../api/llama-flags";-import { resolveEstimateContext } from "../model-config/estimate-context";+import { type MemoryEstimate } from "../api/memory-estimate";+import {+ resolveEstimateContext,+ resolveMlxEstimateContext,+ resolveMlxServedWindow,+ shouldRequestMemoryEstimate,+} from "../model-config/estimate-context";import { resolveReclaimableMemoryCredit } from "../model-config/memory-fit";import {resolveResidentEstimateRequest,@@ -427,6 +433,7 @@ function MaxSeqLengthSetting({inputRef,isMlx,pinned,+ fittedToMemory,windowUnknown,}: {value: number;@@ -436,6 +443,7 @@ function MaxSeqLengthSetting({inputRef?: Ref<NumericValueInputHandle>;isMlx?: boolean;pinned?: boolean;+ fittedToMemory?: boolean;windowUnknown?: boolean;}) {// MLX sizes itself when unpinned, so the control is the GGUF path's Context Length and@@ -448,7 +456,11 @@ function MaxSeqLengthSetting({<span className={LABEL_CLASS}>{label}</span><InfoHint>{isMlx- ? "Tokens of context the model is sized for."+ ? "Tokens of context the model is sized for." ++ (fittedToMemory+ ? " Fitted to this machine's memory, which is less than the model's own " ++ "window. Set a length to ask for a different one."+ : ""): "Maximum context window in tokens. Applies on load."}</InfoHint></div>@@ -2015,6 +2027,7 @@ export function ModelConfigPage({const loadedGpuIds = useChatRuntimeStore((s) => s.loadedGpuIds);const loadedGpuIndexKind = useChatRuntimeStore((s) => s.loadedGpuIndexKind);const loadedCpuFallback = useChatRuntimeStore((s) => s.loadedCpuFallback);+ const residentModelLoading = useChatRuntimeStore((s) => s.modelLoading);const residentEstimateSettings = useChatRuntimeStore(useShallow(selectResidentEstimateSettings),);@@ -2741,18 +2754,6 @@ export function ModelConfigPage({mlxNativeWindow == null? null: Math.min(mlxNativeWindow, MAX_SEQ_LENGTH_MAX);- const mlxServedWindow =- (targetIsMlx && isActiveModel ? servedWindow(loadedContextLength) : null) ??- mlxProspectiveWindow;- const maxSeqLengthValue =- servedWindow(savedContextPin(config)) ??- mlxServedWindow ??- clampMaxSeqLength(DEFAULT_MAX_SEQ_LENGTH, nativeMaxSeqLength);- // The slider picks a request, so it stops at the widest a load may make.- const maxSeqLengthMax = Math.min(- MAX_SEQ_LENGTH_MAX,- Math.max(nativeMaxSeqLength, maxSeqLengthValue),- );// An auto-fit-below-native GGUF shows activeLoadedContext while// customContextLength stays null. If the user fixes GPU Layers (Manual) and// remembers, pin that shown context so a later fresh load keeps the fitted@@ -2790,7 +2791,11 @@ export function ModelConfigPage({// tri-state, not through resolvedIsDiffusion: a GGUF still being classified may be// DiffusionGemma, and guessing paints a footprint from the wrong plan that never clears.const memoryEstimateRequest =- target.isGguf && classifiedIsDiffusion === false+ shouldRequestMemoryEstimate({+ isGguf: Boolean(target.isGguf),+ isAppleUnifiedMemory,+ classifiedIsDiffusion,+ })? {modelPath: target.id,ggufVariant: target.ggufVariant ?? null,@@ -2809,6 +2814,10 @@ export function ModelConfigPage({(target.isGguf === true && activePresetSource === "builtin-default"),),cacheTypeKv: runtimeConfig.kvCacheDtype,+ maxSeqLength: target.isGguf+ ? null+ : resolveMlxEstimateContext(savedContextPin(config)),+ mlxKvQuant: runtimeConfig.mlxKvQuant ?? null,nParallel: runtimeConfig.nParallel,nBatch: runtimeConfig.nBatch,nUbatch: runtimeConfig.nUbatch,@@ -2834,12 +2843,34 @@ export function ModelConfigPage({}: null;const memoryEstimate = useMemoryEstimate(memoryEstimateRequest);+ const mlxFittedWindow = targetIsMlx+ ? servedWindow(memoryEstimate.estimate?.contextFitted)+ : null;+ const mlxServedWindow = resolveMlxServedWindow(+ targetIsMlx && isActiveModel ? servedWindow(loadedContextLength) : null,+ mlxFittedWindow,+ mlxProspectiveWindow,+ );+ const maxSeqLengthValue =+ servedWindow(savedContextPin(config)) ??+ mlxServedWindow ??+ clampMaxSeqLength(DEFAULT_MAX_SEQ_LENGTH, nativeMaxSeqLength);+ const maxSeqLengthMax = Math.min(+ MAX_SEQ_LENGTH_MAX,+ Math.max(nativeMaxSeqLength, maxSeqLengthValue),+ );// No resident credit without a reported context; pending settings cannot price it.- const residentContext = servedWindow(activeLoadedContext);+ // activeLoadedContext is GGUF-only; an MLX resident reports its served window here.+ const residentContext = servedWindow(+ targetIsMlx && isActiveModel ? loadedContextLength : activeLoadedContext,+ );const residentEstimateRequest = resolveResidentEstimateRequest(isActiveModel ? memoryEstimateRequest : null,residentEstimateSettings,residentContext,+ targetIsMlx && !residentModelLoading+ ? { kvQuant: loadedMlxKvQuantRequested ?? null }+ : undefined,);const residentEstimate = useMemoryEstimate(residentEstimateRequest, {refreshMemory: true,@@ -3242,30 +3273,30 @@ export function ModelConfigPage({)}<div className="space-y-5">+ {memoryEstimateRequest != null && (+ <MemoryEstimateRow+ estimate={memoryEstimate.estimate}+ loading={memoryEstimate.loading}+ stale={memoryEstimate.stale}+ gpuCapacityGb={memoryGpuCapacityGb}+ totalCapacityGb={memoryTotalCapacityGb}+ systemRamCapacityGb={inferenceGpu.systemRamTotalGb}+ freeGpuCapacityGb={memoryFreeGpuCapacityGb}+ freeGpuCapacityKnown={memoryFreeGpuCapacityKnown}+ freeGpuReserveDeficitGb={memoryFreeGpuReserveDeficitGb}+ usableSystemRamGb={memoryUsableSystemRamGb}+ usableSystemRamKnown={inferenceGpu.systemRamAvailableKnown}+ systemRamReserveDeficitGb={memorySystemRamReserveDeficitGb}+ isUnifiedMemory={isAppleUnifiedMemory}+ singleMemoryPool={singleMemoryPool}+ reclaimableTotalBytes={reclaimableCredit.totalBytes}+ reclaimableGpuBytes={reclaimableCredit.gpuBytes}+ expanded={memoryBreakdownOpen}+ onExpandedChange={setMemoryBreakdownOpen}+ />+ )}{target.isGguf && (<>- {/* Above Context Length on purpose: that is the control moving this number most, and a readout- below it is one you go looking for. */}- <MemoryEstimateRow- estimate={memoryEstimate.estimate}- loading={memoryEstimate.loading}- stale={memoryEstimate.stale}- gpuCapacityGb={memoryGpuCapacityGb}- totalCapacityGb={memoryTotalCapacityGb}- systemRamCapacityGb={inferenceGpu.systemRamTotalGb}- freeGpuCapacityGb={memoryFreeGpuCapacityGb}- freeGpuCapacityKnown={memoryFreeGpuCapacityKnown}- freeGpuReserveDeficitGb={memoryFreeGpuReserveDeficitGb}- usableSystemRamGb={memoryUsableSystemRamGb}- usableSystemRamKnown={inferenceGpu.systemRamAvailableKnown}- systemRamReserveDeficitGb={memorySystemRamReserveDeficitGb}- isUnifiedMemory={isAppleUnifiedMemory}- singleMemoryPool={singleMemoryPool}- reclaimableTotalBytes={reclaimableCredit.totalBytes}- reclaimableGpuBytes={reclaimableCredit.gpuBytes}- expanded={memoryBreakdownOpen}- onExpandedChange={setMemoryBreakdownOpen}- /><div className="space-y-2"><div className={ROW_CLASS}><div className="flex min-w-0 items-center gap-1.5">@@ -3383,6 +3414,9 @@ export function ModelConfigPage({inputRef={maxSeqLengthInputRef}isMlx={targetIsMlx}pinned={savedContextPin(config) != null}+ fittedToMemory={+ savedContextPin(config) == null && mlxFittedWindow != null+ }windowUnknown={savedContextPin(config) == null && mlxServedWindow == null}diff --git a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.tsindex 0d7e8486557..532dfb94f95 100644--- a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts+++ b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts@@ -36,6 +36,8 @@ function estimateKey(request: MemoryEstimateRequest | null): string | null {request.nativePathToken ?? null,request.nCtx ?? null,request.cacheTypeKv ?? null,+ request.maxSeqLength ?? null,+ request.mlxKvQuant ?? null,request.nParallel ?? null,request.nBatch ?? null,request.nUbatch ?? null,diff --git a/studio/frontend/src/features/model-picker/model-config/estimate-context.ts b/studio/frontend/src/features/model-picker/model-config/estimate-context.tsindex b91a180072d..c52d89e33f2 100644--- a/studio/frontend/src/features/model-picker/model-config/estimate-context.ts+++ b/studio/frontend/src/features/model-picker/model-config/estimate-context.ts@@ -24,6 +24,18 @@ export function resolveEstimateContext(return customContextLength ?? activeLoadedContext ?? 0;}+export function resolveMlxEstimateContext(contextPin: number | null): number {+ return contextPin && contextPin > 0 ? contextPin : 0;+}++export function resolveMlxServedWindow(+ loadedContext: number | null,+ fittedContext: number | null,+ nativeWindow: number | null,+): number | null {+ return fittedContext ?? loadedContext ?? nativeWindow;+}+/** Which MODEL an estimate belongs to, for deciding whether shown numbers still do. Not the same* question as "did anything change" -- that is the full request key, and a slider step must* keep the old figures up and mark them stale. A change here means the numbers describe a@@ -60,3 +72,13 @@ export function resolveEstimateSourceIdentity(nativePathToken ?? null,]);}++export function shouldRequestMemoryEstimate(opts: {+ isGguf: boolean;+ isAppleUnifiedMemory: boolean;+ classifiedIsDiffusion: boolean | undefined;+}): boolean {+ const { isGguf, isAppleUnifiedMemory, classifiedIsDiffusion } = opts;+ if (isGguf) return classifiedIsDiffusion === false;+ return isAppleUnifiedMemory && classifiedIsDiffusion !== true;+}diff --git a/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts b/studio/frontend/src/features/model-picker/model-config/resident-memory-request.tsindex 5f66bd46d1d..45bcf8a1e01 100644--- a/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts+++ b/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts@@ -65,15 +65,17 @@ export function selectResidentEstimateSettings(state: ResidentState) {};}-/** Copy only source identity from the pending request. */+/** Copy only source identity from the pending request. `mlx`: the resident is an MLX load,+ * which reads its window from maxSeqLength and its cache width from mlxKvQuant, not nCtx. */export function resolveResidentEstimateRequest(source: MemoryEstimateRequest | null,settings: ReturnType<typeof selectResidentEstimateSettings>,context: number | null,+ mlx?: { kvQuant: string | null },): MemoryEstimateRequest | null {if (!source ||- !settings ||+ (!settings && !mlx) ||context == null ||!Number.isFinite(context) ||context <= 0@@ -84,7 +86,11 @@ export function resolveResidentEstimateRequest(ggufVariant: source.ggufVariant,hfToken: source.hfToken,nativePathToken: source.nativePathToken,- ...settings,+ // An MLX load records none of the llama.cpp settings the selector requires.+ ...(settings ?? {}),nCtx: Math.floor(context),+ ...(mlx+ ? { maxSeqLength: Math.floor(context), mlxKvQuant: mlx.kvQuant }+ : {}),};}diff --git a/studio/frontend/tests/memory-estimate-context.test.ts b/studio/frontend/tests/memory-estimate-context.test.tsindex 9e9c55405bb..01484944808 100644--- a/studio/frontend/tests/memory-estimate-context.test.ts+++ b/studio/frontend/tests/memory-estimate-context.test.ts@@ -1,9 +1,14 @@import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import { register } from "node:module";import { test } from "node:test";import {resolveEstimateContext,resolveEstimateSourceIdentity,+ resolveMlxEstimateContext,+ resolveMlxServedWindow,+ shouldRequestMemoryEstimate,} from "../src/features/model-picker/model-config/estimate-context.ts";// The regression this file exists for: the Context Length control needs a number to@@ -78,15 +83,24 @@ test("two repositories are different sources", () => {});test("two credentials are different sources: they resolve different files", () => {- assert.notEqual(sourceId("org/gated", "Q4_K_M", "aaa"), sourceId("org/gated", "Q4_K_M", "bbb"));+ assert.notEqual(+ sourceId("org/gated", "Q4_K_M", "aaa"),+ sourceId("org/gated", "Q4_K_M", "bbb"),+ );});test("two native picks of the same file name are different sources", () => {- assert.notEqual(sourceId("model.gguf", null, "", "tok-1"), sourceId("model.gguf", null, "", "tok-2"));+ assert.notEqual(+ sourceId("model.gguf", null, "", "tok-1"),+ sourceId("model.gguf", null, "", "tok-2"),+ );});test("absent and null are the same, so an unset variant does not thrash the row", () => {- assert.equal(sourceId("org/a", null), sourceId("org/a", undefined as unknown as null));+ assert.equal(+ sourceId("org/a", null),+ sourceId("org/a", undefined as unknown as null),+ );});// Manual memory mode with GPU Layers on Auto hands context sizing to llama.cpp --fit.@@ -119,3 +133,128 @@ test("every other shape keeps the resident fallback", () => {test("a builtin-default GGUF load prices the fit, not the resident context", () => {assert.equal(resolveEstimateContext(null, 131072, true), 0);});++const gguf = (classifiedIsDiffusion: boolean | undefined) =>+ shouldRequestMemoryEstimate({+ isGguf: true,+ isAppleUnifiedMemory: true,+ classifiedIsDiffusion,+ });++const safetensors = (+ isAppleUnifiedMemory: boolean,+ classifiedIsDiffusion: boolean | undefined,+) =>+ shouldRequestMemoryEstimate({+ isGguf: false,+ isAppleUnifiedMemory,+ classifiedIsDiffusion,+ });++test("a GGUF is priced only once its probe has cleared it of being diffusion", () => {+ assert.equal(gguf(false), true);+ assert.equal(gguf(undefined), false);+});++test("an MLX load is priced without waiting for a GGUF-only probe", () => {+ assert.equal(safetensors(true, undefined), true);+ assert.equal(safetensors(true, false), true);+});++test("a model already known to be diffusion is priced by neither planner", () => {+ assert.equal(gguf(true), false);+ assert.equal(safetensors(true, true), false);+});++test("safetensors is not priced where MLX is not what would load it", () => {+ assert.equal(safetensors(false, false), false);+ assert.equal(safetensors(false, undefined), false);+});++register("./helpers/memory-estimate-resolver.mjs", import.meta.url);++const auth = await import("./helpers/store-stubs/auth.ts");+const native = await import("./helpers/native-path-stub.ts");+const { fetchMemoryEstimate, resetMemoryEstimateRouteMemo } = await import(+ "../src/features/model-picker/api/memory-estimate.ts"+);++let sent: Record<string, unknown> | null = null;++test.beforeEach(() => {+ resetMemoryEstimateRouteMemo();+ native.setNativePathHandler(null);+ sent = null;+ auth.setAuthFetchHandler((_url, init) => {+ sent = init?.body ? JSON.parse(String(init.body)) : null;+ return new Response("{}", {+ status: 200,+ headers: { "Content-Type": "application/json" },+ });+ });+});++test.after(() => {+ resetMemoryEstimateRouteMemo();+ auth.setAuthFetchHandler(null);+ native.setNativePathHandler(null);+});++test("the non-GGUF context and MLX cache width reach the backend", async () => {+ await fetchMemoryEstimate({+ modelPath: "mlx-community/Qwen3-8B-4bit",+ maxSeqLength: 4096,+ mlxKvQuant: "tq-4",+ });+ assert.equal(sent!.max_seq_length, 4096);+ assert.equal(sent!.mlx_kv_quant, "tq-4");+});++test("every field sent is also a field the hook re-fetches for", () => {+ const read = (path: string, fn: string): Set<string> => {+ const source = readFileSync(new URL(path, import.meta.url), "utf8");+ const start = source.indexOf(`function ${fn}(`);+ assert.ok(start >= 0, `${fn} not found in ${path}; this test is stale`);+ const body = source.slice(start, source.indexOf("\n}\n", start));+ return new Set(+ [...body.matchAll(/\b(?:payload|request)\.([A-Za-z0-9_]+)/g)].map(+ (m) => m[1],+ ),+ );+ };+ const wire = read(+ "../src/features/model-picker/api/memory-estimate.ts",+ "estimateRequestBody",+ );+ const key = read(+ "../src/features/model-picker/hooks/use-memory-estimate.ts",+ "estimateKey",+ );+ assert.ok(wire.has("maxSeqLength") && wire.has("mlxKvQuant"), "test is stale");+ const unwatched = [...wire].filter((field) => !key.has(field));+ assert.deepEqual(+ unwatched,+ [],+ `sent to the backend but absent from estimateKey: ${unwatched.join(", ")}`,+ );+});++test("what an MLX estimate names, and what the control shows", () => {+ assert.equal(resolveMlxEstimateContext(null), 0);+ assert.equal(resolveMlxEstimateContext(0), 0);+ assert.equal(resolveMlxEstimateContext(8192), 8192);+ assert.equal(resolveMlxServedWindow(8192, 24576, 262144), 24576);+ assert.equal(resolveMlxServedWindow(null, 24576, 262144), 24576);+ assert.equal(resolveMlxServedWindow(20480, null, 262144), 20480);+ assert.equal(resolveMlxServedWindow(null, null, 262144), 262144);+ assert.equal(resolveMlxServedWindow(null, null, null), null);+});++test("the fitted window survives the wire, and its absence reads as null", async () => {+ const fitted = async (body: unknown) => {+ auth.setAuthFetchHandler(() => Response.json(body));+ return (await fetchMemoryEstimate({ modelPath: "a" })).contextFitted;+ };+ assert.equal(await fitted({ available: true, n_ctx: 24576, context_fitted: 24576 }), 24576);+ assert.equal(await fitted({ available: true, n_ctx: 262144 }), null);+});diff --git a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts b/studio/frontend/tests/memory-estimate-narrow-panel.test.tsindex 6cc19f16a26..c974c686fe5 100644--- a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts+++ b/studio/frontend/tests/memory-estimate-narrow-panel.test.ts@@ -48,6 +48,7 @@ const props: Props = {drafterKvUnsized: false,adaptersUnsized: false,totalBytes: 5.5 * GIB,+ contextFitted: null,gpuBytes: 4.75 * GIB,kvEstimable: true,kvOnGpu: true,diff --git a/studio/frontend/tests/resident-memory-request.test.ts b/studio/frontend/tests/resident-memory-request.test.tsindex bc33520d3b5..79a87e332bf 100644--- a/studio/frontend/tests/resident-memory-request.test.ts+++ b/studio/frontend/tests/resident-memory-request.test.ts@@ -3,6 +3,7 @@import assert from "node:assert/strict";import test from "node:test";+import { readFileSync } from "node:fs";import {resolveResidentEstimateRequest,selectResidentEstimateSettings,@@ -134,3 +135,46 @@ test("fixed Manual layers and automatic layers retain their loaded meaning", ()assert.equal(settings?.nCpuMoe, 4);}});++test("an MLX resident is priced at its served window and loaded cache width", () => {+ const pending = {+ modelPath: "mlx-community/Qwen3-8B-4bit",+ ggufVariant: null,+ };+ const settings = selectResidentEstimateSettings({ ...loaded });+ const request = resolveResidentEstimateRequest(pending, settings, 8192, {+ kvQuant: "4",+ });+ assert.equal(request?.maxSeqLength, 8192);+ assert.equal(request?.mlxKvQuant, "4");+ const auto = resolveResidentEstimateRequest(pending, settings, 8192, {+ kvQuant: null,+ });+ assert.equal(auto?.maxSeqLength, 8192);+ assert.equal(auto?.mlxKvQuant, null);+ // A GGUF resident keeps its old shape: nothing MLX reads is added.+ const gguf = resolveResidentEstimateRequest(pending, settings, 8192);+ assert.ok(gguf && !("maxSeqLength" in gguf) && !("mlxKvQuant" in gguf));+});++test("an MLX resident is credited without the llama.cpp settings it never records", () => {+ const pending = { modelPath: "mlx-community/Qwen3-8B-4bit", ggufVariant: null };+ const request = resolveResidentEstimateRequest(pending, null, 20736, {+ kvQuant: null,+ });+ assert.equal(request?.maxSeqLength, 20736);+ assert.equal(resolveResidentEstimateRequest(pending, null, 20736), null);+ // The page reads an MLX resident's window from loadedContextLength: activeLoadedContext+ // is GGUF-only, and reading it left the MLX branch unreachable.+ const page = readFileSync(+ new URL(+ "../src/features/model-picker/components/model-config-page.tsx",+ import.meta.url,+ ),+ "utf8",+ );+ assert.match(+ page,+ /targetIsMlx && isActiveModel \? loadedContextLength : activeLoadedContext/,+ );+});diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.pyindex d3bb35fc03b..73bf24437b5 100644--- a/tests/studio/test_model_picker_contracts.py+++ b/tests/studio/test_model_picker_contracts.py@@ -1318,8 +1318,8 @@ def test_an_mlx_target_is_offered_a_context_length_not_a_sequence_length():# A number, not a word: the placeholder is only for a window nobody has read.assert 'displayValue={isMlx && windowUnknown ? "—" : undefined}' in pageassert "savedContextPin(config) == null && mlxServedWindow == null\n" in page- # The resident model's window, else this model's; request bounds would shorten it.- assert "(targetIsMlx && isActiveModel ? servedWindow(loadedContextLength) : null) ??" in page+ assert "const mlxServedWindow = resolveMlxServedWindow(" in page+ assert "targetIsMlx && isActiveModel ? servedWindow(loadedContextLength) : null,\n" in pageassert "? servedWindow(modelMaxPosition.maxPositionEmbeddings)" in pageassert re.search(r"const servedWindow = [^;]*Math\.floor\(value\)\n\s*: null;", page), pagenumeric = _read("features/model-picker/components/numeric-value-input.tsx")