Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
feat(studio): MLX PLE and expert offload in Unsloth Studio (#14326)
Long Yixing9880bc0af996
diff --git a/studio/backend/core/inference/memory_contract.py b/studio/backend/core/inference/memory_contract.pyindex 20f8b3937f9..f43d3acdc8b 100644--- a/studio/backend/core/inference/memory_contract.py+++ b/studio/backend/core/inference/memory_contract.py@@ -172,6 +172,15 @@ def build_memory_estimate(layer_count = getattr(breakdown, "layer_count", None),gpu_layers = getattr(breakdown, "gpu_layers", None),moe_offload_unmodelled = bool(moe_offload_unmodelled),+ mlx_offload = (+ None+ if getattr(breakdown, "offloadable", None) is None+ else {+ "ple": breakdown.ple_offload,+ "experts": breakdown.expert_offload,+ "offloadable": list(breakdown.offloadable),+ }+ ),context_is_pinned = bool(context_is_pinned),inherited_device_pin = bool(inherited_device_pin),spec_unpriced = bool(spec_unpriced),@@ -211,6 +220,7 @@ def project_estimate_memory_response(estimate: MemoryEstimate) -> dict:"layer_count": estimate.layer_count,"gpu_layers": estimate.gpu_layers,"moe_offload_unmodelled": estimate.moe_offload_unmodelled,+ "mlx_offload": estimate.mlx_offload,}diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex b4d8f782286..9d4af942017 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -1015,6 +1015,147 @@ def _int8_prefill_status(requested, model):return status+_MLX_OFFLOAD_PARTS = (("ple", "PLE tables"), ("experts", "quantized routed experts"))+++def normalize_mlx_offload(value):+ """An offload knob as a load reads it: ``"auto"`` (also null), the fraction read from disk, or+ None for off."""+ if value in (None, "auto"):+ return "auto"+ return None if value == "off" else float(value)+++def mlx_resolve_offload(+ model_dir,+ knobs,+ *,+ pinned,+ load_in_4bit: bool,+ retains_history: bool,+ kv_bits = None,+ vision = None,+) -> tuple:+ """The ``(ple, expert)`` fractions a load of *model_dir* reads from disk (None for off), from+ knobs as ``normalize_mlx_offload`` gives them: an ``"auto"`` knob decided by unsloth_zoo's 50/50+ rule on this load's pricing, and off where that cannot be planned; any other knob as given.+ ``vision=True`` prices the mlx-vlm route a speculative text load takes."""+ fixed = tuple(None if knob == "auto" else knob for knob in knobs)+ if "auto" not in knobs or not model_dir:+ return fixed+ from core.inference.mlx_memory import mlx_offload_plan, mlx_offload_planner++ if mlx_offload_planner() is None:+ return fixed+ budget = mlx_memory_budget(retains_history = retains_history)+ ceiling = pinned or mlx_checkpoint_window(model_dir)+ if budget is None or not ceiling:+ return fixed+ if kv_bits is not None and mlx_kv_quant_is_refused(model_dir, kv_bits):+ kv_bits = None+ with mlx_rng_preserved():+ try:+ plan = mlx_offload_plan(+ model_dir,+ budget_bytes = budget,+ max_ctx = int(ceiling),+ context = pinned or None,+ ple = knobs[0],+ experts = knobs[1],+ kv_bits = kv_bits,+ load_in_4bit = load_in_4bit,+ vision = vision,+ )+ except Exception as exc:+ logger.info("MLX auto offload not planned for %s: %s", model_dir, exc)+ return fixed+ return fixed if plan is None else plan+++def mlx_checkpoint_window(model_dir) -> Optional[int]:+ from types import SimpleNamespace++ from core.inference.mlx_memory import _snapshot_config++ try:+ config = _snapshot_config(model_dir)+ except Exception:+ return None+ if config is None:+ return None+ native = mlx_native_context_length(SimpleNamespace(config = config))+ return None if native is None else min(int(native), MAX_REQUESTABLE_CONTEXT)+++def _mlx_offload_request(knobs, plan):+ """The offload a load asks for: the ``(ple, experts)`` knobs as sent, the fraction each wants,+ the ``FastMLXModel.from_pretrained`` kwargs, and why none is passed when unsloth_zoo cannot+ offload. unsloth_zoo hands an ``"auto"`` knob to *plan* once it has the checkpoint (the one it+ resolved and downloaded), and never does for a load it cannot offload; with no *plan* it is off."""+ from core.inference.mlx_memory import mlx_offload_planner++ requested = dict(zip(("ple", "experts"), knobs))+ wanted = {part: normalize_mlx_offload(value) for part, value in requested.items()}+ if plan is None:+ wanted = {part: None if value == "auto" else value for part, value in wanted.items()}+ offload = {"requested": requested, "wanted": wanted, "reason": None, "kwargs": {}}+ if not any(wanted.values()):+ return offload+ if mlx_offload_planner() is None:+ offload["reason"] = "the installed unsloth_zoo cannot read weights from disk"+ elif "auto" in wanted.values():++ def decide(model_dir):+ wanted.update(zip(wanted, plan(model_dir, tuple(wanted.values()))))+ offload["decided"] = True+ return dict(wanted)++ offload["decided"] = False+ offload["kwargs"]["offload"] = decide+ else:+ offload["kwargs"]["offload"] = dict(wanted)+ return offload+++def _load_offloading(load, source, load_kwargs, offload):+ """``load(source, **load_kwargs)``, again without offload when unsloth_zoo refuses it for this load+ (an adapter, a bitsandbytes checkpoint, runtime quantization); ``offload`` keeps the reason."""+ try:+ return load(source, **load_kwargs)+ except ValueError as exc:+ # unsloth_zoo's offload refusals all open so.+ if not offload["kwargs"] or not str(exc).startswith("Unsloth: offload "):+ raise+ logger.info("MLX offload not applied to %s: %s", source, exc)+ offload["reason"] = str(exc)+ load_kwargs.pop("offload", None)+ offload["kwargs"] = {}+ return load(source, **load_kwargs)+++def _mlx_offload_status(offload, model):+ """Per knob the fraction read from disk, the request and why it is not applied; and the bytes+ the loaded model reads from disk."""+ reports = getattr(model, "_unsloth_offload", None) or {}+ status = {"disk_bytes": sum(report["on_disk_bytes"] for report in reports.values()) or None}+ for part, what in _MLX_OFFLOAD_PARTS:+ applied = offload["wanted"][part] if part in reports else None+ reason = None+ if offload["wanted"][part] not in (None, "auto") and applied is None:+ reason = offload["reason"] or (+ "unsloth_zoo cannot offload this load (an adapter, a bitsandbytes checkpoint or "+ "runtime quantization)"+ if offload.get("decided") is False+ else f"the model has no {what} unsloth_zoo can read from disk"+ )+ status[part] = {+ "applied": applied,+ "requested": offload["requested"][part],+ "reason": reason,+ }+ return status++# Keyed by checkpoint fingerprint: the panel asks again on every model switch._int8_prefill_checkpoint_cache: dict = {}_int8_prefill_checkpoint_lock = threading.Lock()@@ -2691,6 +2832,7 @@ def mlx_drafter_fit(*,load_in_4bit: bool,costless = False,+ offload = None,):"""``(attaches, fitted context or None)`` for *source*'s drafter on an mlx-vlm load of *model_dir*:only a priced fit reserves it, a pinned context must fit whole, and a ``costless`` one must fit@@ -2710,6 +2852,7 @@ def mlx_drafter_fit(max_ctx = int(ceiling),load_in_4bit = load_in_4bit,vision = True,+ offload = offload,)outcome, fitted = fit(drafter = (source.path, source.builtin))if outcome == "unsizable":@@ -4883,6 +5026,7 @@ class MLXInferenceBackend:spec_draft_n_max = None,spec_draft_model = None,spec_drafters_allowed = None,+ offload = (None, None),) -> bool:import mlx.core as mx@@ -5004,7 +5148,6 @@ class MLXInferenceBackend:load_kwargs["pipeline_group"] = distributed_groupelse:load_kwargs["tensor_group"] = distributed_group-precision_folder = (None if getattr(config, "is_gguf", False) else getattr(config, "gguf_variant", None))@@ -5013,14 +5156,34 @@ class MLXInferenceBackend:from hub.utils.mlx_variants import download_precision_folderload_source = download_precision_folder(model_name, precision_folder, hf_token)+ knobs = offload++ # Auto is planned only where the load is priced: not distributed, an adapter or an explicit dtype.+ def offload_request():+ return _mlx_offload_request(+ knobs,+ None+ if is_distributed or is_lora or dtype is not None+ else functools.partial(+ mlx_resolve_offload,+ pinned = _positive_int(max_seq_length),+ load_in_4bit = load_in_4bit,+ retains_history = not use_vlm or mlx_vlm_snapshot_store_available(),+ kv_bits = None if self._turboquant else _normalize_mlx_kv_bits(kv_bits),+ vision = True if speculates and not is_vision else None,+ ),+ )++ offload = offload_request()+ load_kwargs.update(offload["kwargs"])+# Freed before the replacement weights are allocated, for headroom.self._model_fusion.close()self._clear_prompt_cache()model = Nonetry:- model, tokenizer_or_processor = FastMLXModel.from_pretrained(- load_source,- **load_kwargs,+ model, tokenizer_or_processor = _load_offloading(+ FastMLXModel.from_pretrained, load_source, load_kwargs, offload)except Exception as exc:if is_vision or not use_vlm or is_metal_queue_dead(exc):@@ -5067,9 +5230,11 @@ class MLXInferenceBackend:_drain_generation_streams(mx)mx.clear_cache()load_kwargs["text_only"] = True- model, tokenizer_or_processor = FastMLXModel.from_pretrained(- load_source,- **load_kwargs,+ # Auto planned on the mlx-vlm route, and decide() consumed its knobs: plan this route afresh.+ offload = offload_request()+ load_kwargs.update(offload["kwargs"])+ model, tokenizer_or_processor = _load_offloading(+ FastMLXModel.from_pretrained, load_source, load_kwargs, offload)self._reads_vision = bool(is_vision)@@ -5103,6 +5268,9 @@ class MLXInferenceBackend:_served_ctx, _native_ctx, _max_ctx = self._resolve_context_lengths(self._model, max_seq_length)+ _offload = _mlx_offload_status(offload, self._model)+ _offloaded = (_offload["ple"]["applied"], _offload["experts"]["applied"])+ _offloaded = {"offload": _offloaded} if any(_offloaded) else {}_priceable = not (is_distributed or is_lora or dtype is not None)spec_kind = _drafter_ctx = None_drafter_fitted = False@@ -5135,6 +5303,7 @@ class MLXInferenceBackend:source,load_in_4bit = load_in_4bit,costless = spec_mode == "auto",+ **_offloaded,)(@@ -5171,6 +5340,7 @@ class MLXInferenceBackend:kv_bits = _requested_bits,is_vlm = use_vlm,eligibility = _eligibility,+ **_offloaded,**({"vision": True} if _vlm_route else {}),))@@ -5288,6 +5458,7 @@ class MLXInferenceBackend:"mlx_int8_prefill": self._int8_prefill["active"],"mlx_int8_prefill_requested": self._int8_prefill["requested"],"mlx_int8_prefill_reason": self._int8_prefill["reason"],+ "mlx_offload": _offload,"chat_template_override_requested": self._template_override["requested"],"chat_template_override_reason": self._template_override["reason"],"speculative_type": spec_mode,diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.pyindex 037c65af123..2909688cb2d 100644--- a/studio/backend/core/inference/mlx_memory.py+++ b/studio/backend/core/inference/mlx_memory.py@@ -29,6 +29,7 @@ __all__ = ["mlx_fit_context","mlx_fit_outcome","mlx_memory_breakdown",+ "mlx_offload_plan","mlx_shard_files","mlx_weight_bytes",]@@ -75,6 +76,10 @@ class MlxMemoryBreakdown:kv_on_gpu: bool = Truen_parallel: int = 1gpu_layers: Optional[int] = None+ # The fractions of the PLE tables / routed experts the load reads from disk; None for none.+ ple_offload: Optional[float] = None+ expert_offload: Optional[float] = None+ offloadable: tuple = ()def _snapshot_config(model_dir: str) -> Optional[dict]:@@ -182,13 +187,7 @@ def _checkpoint_tensors(model_dir: str, config: Optional[dict], dtype):tensors = {}for shard in mlx_shard_files(model_dir, config):- with open(shard, "rb") as handle:- length = int.from_bytes(handle.read(8), "little")- limit = min(_MAX_SAFETENSORS_HEADER, os.fstat(handle.fileno()).st_size - 8)- if not 0 < length <= limit:- raise ValueError(f"{shard} declares a {length}-byte safetensors header")- header = json.loads(handle.read(length))- for name, meta in header.items():+ for name, meta in _safetensors_header(shard).items():if name == "__metadata__":continuestored = getattr(mx, _SAFETENSORS_DTYPES.get(meta["dtype"], "float32"))@@ -256,7 +255,9 @@ def _resident_bytes(model_dir: str, config: dict, quantize: bool) -> int:is_vlm = _builds_on_mlx_vlm(config),)return sum(- value.nbytes for _, value in tree_flatten(model.parameters()) if hasattr(value, "nbytes")+ value.nbytes+ for name, value in tree_flatten(model.parameters())+ if hasattr(value, "nbytes") and not _mapped(name, config, quantize))@@ -270,12 +271,55 @@ def mlx_weight_bytes(if not config:return shardsif config.get("quantization") or config.get("quantization_config"):- return shards+ return shards - _mapped_bytes(model_dir, config)try:return _resident_bytes(model_dir, config, quantize = load_in_4bit)except Exception as exc:logger.debug("MLX estimate priced %s as stored: %s", model_dir, exc)- return shards+ return shards - _mapped_bytes(model_dir, config, load_in_4bit)+++def _mapped(+ name: str,+ config: dict,+ quantize: bool = False,+) -> bool:+ """A table mlx-vlm reads row by row from the checkpoint rather than holding it: DeepSeek+ V4.1's Engram, unless the load quantizes it."""+ return not quantize and _maps_engram(config) and ".engram.embed." in name+++def _maps_engram(config: dict) -> bool:+ # mlx-vlm offloads Engram for every deepseek_v41 load, whatever the config says of its layers.+ return "deepseek_v41" in (+ config.get("model_type"),+ (config.get("text_config") or {}).get("model_type"),+ )+++def _mapped_bytes(+ model_dir: str,+ config: dict,+ quantize: bool = False,+) -> int:+ if quantize or not _maps_engram(config):+ return 0+ total = 0+ for shard in mlx_shard_files(model_dir, config):+ for name, meta in _safetensors_header(shard).items():+ if name != "__metadata__" and _mapped(name, config):+ start, end = meta["data_offsets"]+ total += end - start+ return total+++def _safetensors_header(shard: str) -> dict:+ with open(shard, "rb") as handle:+ length = int.from_bytes(handle.read(8), "little")+ limit = min(_MAX_SAFETENSORS_HEADER, os.fstat(handle.fileno()).st_size - 8)+ if not 0 < length <= limit:+ raise ValueError(f"{shard} declares a {length}-byte safetensors header")+ return json.loads(handle.read(length))def _runtime_dtype():@@ -320,6 +364,15 @@ def _loads_as_vision(config: dict) -> bool:return bool(_is_vlm(SimpleNamespace(**config)))+def _on_route(vision: Optional[bool]):+ token = _ROUTE_VISION.set(vision)+ try:+ yield+ finally:+ _ROUTE_VISION.reset(token)++def _builds_on_mlx_vlm(config: dict) -> bool:"""Whether mlx-vlm builds the weights: on a vision route, or for a text model unsloth_zoo buildsthere because mlx-lm cannot; that one is still served as text."""@@ -1065,6 +1118,87 @@ def _refused_by_its_own_tensors(model_dir: str, config: dict) -> Optional[str]:return None[email protected]_cache(maxsize = 1)+def mlx_offload_planner():+ """unsloth_zoo's offload planner, or None when the installed unsloth_zoo cannot offload as+ Studio prices it."""+ import inspect++ try:+ from unsloth_zoo.mlx import offload+ from unsloth_zoo.mlx.loader import FastMLXModel+ hook = "offload" in inspect.signature(FastMLXModel.from_pretrained).parameters+ except Exception:+ return None+ if not (hook and hasattr(offload, "plan_offload") and hasattr(offload, "offloadable")):+ return None+ return offload+++def _lazy_checkpoint(model_dir: str, config: dict):+ """``model_dir``'s model on the route a load takes, its weights mapped but never read."""+ from pathlib import Path++ # mlx-vlm imports a checkpoint's own module unasked; pricing must never run it.+ if config.get("model_file"):+ raise ValueError("this checkpoint carries its own model module")+ # Refuses an index naming a shard outside the snapshot, as every other read here does.+ _indexed_shards(model_dir)+ # Not strict: unsloth_zoo loads past known unused tensors (Gemma 4's). Weightless: mlx-vlm's+ # checkpoint conversion (ModelOpt FP8) evaluates its dense weights even on a lazy load.+ with weightless_forward():+ if _builds_on_mlx_vlm(config):+ from mlx_vlm.utils import load_model+ return load_model(Path(model_dir), lazy = True, strict = False)+ from mlx_lm.utils import load_model+ return load_model(Path(model_dir), lazy = True, strict = False)[0]++[email protected]_cache(maxsize = 4)+def _offloadable_cached(model_dir: str, fingerprint: tuple, cast_dtype, on_mlx_vlm: bool):+ from unsloth_zoo.mlx.offload import offloadable+ return offloadable(+ _lazy_checkpoint(model_dir, _snapshot_config(model_dir)), model_dir, cast_dtype = cast_dtype+ )+++def _offloadable(model_dir: str, config: dict) -> dict:+ """What unsloth_zoo's offload can take from this checkpoint, sized as the rest of the weights+ are: shards as stored, else at the runtime width."""+ quantized = config.get("quantization") or config.get("quantization_config")+ return _offloadable_cached(+ os.path.realpath(model_dir),+ _checkpoint_fingerprint(model_dir),+ None if quantized else _runtime_dtype(),+ _builds_on_mlx_vlm(config),+ )+++def mlx_offloadable_parts(model_dir: str, vision: Optional[bool] = None) -> tuple:+ if mlx_offload_planner() is None:+ return ()+ with _on_route(vision):+ found = _offloadable(model_dir, _snapshot_config(model_dir) or {})+ return tuple(key for key in ("ple", "experts") if found[key] is None)+++def _offloaded(model_dir: str, config: dict, offload: Optional[tuple]) -> tuple:+ """``(weights change, (ple, experts) applied)`` of reading ``offload``'s fractions from disk: what+ unsloth_zoo's offload can take swaps its size for what it holds offloaded, and a knob with+ nothing to take applies nothing."""+ planner = mlx_offload_planner()+ if not offload or not any(offload) or planner is None:+ return 0, (None, None)+ found = _offloadable(model_dir, config)+ applied = tuple(+ value if found[key] is None else None for value, key in zip(offload, ("ple", "experts"))+ )+ delta = (+ int(planner.offload_footprint(found, *applied)) - found["ple_bytes"] - found["expert_bytes"]+ )+ return delta, applied++@dataclassclass _MlxSizing:"""What a load costs before a context is named, so a search over contexts probes once."""@@ -1081,6 +1215,7 @@ class _MlxSizing:drafter_weights: int = 0drafter_plan: tuple = ()capture_per_token: int = 0+ offload: tuple = (None, None)_SIZING_CACHE_MAX = 16@@ -1109,6 +1244,7 @@ def _size_load(load_in_4bit: bool,vision: Optional[bool] = None,drafter: Optional[tuple] = None,+ offload: Optional[tuple] = None,) -> Optional[_MlxSizing]:"""Cached ``_size_load_uncached``: every estimate prices the fit and the breakdown, and everyslider step prices again, each otherwise rebuilding the same weightless architecture."""@@ -1123,14 +1259,19 @@ def _size_load(vision,drafter,drafter and _checkpoint_fingerprint(drafter[0]),+ offload,)except Exception:- return _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)+ return _size_load_uncached(+ model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload+ )with _sizing_lock:if key in _sizing_cache:_sizing_cache.move_to_end(key)return _sizing_cache[key]- sizing = _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)+ sizing = _size_load_uncached(+ model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload+ )with _sizing_lock:_sizing_cache[key] = sizingwhile len(_sizing_cache) > _SIZING_CACHE_MAX:@@ -1141,6 +1282,7 @@ def _size_load(def _clear_sizing_cache() -> None:with _sizing_lock:_sizing_cache.clear()+ _offloadable_cached.cache_clear()def _size_load_uncached(@@ -1150,10 +1292,11 @@ def _size_load_uncached(load_in_4bit: bool,vision: Optional[bool] = None,drafter: Optional[tuple] = None,+ offload: Optional[tuple] = None,) -> Optional[_MlxSizing]:token = _ROUTE_VISION.set(vision)try:- return _sized(model_dir, kv_bits, prefill_chunk, load_in_4bit, drafter)+ return _sized(model_dir, kv_bits, prefill_chunk, load_in_4bit, drafter, offload)finally:_ROUTE_VISION.reset(token)@@ -1164,8 +1307,10 @@ def _sized(prefill_chunk: Optional[int],load_in_4bit: bool,drafter: Optional[tuple],+ offload: Optional[tuple] = None,) -> Optional[_MlxSizing]:- """Everything about a load that does not move with the context, or None if it cannot be sized."""+ """Everything about a load that does not move with the context, or None if it cannot be sized.+ ``offload`` is the ``(ple, experts)`` fractions the load reads from disk."""config = _snapshot_config(model_dir)if config is None:return None@@ -1180,6 +1325,12 @@ def _sized(return Noneif weights <= 0:return None+ try:+ delta, offload = _offloaded(model_dir, config, offload)+ weights += delta+ except Exception as exc:+ logger.debug("MLX estimate cannot size the offload of %s: %s", model_dir, exc)+ return Nonetry:dtype = _runtime_dtype()loaded_chunk, loaded_group = _generation_settings(config)@@ -1208,6 +1359,7 @@ def _sized(drafter_weights = drafted[0],drafter_plan = tuple(drafted[1]),capture_per_token = drafted[2],+ offload = offload,)@@ -1249,6 +1401,8 @@ def _priced_at(sizing: _MlxSizing, n_ctx: int) -> Optional[MlxMemoryBreakdown]:context,chunk,),+ ple_offload = sizing.offload[0],+ expert_offload = sizing.offload[1],)@@ -1261,10 +1415,12 @@ def mlx_memory_breakdown(load_in_4bit: bool = False,vision: Optional[bool] = None,drafter: Optional[tuple] = None,+ offload: Optional[tuple] = None,) -> Optional[MlxMemoryBreakdown]:"""Price an MLX load, or None when it cannot honestly be sized. ``vision`` overrides the loader route- the config implies; ``drafter`` is the ``(path, builtin)`` drafter a speculative load attaches."""- sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)+ the config implies; ``drafter`` is the ``(path, builtin)`` drafter a speculative load attaches;+ ``offload`` the ``(ple, experts)`` fractions it reads from disk."""+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload)return None if sizing is None else _priced_at(sizing, n_ctx)@@ -1279,12 +1435,17 @@ def mlx_fit_outcome(load_in_4bit: bool = False,vision: Optional[bool] = None,drafter: Optional[tuple] = None,+ offload: Optional[tuple] = None,) -> tuple:"""``("unsizable" | "fits" | "fitted" | "no_fit", context)``: the largest context inside- ``budget_bytes`` for ``"fitted"``, else None."""- sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)+ ``budget_bytes`` for ``"fitted"``, else None. An offloaded load keeps unsloth_zoo's offload+ safety margin of the budget free, as its offload planner does."""+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload)if sizing is None:return "unsizable", None+ if any(sizing.offload):+ least, share = mlx_offload_planner().SAFETY_MARGIN+ budget_bytes = min(budget_bytes - least, budget_bytes / (1 + share))ceiling = max(int(max_ctx or 0), MLX_KV_BLOCK)at_ceiling = _priced_at(sizing, ceiling)if at_ceiling is None:@@ -1306,6 +1467,54 @@ def mlx_fit_outcome(return ("no_fit", None) if best is None else ("fitted", (best // MLX_KV_BLOCK) * MLX_KV_BLOCK)+def mlx_offload_plan(+ model_dir: str,+ *,+ budget_bytes: int,+ max_ctx: int,+ context: Optional[int] = None,+ ple = "auto",+ experts = "auto",+ kv_bits: Optional[int] = None,+ prefill_chunk: Optional[int] = None,+ load_in_4bit: bool = False,+ vision: Optional[bool] = None,+) -> Optional[tuple]:+ """The ``(ple, experts)`` fractions unsloth_zoo's 50/50 rule reads from disk for this load, priced+ as Studio prices it; a knob not ``"auto"`` and a pinned ``context`` are kept. When nothing fits,+ everything offloadable goes to disk. None when the load cannot be planned."""+ planner = mlx_offload_planner()+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision)+ if planner is None or sizing is None:+ return None+ with _on_route(vision):+ found = _offloadable(model_dir, _snapshot_config(model_dir))++ def context_cost(tokens, _step):+ priced = _priced_at(sizing, tokens)+ if priced is None:+ raise ValueError(f"cannot price {tokens} tokens")+ return priced.total_bytes - sizing.weights++ plan = planner.plan_offload(+ budget = budget_bytes,+ resident = sizing.weights - found["ple_bytes"] - found["expert_bytes"],+ inventory = found,+ context_cost = context_cost,+ ceiling = max_ctx,+ minimum = MLX_FIT_MIN_CONTEXT,+ context = context,+ prefill_step = sizing.chunk,+ offload = {"ple": ple, "experts": experts},+ )+ if plan["context"] is None:+ return tuple(+ (1.0 if found[key] is None else None) if value == "auto" else value+ for value, key in ((ple, "ple"), (experts, "experts"))+ )+ return plan["ple"], plan["experts"]++def mlx_fit_context(model_dir: str, **kwargs) -> Optional[int]:"""Largest context whose estimated footprint stays inside ``budget_bytes``; None when the ceilingalready fits or nothing can be said."""diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex a7cd3e9f660..ea8f2bae513 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -275,6 +275,7 @@ _MLX_RUNTIME_MIRROR_FIELDS = ("mlx_int8_prefill","mlx_int8_prefill_requested","mlx_int8_prefill_reason",+ "mlx_offload","chat_template_override_requested","chat_template_override_reason","speculative_type",@@ -2082,6 +2083,7 @@ class InferenceOrchestrator:mlx_distributed: bool = False,mlx_kv_quant: Optional[str] = None,mlx_int8_prefill: bool = False,+ mlx_offload: tuple = (None, None),speculative_type: Optional[str] = None,spec_draft_n_max: Optional[int] = None,spec_draft_model: Optional[str] = None,@@ -2159,6 +2161,7 @@ class InferenceOrchestrator:else None,"mlx_kv_quant": mlx_kv_quant,"mlx_int8_prefill": bool(mlx_int8_prefill),+ "mlx_offload": tuple(mlx_offload),"speculative_type": speculative_type,"spec_draft_n_max": spec_draft_n_max,"spec_draft_model": spec_draft_model,diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex c86e18aa34e..0f4328e6cbb 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -815,6 +815,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:load_kwargs["distributed_group"] = config.get("_mlx_distributed_group")load_kwargs["kv_quant"] = config.get("mlx_kv_quant")load_kwargs["int8_prefill"] = bool(config.get("mlx_int8_prefill"))+ load_kwargs["offload"] = tuple(config.get("mlx_offload") or (None, None))load_kwargs["speculative_type"] = config.get("speculative_type")load_kwargs["spec_draft_n_max"] = config.get("spec_draft_n_max")load_kwargs["spec_draft_model"] = config.get("spec_draft_model")@@ -922,6 +923,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:"mlx_int8_prefill","mlx_int8_prefill_requested","mlx_int8_prefill_reason",+ "mlx_offload","chat_template_override_requested","chat_template_override_reason","speculative_type",diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex afceaa2f9ef..b86c9207c6e 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -60,6 +60,19 @@ def resolve_inventory_handle(value: str) -> str:return resolved+MlxOffload = Optional[Union[Literal["auto", "off"], float]]+++def _mlx_offload_amount(value: Any) -> Any:+ """``None`` (the default, as ``"auto"``), ``"auto"``, ``"off"``, or the fraction to read from+ disk, in (0, 1]."""+ if value is None or value in ("auto", "off"):+ return value+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 < value <= 1:+ raise ValueError("must be 'auto', 'off' or the fraction to keep on disk, in (0, 1]")+ return float(value)++class LoadRequest(BaseModel):"""Request to load a model for inference"""@@ -205,6 +218,28 @@ class LoadRequest(BaseModel):"reason otherwise."),)+ mlx_ple_offload: MlxOffload = Field(+ None,+ description = (+ "MLX only: the fraction of the per-layer embedding (PLE) tables read from the "+ "checkpoint on disk instead of held in memory, 'off', or 'auto' (also null): read "+ "from disk only when the load would otherwise leave less than half the memory past "+ "the weights for context. Applied only to a model with PLE tables; the status "+ "reports the reason otherwise."+ ),+ )+ mlx_expert_offload: MlxOffload = Field(+ None,+ description = (+ "MLX only: the fraction of each MoE layer's routed experts read from the checkpoint "+ "on disk on demand instead of held in memory, 'off', or 'auto' (also null): as "+ "mlx_ple_offload, after the PLE tables. Slower decode; applied only to quantized "+ "routed experts."+ ),+ )+ _mlx_offload_amounts = field_validator("mlx_ple_offload", "mlx_expert_offload", mode = "before")(+ _mlx_offload_amount+ )gpu_ids: Optional[List[int]] = Field(None,@@ -1125,6 +1160,11 @@ class EstimateMemoryRequest(BaseModel):"a model that never used it, and reading it here would quantize an estimate ""for a load whose cache stays full width.",)+ mlx_ple_offload: MlxOffload = Field(None, description = "As /load's mlx_ple_offload.")+ mlx_expert_offload: MlxOffload = Field(None, description = "As /load's mlx_expert_offload.")+ _mlx_offload_amounts = field_validator("mlx_ple_offload", "mlx_expert_offload", mode = "before")(+ _mlx_offload_amount+ )_no_booleans = field_validator("n_batch", "n_ubatch", "ctx_checkpoints", "n_ctx", "max_seq_length", mode = "before"@@ -1177,6 +1217,20 @@ class MlxDraftersResponse(BaseModel):drafters: list[MlxDrafter] = Field(default_factory = list)+class MlxOffloadEstimate(BaseModel):+ """What an MLX load reads from disk instead of holding in memory, "auto" decided."""++ ple: Optional[float] = Field(+ None, description = "Fraction of the PLE tables read from disk; null for none"+ )+ experts: Optional[float] = Field(+ None, description = "Fraction of the routed experts read from disk; null for none"+ )+ offloadable: list[Literal["ple", "experts"]] = Field(+ default_factory = list, description = "What the load can read from disk at all"+ )++class EstimateMemoryResponse(BaseModel):"""Itemized memory an inference load would occupy, or why it could not be sized."""@@ -1282,6 +1336,11 @@ class EstimateMemoryResponse(BaseModel):description = "True when --n-cpu-moe is set: experts move per-tensor, not ""per-block, so the GPU/host split above ignores it and reads high.",)+ mlx_offload: Optional[MlxOffloadEstimate] = Field(+ None,+ description = "MLX: what the load reads from disk instead of memory, priced in "+ "weights_bytes; null when not MLX.",+ )class MemoryEstimate(BaseModel):@@ -1389,6 +1448,9 @@ class MemoryEstimate(BaseModel):moe_offload_unmodelled: bool = Field(False, description = "--n-cpu-moe is set, so the GPU figure reads high")+ mlx_offload: Optional[MlxOffloadEstimate] = Field(+ None, description = "MLX: what the load reads from disk; null when not MLX"+ )context_fitted: Optional[int] = Field(None,description = "The window MLX would fit to this machine's memory for a load that "@@ -1467,6 +1529,22 @@ class GenerateRequest(BaseModel):image_base64: Optional[str] = Field(None, description = "Base64 encoded image for vision models")+class MlxOffloadPart(BaseModel):+ applied: Optional[float] = Field(None, description = "Fraction read from disk; null for none")+ requested: Optional[Union[str, float]] = Field(None, description = "As the load asked for it")+ reason: Optional[str] = Field(None, description = "Why a requested offload is not applied")+++class MlxOffloadStatus(BaseModel):+ """The PLE tables and routed experts an MLX load reads from disk instead of memory."""++ ple: MlxOffloadPart+ experts: MlxOffloadPart+ disk_bytes: Optional[int] = Field(+ None, description = "Weight bytes the loaded model reads from the checkpoint on demand"+ )++class _InferenceRuntimeFields(BaseModel):"""Runtime fields shared by load and status responses."""@@ -1749,6 +1827,7 @@ class _InferenceRuntimeFields(BaseModel):description = "Why requested int8 prefill is not active, as unsloth_zoo reports it ""(for example nax_unavailable or no_eligible_projections)",)+ mlx_offload: Optional[MlxOffloadStatus] = Field(None, description = "MLX: weights read from disk")chat_template: Optional[str] = Field(None,description = "Jinja2 chat template string (from GGUF metadata or tokenizer)",diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 0867209d61a..a523da9a57e 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -41,6 +41,7 @@ from typing import (TYPE_CHECKING,Union,)+import dataclassesimport functoolsimport jsonimport re@@ -8793,6 +8794,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:mlx_int8_prefill = None,mlx_int8_prefill_requested = None,mlx_int8_prefill_reason = None,+ mlx_offload = None,mlx_context_budget = None,spec_draft_model = None,chat_template_override_reason = None,@@ -15981,20 +15983,8 @@ def _local_gguf_main_path(config: ModelConfig) -> Optional[str]:def _mlx_estimate_ceiling(model_dir: str) -> Optional[int]:"""The window this checkpoint declares, held to what a load may be asked for."""- from types import SimpleNamespace-- from core.inference.mlx_inference import mlx_native_context_length- from core.inference.mlx_memory import _snapshot_config- from core.inference.runtime_context import MAX_REQUESTABLE_CONTEXT-- try:- config = _snapshot_config(model_dir)- except Exception:- return None- if config is None:- return None- native = mlx_native_context_length(SimpleNamespace(config = config))- return None if native is None else min(int(native), MAX_REQUESTABLE_CONTEXT)+ from core.inference.mlx_inference import mlx_checkpoint_window+ return mlx_checkpoint_window(model_dir)def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_bits, **route):@@ -16019,18 +16009,98 @@ def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_)-def _mlx_estimate_drafter(request, model_identifier, model_dir, ceiling, load_in_4bit: bool):- """``((path, builtin), fitted context, speculates)`` of an MLX load of *request*: the drafter it- would attach, chosen as the load chooses it, and whether it speculates at all (so loads through mlx-vlm)."""+def _mlx_estimate_offload(+ request, config, model_dir: str, load_in_4bit: bool, kv_bits: Optional[int]+) -> tuple[dict, tuple]:+ """``({"offload": (ple, experts)} or {}, offloadable parts)`` for an MLX load of *request*,+ "auto" decided as the load does, on its route; neither if the load quantizes, which unsloth_zoo+ refuses."""+ from core.inference.mlx_inference import (+ mlx_resolve_offload,+ mlx_vlm_snapshot_store_available,+ normalize_mlx_offload,+ )+ from core.inference.mlx_memory import _snapshot_config, mlx_offloadable_parts++ checkpoint = _snapshot_config(model_dir) or {}+ if load_in_4bit and not (+ checkpoint.get("quantization") or checkpoint.get("quantization_config")+ ):+ return {}, ()+ vision = True if _mlx_estimate_speculates(request, model_dir) else None+ try:+ parts = mlx_offloadable_parts(model_dir, vision)+ except Exception as exc:+ logger.info("MLX offload eligibility unknown for %s: %s", model_dir, exc)+ parts = ()+ offload = mlx_resolve_offload(+ model_dir,+ (+ normalize_mlx_offload(request.mlx_ple_offload),+ normalize_mlx_offload(request.mlx_expert_offload),+ ),+ pinned = request.max_seq_length or None,+ load_in_4bit = load_in_4bit,+ # As the load: any mlx-vlm route, a speculative text load included, keeps no history without the store.+ retains_history = (+ not (getattr(config, "is_vision", False) or vision)+ or mlx_vlm_snapshot_store_available()+ ),+ kv_bits = kv_bits,+ vision = vision,+ )+ return ({"offload": offload} if any(offload) else {}), parts+++def _mlx_offload_warning(info: dict) -> Optional[str]:+ """What an MLX load tells the user when its automatic offload put weights on disk."""+ offload = info.get("mlx_offload") or {}+ parts = [+ f"{round(applied * 100)}% of its {what}"+ for part, what in (("ple", "per-layer embedding tables"), ("experts", "MoE experts"))+ if (applied := (offload.get(part) or {}).get("applied"))+ and offload[part]["requested"] in (None, "auto")+ ]+ if not parts:+ return None+ return (+ f"To fit in memory, this model reads {' and '.join(parts)} from disk (automatic offload), "+ "so it generates more slowly. Set offload to Off to keep them in memory."+ )+++def _mlx_estimate_speculates(request, model_dir) -> bool:from core.inference import mlx_speculative- from core.inference.mlx_inference import mlx_drafter_fit, parse_mlx_kv_quant+ from core.inference.mlx_inference import parse_mlx_kv_quantfrom core.inference.mlx_memory import _loads_as_vision, _snapshot_configmode = mlx_speculative.mlx_spec_mode(request.speculative_type)if not mlx_speculative.speculates_on_route(mode, _loads_as_vision(_snapshot_config(model_dir) or {}), request.spec_draft_model):+ return False+ bits, turboquant = parse_mlx_kv_quant(request.mlx_kv_quant)+ legacy_bits = request.mlx_kv_quant is None and request.mlx_kv_bits is not None+ kv_quant = bits is not None or turboquant or legacy_bits+ return not mlx_speculative.speculation_refusal(kv_quant = kv_quant, distributed = False, lora = False)+++def _mlx_estimate_drafter(+ request,+ model_identifier,+ model_dir,+ ceiling,+ load_in_4bit: bool,+ offload_route: Optional[dict] = None,+):+ """``((path, builtin), fitted context, speculates)`` of an MLX load of *request*: the drafter it+ would attach, chosen as the load chooses it, and whether it speculates at all (so loads through mlx-vlm)."""+ from core.inference import mlx_speculative+ from core.inference.mlx_inference import mlx_drafter_fit++ if not _mlx_estimate_speculates(request, model_dir):return None, None, False+ mode = mlx_speculative.mlx_spec_mode(request.speculative_type)resolution = mlx_speculative.resolve_speculation(request.speculative_type,request.spec_draft_model,@@ -16038,11 +16108,6 @@ def _mlx_estimate_drafter(request, model_identifier, model_dir, ceiling, load_intarget_name = model_identifier,allowed = _discoverable_drafters(model_identifier),)- bits, turboquant = parse_mlx_kv_quant(request.mlx_kv_quant)- legacy_bits = request.mlx_kv_quant is None and request.mlx_kv_bits is not None- kv_quant = bits is not None or turboquant or legacy_bits- if mlx_speculative.speculation_refusal(kv_quant = kv_quant, distributed = False, lora = False):- return None, None, Falsefor source in resolution.sources:attaches, fitted = mlx_drafter_fit(model_dir,@@ -16051,6 +16116,7 @@ def _mlx_estimate_drafter(request, model_identifier, model_dir, ceiling, load_insource,load_in_4bit = load_in_4bit,costless = mode == "auto",+ **(offload_route or {}),)if attaches:return (source.path, source.builtin), fitted, True@@ -18409,7 +18475,11 @@ def _mlx_runtime_settings_match(backend, request) -> bool:entry = backend.models.get(backend.active_model_name, {}) or {}if "mlx_kv_quant_requested" not in entry:return True- from core.inference.mlx_inference import encode_mlx_kv_quant, parse_mlx_kv_quant+ from core.inference.mlx_inference import (+ encode_mlx_kv_quant,+ normalize_mlx_offload,+ parse_mlx_kv_quant,+ )from core.inference.mlx_speculative import mlx_spec_mode@@ -18420,6 +18490,13 @@ def _mlx_runtime_settings_match(backend, request) -> bool:== (request.chat_template_override or None)and bool(entry.get("mlx_int8_prefill_requested"))== bool(getattr(request, "mlx_int8_prefill", False))+ and all(+ normalize_mlx_offload(+ ((entry.get("mlx_offload") or {}).get(part) or {}).get("requested")+ )+ == normalize_mlx_offload(getattr(request, f"mlx_{knob}_offload", None))+ for part, knob in (("ple", "ple"), ("experts", "expert"))+ )and entry.get("speculative_type", "auto") == mlx_spec_mode(request.speculative_type)and entry.get("spec_draft_model") == (getattr(request, "spec_draft_model", None) or None)and entry.get("spec_draft_n_max") == _positive_int_or_none(request.spec_draft_n_max)@@ -20196,6 +20273,8 @@ async def _load_model_impl(mlx_int8_prefill = _model_info.get("mlx_int8_prefill"),mlx_int8_prefill_requested = _model_info.get("mlx_int8_prefill_requested"),mlx_int8_prefill_reason = _model_info.get("mlx_int8_prefill_reason"),+ mlx_offload = _model_info.get("mlx_offload"),+ memory_warning = _mlx_offload_warning(_model_info),# Requested, as /status reports it: a null override would read# as "using the default".chat_template_override = _model_info.get("chat_template_override_requested"),@@ -20936,6 +21015,7 @@ async def _load_model_impl(subject = current_subject,mlx_kv_quant = request.mlx_kv_quant,mlx_int8_prefill = request.mlx_int8_prefill,+ mlx_offload = (request.mlx_ple_offload, request.mlx_expert_offload),speculative_type = request.speculative_type,spec_draft_n_max = request.spec_draft_n_max,spec_draft_model = request.spec_draft_model,@@ -21115,6 +21195,8 @@ async def _load_model_impl(mlx_int8_prefill = _model_info.get("mlx_int8_prefill"),mlx_int8_prefill_requested = _model_info.get("mlx_int8_prefill_requested"),mlx_int8_prefill_reason = _model_info.get("mlx_int8_prefill_reason"),+ mlx_offload = _model_info.get("mlx_offload"),+ memory_warning = _mlx_offload_warning(_model_info),# Requested, as /status reports it: a null override would read as# "using the default".chat_template_override = _model_info.get("chat_template_override_requested"),@@ -22621,14 +22703,18 @@ async def estimate_memory(if mlx_kv_bits is not None and mlx_kv_quant_is_refused(model_dir, mlx_kv_bits):mlx_kv_bits = Nonemlx_named_ctx = request.max_seq_length or 0+ mlx_offload, mlx_offloadable = _mlx_estimate_offload(+ request, config, model_dir, mlx_load_in_4bit, mlx_kv_bits+ )mlx_drafter, mlx_fitted_ctx, mlx_speculates = _mlx_estimate_drafter(request,model_identifier,model_dir,_mlx_estimate_ceiling(model_dir),mlx_load_in_4bit,+ mlx_offload,)- mlx_route = {"vision": True} if mlx_speculates else {}+ mlx_route = {**({"vision": True} if mlx_speculates else {}), **mlx_offload}if not mlx_named_ctx and mlx_drafter is None:mlx_fitted_ctx = _mlx_estimate_fitted_context(config, model_dir, mlx_load_in_4bit, mlx_kv_bits, **mlx_route@@ -22649,7 +22735,7 @@ async def estimate_memory(return EstimateMemoryResponse(**project_estimate_memory_response(build_memory_estimate(- mlx_breakdown,+ dataclasses.replace(mlx_breakdown, offloadable = mlx_offloadable),quant_file_bytes = 0,context_fitted = mlx_fitted_ctx,)@@ -24020,6 +24106,8 @@ async def _slot_status(current_subject: str):mlx_int8_prefill = model_info.get("mlx_int8_prefill"),mlx_int8_prefill_requested = model_info.get("mlx_int8_prefill_requested"),mlx_int8_prefill_reason = model_info.get("mlx_int8_prefill_reason"),+ mlx_offload = model_info.get("mlx_offload"),+ memory_warning = _mlx_offload_warning(model_info),chat_template_override = model_info.get("chat_template_override_requested"),chat_template_override_reason = model_info.get("chat_template_override_reason"),loading = _loading_models,diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.pyindex 370586a1660..c24aaf5471d 100644--- a/studio/backend/routes/settings.py+++ b/studio/backend/routes/settings.py@@ -11,7 +11,7 @@ import reimport threadingimport timefrom contextvars import ContextVar-from typing import Annotated, Any, Literal, Optional, get_args+from typing import Annotated, Any, Literal, Optional, Union, get_argsfrom urllib.parse import unquote, urlsplitfrom fastapi import APIRouter, BackgroundTasks, Depends, Header, HTTPException, Request@@ -1083,6 +1083,16 @@ class ModelOverridePayload(BaseModel):tensor_parallel: bool = Falsedisable_vision: bool = Falsemlx_int8_prefill: bool = False+ # Auto (null or "auto") is stored as absence; omitted keeps the stored value.+ mlx_ple_offload: Optional[Union[Literal["auto", "off"], float]] = None+ mlx_expert_offload: Optional[Union[Literal["auto", "off"], float]] = None++ @field_validator("mlx_ple_offload", "mlx_expert_offload", mode = "before")+ @classmethod+ def _mlx_offload_amount(cls, value: Any) -> Any:+ from models.inference import _mlx_offload_amount+ return _mlx_offload_amount(value)+# Default-engine 16-bit load; only False is stored.load_in_4bit: bool = True# Validated in bytes below: pydantic counts characters, so a multi-byte template would pass.@@ -2728,6 +2738,8 @@ def update_openai_auto_switch_override(and not payload.disable_visionand not payload.mlx_int8_prefilland payload.load_in_4bit+ # null clears just that knob, so naming one is not a forget.+ and not {"mlx_ple_offload", "mlx_expert_offload"} & payload.model_fields_setand not {key: valuefor key, value in saved_fields.items()@@ -2805,7 +2817,8 @@ def update_openai_auto_switch_override(+ (() if payload.mirrors_reasoning_budget else _reasoning_fields)+ (() if payload.mirrors_spec_draft_model else _drafter_fields))- if _carried_fields and not is_removal:+ _alias_row: dict = {}+ if not is_removal:# The same spellings the extra-args carry-over walks: a cached repo is not an ordinary folded match,# so a save under the repo id would find nothing and retire the alias with its tuning._alias_ids = [payload.model_id]@@ -2823,12 +2836,12 @@ def update_openai_auto_switch_override(# first non-empty row rather than merging, so filling a gap in the winner from a loser would switch# dormant tuning on.for _alias_id in _alias_ids:- _stored_tuning = get_model_override(_alias_id)- if not _stored_tuning:+ _alias_row = get_model_override(_alias_id)+ if not _alias_row:continuefor name in _carried_fields:if _kept_tuning[name] is None:- _kept_tuning[name] = _stored_tuning.get(name)+ _kept_tuning[name] = _alias_row.get(name)breakremoved_keys: list[str] = []if payload.remove is True:@@ -2938,6 +2951,13 @@ def update_openai_auto_switch_override("engine_gpu_memory_utilization",)},+ # Omitted keeps what the row a load reads holds, which may be an alias retired below.+ **{+ field: getattr(payload, field)+ if field in fields_set or is_removal+ else (_alias_row or get_model_override(target_id)).get(field)+ for field in ("mlx_ple_offload", "mlx_expert_offload")+ },max_seq_length = max_seq_length,custom_context_length = custom_context_length,kv_cache_dtype = payload.kv_cache_dtype,diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.pyindex 5d3b63023fb..c08d1be88ac 100644--- a/studio/backend/tests/test_gguf_load_cache_reuse.py+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py@@ -1187,6 +1187,7 @@ class TestLoadHubDownloadExclusion:"mlx_int8_prefill","mlx_int8_prefill_requested","mlx_int8_prefill_reason",+ "mlx_offload","mlx_context_budget","spec_draft_model","chat_template_override_reason",diff --git a/studio/backend/tests/test_memory_contract.py b/studio/backend/tests/test_memory_contract.pyindex e38a7107279..7a8d75294e7 100644--- a/studio/backend/tests/test_memory_contract.py+++ b/studio/backend/tests/test_memory_contract.py@@ -206,6 +206,19 @@ class TestTheLegacyProjections:)assert out["gpu_bytes"] is None+ def test_the_mlx_offload_the_breakdown_priced_is_reported(self):+ mlx = {"ple_offload": None, "expert_offload": 0.6, "offloadable": ("experts",)}+ offloaded = SimpleNamespace(**{**_BREAKDOWN.__dict__, **mlx})+ panel = project_estimate_memory_response(+ build_memory_estimate(offloaded, quant_file_bytes = 0)+ )+ assert panel["mlx_offload"].model_dump() == {+ "ple": None,+ "experts": 0.6,+ "offloadable": ["experts"],+ }+ assert build_memory_estimate(_BREAKDOWN, quant_file_bytes = 0).mlx_offload is None+def test_the_kv_projection_does_not_borrow_the_planners_kv(self, estimate):assert project_kv_cache_estimate(estimate)["kv_bytes"] is Nonediff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex 8313a247f19..c5db7aab990 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -1105,6 +1105,82 @@ class TestEstimateMemoryRoute:)assert ri._local_mlx_model_dir(config) == str(tmp_path)+ def test_an_mlx_estimate_prices_the_offload_the_load_applies(self, monkeypatch, tmp_path):+ import core.inference.mlx_inference as mlx_inference+ import core.inference.mlx_memory as mlx_memory++ self._mlx_target(monkeypatch, str(tmp_path))+ seen, four_bit, planned = [], {"on": False}, []+ monkeypatch.setattr(ri, "_mlx_estimate_load_in_4bit", lambda *a: four_bit["on"])+ monkeypatch.setattr(+ mlx_memory, "mlx_offloadable_parts", lambda model_dir, vision = None: ("experts",)+ )+ monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: object())+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda retains_history: 1 << 40)+ monkeypatch.setattr(+ mlx_memory, "mlx_offload_plan", lambda d, **kw: planned.append(kw) or (kw["ple"], 0.6)+ )+ monkeypatch.setattr(+ mlx_memory,+ "mlx_memory_breakdown",+ lambda model_dir, *, n_ctx, **kw: seen.append(kw.get("offload"))+ or mlx_memory.MlxMemoryBreakdown(1, 1, 1, 3, 3, n_ctx),+ )+ estimate = lambda **knobs: _estimate(model_path = "org/model", max_seq_length = 4096, **knobs)+ price = lambda **knobs: estimate(**knobs) and seen[-1]+ (tmp_path / "config.json").write_text(json.dumps({"quantization": {"bits": 4}}))+ assert price(mlx_ple_offload = "off", mlx_expert_offload = 0.5) == (None, 0.5)+ off = estimate(mlx_ple_offload = "off", mlx_expert_offload = "off")+ assert off.mlx_offload.offloadable == ["experts"]+ assert price(mlx_ple_offload = "off") == (None, 0.6) and planned[-1]["context"] == 4096+ assert planned[-1]["vision"] is None+ with monkeypatch.context() as patch:+ patch.setattr(ri, "_mlx_estimate_speculates", lambda request, model_dir: True)+ estimate(mlx_ple_offload = "off")+ assert planned[-1]["vision"] is True and len(planned) == 2+ four_bit["on"] = True # already quantized: nothing for the load to quantize, so it offloads+ assert price(mlx_ple_offload = 1.0) == (1.0, 0.6)+ (tmp_path / "config.json").write_text("{}")+ assert price(mlx_ple_offload = 1.0) is None and len(planned) == 3+ off = estimate(mlx_ple_offload = "off", mlx_expert_offload = "off")+ assert off.mlx_offload.offloadable == []+ four_bit["on"] = False+ assert price(mlx_ple_offload = 1.0) == (1.0, 0.6)++ def test_a_speculative_mlx_estimate_plans_offload_without_the_history_reserve(+ self, monkeypatch, tmp_path+ ):+ import core.inference.mlx_inference as mlx_inference+ import core.inference.mlx_memory as mlx_memory++ self._mlx_target(monkeypatch, str(tmp_path))+ (tmp_path / "config.json").write_text(json.dumps({"quantization": {"bits": 4}}))+ reserves = []+ monkeypatch.setattr(ri, "_mlx_estimate_load_in_4bit", lambda *a: False)+ monkeypatch.setattr(+ mlx_memory, "mlx_offloadable_parts", lambda d, vision = None: ("experts",)+ )+ monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: object())+ monkeypatch.setattr(mlx_inference, "mlx_vlm_snapshot_store_available", lambda: False)+ monkeypatch.setattr(+ mlx_inference,+ "mlx_memory_budget",+ lambda retains_history: reserves.append(retains_history) or 1 << 40,+ )+ monkeypatch.setattr(mlx_memory, "mlx_offload_plan", lambda d, **kw: (None, 0.5))+ monkeypatch.setattr(+ mlx_memory,+ "mlx_memory_breakdown",+ lambda model_dir, *, n_ctx, **kw: mlx_memory.MlxMemoryBreakdown(1, 1, 1, 3, 3, n_ctx),+ )+ estimate = lambda: _estimate(model_path = "org/model", max_seq_length = 4096)+ estimate()+ assert reserves[-1] is True+ # The load speculates through mlx-vlm, which keeps no history without the snapshot store.+ monkeypatch.setattr(ri, "_mlx_estimate_speculates", lambda request, model_dir: True)+ estimate()+ assert reserves[-1] is False+def test_an_mlx_estimate_prices_max_seq_length_not_n_ctx(self, monkeypatch):self._mlx_target(monkeypatch, "/models/thing")seen = self._record_breakdown(@@ -3507,6 +3583,209 @@ def test_a_load_is_sized_once_until_its_files_change(monkeypatch, tmp_path):first = sized()(drafter / "config.json").write_text("{}")assert sized() != first # a changed local drafter is re-sized too+ offloaded = lambda fraction: mm._size_load(+ str(tmp_path), 4, None, True, None, None, (None, fraction)+ )+ assert offloaded(0.5) is offloaded(0.5) and offloaded(0.5) != offloaded(0.75)+++def test_offload_is_priced_by_what_unsloth_zoos_offload_takes(tmp_path, monkeypatch):+ offload = pytest.importorskip("unsloth_zoo.mlx.offload")+ if not _HAVE_MLX or not hasattr(offload, "plan_offload"):+ pytest.skip("needs MLX and an unsloth_zoo that offloads")+ mm.mlx_offload_planner.cache_clear()+ planner = mm.mlx_offload_planner()+ assert planner is offload+ import mlx.core as mx+ import mlx.nn as nn+ from mlx.utils import tree_flatten+ from mlx_lm.utils import _get_classes++ config = {+ "model_type": "qwen3_moe",+ "hidden_size": 64,+ "num_hidden_layers": 1,+ "intermediate_size": 64,+ "moe_intermediate_size": 64,+ "num_attention_heads": 2,+ "num_key_value_heads": 1,+ "head_dim": 32,+ "vocab_size": 64,+ "num_experts": 16,+ "num_experts_per_tok": 2,+ "decoder_sparse_step": 1,+ "mlp_only_layers": [],+ "rms_norm_eps": 1e-6,+ "rope_theta": 10000.0,+ "tie_word_embeddings": True,+ "max_position_embeddings": 512,+ "norm_topk_prob": True,+ }+ model_class, args_class = _get_classes(config)+ model = model_class(args_class.from_dict(config))+ nn.quantize(+ model,+ group_size = 32,+ bits = 4,+ class_predicate = lambda path, m: hasattr(m, "to_quantized") and "switch_mlp" in path,+ )+ weights = dict(tree_flatten(model.parameters()))+ # A tensor the model does not instantiate, as Gemma 4 checkpoints carry and unsloth_zoo loads past.+ mx.save_safetensors(+ str(tmp_path / "model.safetensors"), {**weights, "model.unused.scales": mx.zeros(4)}+ )+ config["quantization"] = {"group_size": 32, "bits": 4}+ (tmp_path / "config.json").write_text(json.dumps(config))+ experts = sum(v.nbytes for k, v in weights.items() if ".switch_mlp." in k)+ found = offload.offloadable(mm._lazy_checkpoint(str(tmp_path), config), str(tmp_path))+ assert (found["expert_bytes"], found["experts"]) == (experts, None)+ assert mm.mlx_offloadable_parts(str(tmp_path)) == ("experts",)+ for fractions in ((None, 0.5), (1.0, 1.0)): # no PLE table: its knob applies nothing+ applied = (None, fractions[1])+ delta = planner.offload_footprint(found, *applied) - experts+ assert mm._offloaded(str(tmp_path), config, fractions) == (delta, applied)++ # Auto: the 50/50 rule on this load's own pricing; when nothing fits, all of it goes to disk.+ plan = lambda **kw: mm.mlx_offload_plan(str(tmp_path), max_ctx = 512, **kw)+ asked, decide = {}, planner.plan_offload+ monkeypatch.setattr(planner, "plan_offload", lambda **kw: asked.update(kw) or decide(**kw))+ assert plan(budget_bytes = 1 << 40) == (None, None)+ priced = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 512).total_bytes+ assert asked["resident"] + experts + asked["context_cost"](512, None) == priced+ assert plan(budget_bytes = 1) == (None, 1.0)+ assert plan(budget_bytes = 1 << 40, experts = 0.5) == (None, 0.5)++ mm.mlx_offload_planner.cache_clear()+ with monkeypatch.context() as patch: # an unsloth_zoo that would not hand "auto" to Studio+ patch.setattr(sys.modules["unsloth_zoo.mlx.loader"].FastMLXModel, "from_pretrained", print)+ assert mm.mlx_offload_planner() is None+ mm.mlx_offload_planner.cache_clear()++ mm._clear_sizing_cache()+ seen = []+ kept = {+ **found,+ "expert_bytes": 0,+ "expert_floor_bytes": 0,+ "experts": "its routing top-k is unknown",+ }+ monkeypatch.setattr(+ offload, "offloadable", lambda model, d, cast_dtype: seen.append(cast_dtype) or kept+ )+ assert mm._offloaded(str(tmp_path), config, (None, 1.0)) == (0, (None, None)) and seen == [None]+ mm._offloaded(str(tmp_path), {**config, "quantization": None}, (None, 1.0))+ assert seen[-1] == mm._runtime_dtype()+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: None)+ assert mm._offloaded(str(tmp_path), config, (1.0, 1.0)) == (0, (None, None))+ assert plan(budget_bytes = 1) is None+ (tmp_path / "own.py").write_text(f"open({str(tmp_path / 'ran')!r}, 'w')")+ (tmp_path / "config.json").write_text(json.dumps({**config, "model_file": "own.py"}))+ monkeypatch.setattr(mm, "_loads_as_vision", lambda config: True)+ with pytest.raises(ValueError):+ mm._lazy_checkpoint(str(tmp_path), mm._snapshot_config(str(tmp_path)))+ assert not (tmp_path / "ran").exists()+++def test_auto_offload_is_planned_on_the_route_the_load_takes(monkeypatch):+ routes = []+ decide = lambda inventory, **kw: {+ "context": 512,+ "ple": None if inventory["ple"] else 1.0,+ "experts": 0.5,+ }+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: SimpleNamespace(plan_offload = decide))+ monkeypatch.setattr(+ mm, "_snapshot_config", lambda d: {"model_type": "qwen3_moe", "quantization": {"bits": 4}}+ )+ monkeypatch.setattr(mm, "_checkpoint_fingerprint", lambda d: ())+ monkeypatch.setattr(+ mm,+ "_size_load",+ lambda d, kv, chunk, four, vision = None: routes.append(("sized", vision))+ or SimpleNamespace(weights = 10, chunk = 512),+ )+ monkeypatch.setattr(mm, "_lazy_checkpoint", lambda d, config: mm._builds_on_mlx_vlm(config))+ found = lambda tree: {+ "ple_bytes": 0,+ "expert_bytes": 4,+ "ple": None if tree else "no",+ "experts": None,+ }+ offload = SimpleNamespace(+ offloadable = lambda tree, d, cast_dtype: routes.append(("inventory", tree)) or found(tree)+ )+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.offload", offload)+ mm._clear_sizing_cache()+ for vision in (None, True): # a speculative text load goes through mlx-vlm+ planned = mm.mlx_offload_plan("/x", budget_bytes = 1, max_ctx = 512, vision = vision)+ assert planned == ((1.0, 0.5) if vision else (None, 0.5))+ parts = mm.mlx_offloadable_parts("/x", vision)+ assert parts == (("ple", "experts") if vision else ("experts",))+ assert routes == [("sized", None), ("inventory", False), ("sized", True), ("inventory", True)]+++def test_the_offload_probe_maps_a_text_model_on_the_route_unsloth_zoo_builds_it(monkeypatch):+ if not _HAVE_MLX:+ pytest.skip("resolves the loaders")+ import mlx_lm.utils+ import mlx_vlm.utils++ monkeypatch.setattr(+ mm, "_builds_on_mlx_vlm", lambda config: config["model_type"] == "deepseek_v4"+ )+ monkeypatch.setattr(mlx_vlm.utils, "load_model", lambda path, **kw: "mlx-vlm")+ monkeypatch.setattr(mlx_lm.utils, "load_model", lambda path, **kw: ("mlx-lm", None))+ assert mm._lazy_checkpoint("/x", {"model_type": "deepseek_v4"}) == "mlx-vlm"+ assert mm._lazy_checkpoint("/x", {"model_type": "llama"}) == "mlx-lm"+++def test_the_offload_probe_never_evaluates_a_converted_checkpoint(tmp_path, monkeypatch):+ if not _HAVE_MLX:+ pytest.skip("needs MLX to build and load a checkpoint")+ checkpoint = pytest.importorskip("mlx_vlm.quantization.checkpoint")+ if not hasattr(checkpoint, "transform_quantized_weights"):+ pytest.skip("this mlx-vlm converts no ModelOpt checkpoint")+ import mlx.core as mx+ from mlx.utils import tree_flatten+ from mlx_lm.models import qwen3++ config = {+ "model_type": "qwen3",+ "hidden_size": 128,+ "intermediate_size": 256,+ "num_hidden_layers": 1,+ "num_attention_heads": 2,+ "num_key_value_heads": 1,+ "head_dim": 64,+ "vocab_size": 64,+ "rms_norm_eps": 1e-6,+ "rope_theta": 1e6,+ "max_position_embeddings": 512,+ "tie_word_embeddings": True,+ # mlx-vlm decodes ModelOpt's lone-scale FP8 linears to dense weights, evaluating them.+ "quantization_config": {"quant_method": "modelopt", "quant_algo": "MIXED_PRECISION"},+ }+ (tmp_path / "config.json").write_text(json.dumps(config))+ weights = {}+ for name, value in tree_flatten(qwen3.Model(qwen3.ModelArgs.from_dict(config)).parameters()):+ if name.endswith("proj.weight"):+ weights[name] = mx.zeros(value.shape, dtype = mx.uint8)+ weights[name[: -len("weight")] + "weight_scale"] = mx.array(0.5)+ else:+ weights[name] = value+ mx.save_safetensors(str(tmp_path / "model.safetensors"), weights)+ evaluated = []+ real_eval = mx.eval+ monkeypatch.setattr(mx, "eval", lambda *a, **k: evaluated.append(1) or real_eval(*a, **k))+ with mm._on_route(True):+ mm._lazy_checkpoint(str(tmp_path), json.loads((tmp_path / "config.json").read_text()))+ assert evaluated == []++ (tmp_path / "model.safetensors.index.json").write_text(+ json.dumps({"weight_map": {"lm_head.weight": "../outside.safetensors"}})+ )+ with pytest.raises(ValueError, match = "outside the snapshot"):+ mm._lazy_checkpoint(str(tmp_path), config)try:@@ -4206,6 +4485,25 @@ class TestPricingALoad:config = json.loads((Path(snapshot) / "config.json").read_text())assert mm.mlx_weight_bytes(snapshot, config, load_in_4bit = False) == 2_064_544+ def test_an_offloaded_load_prices_what_the_offload_holds(self, tmp_path, monkeypatch):+ self._checkpoint(tmp_path)+ resident = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 256)+ monkeypatch.setattr(+ mm, "_offloaded", lambda model_dir, config, offload: (-1000 * bool(offload), offload)+ )+ offloaded = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 256, offload = (None, 0.5))+ assert offloaded.weights_bytes == resident.weights_bytes - 1000+ assert offloaded.total_bytes == resident.total_bytes - 1000+ assert (offloaded.ple_offload, offloaded.expert_offload) == (None, 0.5)+ monkeypatch.setattr(+ mm, "mlx_offload_planner", lambda: SimpleNamespace(SAFETY_MARGIN = (2 << 30, 0.03))+ )+ price = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 8192, offload = (None, 0.5)).total_bytes+ fit = lambda offload: mm.mlx_fit_outcome(+ str(tmp_path), budget_bytes = price + (2 << 30), max_ctx = 1 << 20, offload = offload+ )+ assert fit((None, 0.5)) == ("fitted", 8192) != fit(None)+def test_a_quantizing_load_is_priced_below_the_shards(self, tmp_path):config = self._checkpoint(tmp_path)wide = mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = False)@@ -4213,6 +4511,37 @@ class TestPricingALoad:assert (wide, quantized) == (2_886_144, 1_001_984)assert quantized < wide < mm._shard_bytes(str(tmp_path), config) == 2_888_265+ def test_engram_mlx_vlm_reads_from_disk_is_not_priced(self, tmp_path, monkeypatch):+ import mlx.core as mx+ import mlx.nn as nn+ from mlx.utils import tree_flatten++ def build(*_):+ model = nn.Module()+ model.layer = nn.Module()+ model.layer.engram, model.layer.gate = nn.Module(), nn.Linear(128, 64, bias = False)+ model.layer.engram.embed = nn.Embedding(64, 128)+ return model++ monkeypatch.setattr(mm, "_whole_model", build)+ path = str(tmp_path / "model.safetensors")+ mx.save_safetensors(path, dict(tree_flatten(build().parameters())))+ price = lambda q = False, **change: mm.mlx_weight_bytes(+ str(tmp_path), dict(config, **change), q+ )+ config = {"model_type": "deepseek_v41"}+ # Quantized at load, the table becomes a resident one.+ held = 64 * 128 * mm._runtime_dtype().size+ assert price(quantization = {"bits": 2}) == os.path.getsize(path) - 64 * 128 * 4+ assert (price(), price(model_type = "deepseek_v4")) == (held, 2 * held)+ assert price(True) == price(True, model_type = "deepseek_v4")+ assert price(model_type = "deepseek_v4", text_config = {"model_type": "deepseek_v41"}) == held+ monkeypatch.setattr(mm, "_whole_model", lambda *_: 1 / 0)+ assert (price(), price(True)) == (+ os.path.getsize(path) - 64 * 128 * 4,+ os.path.getsize(path),+ )+def test_shards_that_name_nothing_the_architecture_has_are_not_priced_from_it(self, tmp_path):config = self._checkpoint(tmp_path, prefix = "draft.")with pytest.raises(ValueError, match = "does not supply"):@@ -4545,7 +4874,9 @@ class TestTheContextSearch:def _priced(sizing, n_ctx):return None if n_ctx in unpriceable else SimpleNamespace(total_bytes = n_ctx)- monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: object())+ monkeypatch.setattr(+ mm, "_size_load", lambda *a, **kw: SimpleNamespace(offload = (None, None))+ )monkeypatch.setattr(mm, "_priced_at", _priced)def test_the_answer_is_the_largest_whole_block_under_budget(self, monkeypatch):diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 80b46312574..fe8d2f14665 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -1033,6 +1033,289 @@ def test_mlx_inference_text_load_forwards_studio_settings(monkeypatch):assert backend.models["fake/text"]["base_model"] is None+def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatch):+ from core.inference import mlx_inference, mlx_memory+ from core.inference.mlx_inference import MLXInferenceBackend++ _install_fake_mlx(monkeypatch)+ calls, fits = [], []+ _install_fake_fast_mlx(monkeypatch, calls)+ loader = sys.modules["unsloth_zoo.mlx.loader"].FastMLXModel+ stock, refusal = loader.from_pretrained, {"on": False}++ def from_pretrained(+ *args,+ offload = None,+ **kwargs,+ ):+ refused = refusal["on"] and isinstance(offload, dict)+ if callable(offload):+ offload = None if refusal["on"] else offload("/snapshot")+ kwargs.update(+ {"offload": {k: v for k, v in offload.items() if v is not None}} if offload else {}+ )+ model, tokenizer = stock(*args, **kwargs)+ if refused:+ raise ValueError("Unsloth: offload reads the checkpoint's tensors as stored")+ if (offload or {}).get("experts"): # this model has experts, no PLE table+ model._unsloth_offload = {"experts": {"on_disk_bytes": 7}}+ return model, tokenizer++ monkeypatch.setattr(loader, "from_pretrained", staticmethod(from_pretrained))+ monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: object())+ monkeypatch.setattr(+ mlx_inference,+ "_fitted_context",+ lambda *a, **kw: fits.append(kw.get("offload")) or (None, None),+ )+ planned, history = [], [] # here auto takes 0.6 of the experts+ budget = lambda retains_history: history.append(retains_history) or 1 << 40+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", budget)+ monkeypatch.setattr(mlx_inference, "mlx_checkpoint_window", lambda model_dir: 32768)+ monkeypatch.setattr(+ mlx_memory,+ "mlx_offload_plan",+ lambda model_dir, **kw: planned.append((model_dir, kw))+ or (None if kw["ple"] == "auto" else kw["ple"], 0.6),+ )+ config = SimpleNamespace(identifier = "fake/moe", is_vision = False, is_lora = False)++ base = {"max_seq_length": 0, "load_in_4bit": False}++ def load(ple, experts, *more):+ calls.clear()+ backend = MLXInferenceBackend()+ backend.load_model(config, offload = (ple, experts), **{**base, **dict(*more)})+ record = backend.models["fake/moe"]["mlx_offload"]+ return calls[-1][1].get("offload", {}), record++ sent, record = load(0.5, 0.75)+ assert sent == {"ple": 0.5, "experts": 0.75} and fits[-1] == (None, 0.75)+ assert record == {+ "ple": {+ "applied": None,+ "requested": 0.5,+ "reason": "the model has no PLE tables unsloth_zoo can read from disk",+ },+ "experts": {"applied": 0.75, "requested": 0.75, "reason": None},+ "disk_bytes": 7,+ }+ sent, record = load("off", "off")+ assert sent == {} and fits[-1] is None and record["ple"]["reason"] is None+ assert not planned+ sent, record = load(None, None)+ assert sent == {"experts": 0.6} and fits[-1] == (None, 0.6) and history == [True]+ pick = lambda *keys: tuple(planned[-1][1][key] for key in keys)+ assert planned[-1][0] == "/snapshot" and pick("ple", "experts") == ("auto", "auto")+ assert pick("context", "max_ctx", "kv_bits", "load_in_4bit", "vision") == (+ None,+ 32768,+ None,+ False,+ None,+ )+ assert (record["experts"]["applied"], record["experts"]["requested"]) == (0.6, None)+ with monkeypatch.context() as patch: # a speculative text load is priced on mlx-vlm+ from core.inference import mlx_speculative++ patch.setattr(mlx_speculative, "speculates_on_route", lambda *a: True)+ patch.setattr(mlx_speculative, "speculation_refusal", lambda **k: None)+ patch.setattr(+ mlx_speculative,+ "resolve_speculation",+ lambda mode, *a, **k: mlx_speculative.SpecResolution(mode),+ )+ patch.setattr(mlx_speculative, "build_draft", lambda *a, **k: (None, None, None, None))+ load(None, None)+ assert planned[-1][1]["vision"] is True+ load("off", None, {"max_seq_length": 8192}) # an explicit off is planned as off+ assert pick("ple", "context", "max_ctx") == (None, 8192, 8192)+ sent, record = load(None, None, {"dtype": "float16"}) # not priced, so not planned: off+ assert sent == {} and len(planned) == 3 and record["experts"]["reason"] is None+ sent, record = load(0.5, None) # decided: the hand-set part the model lacks says so+ assert sent == {"ple": 0.5, "experts": 0.6} and record["ple"]["reason"].startswith(+ "the model has"+ )+ # Refused by unsloth_zoo for this load: loaded again without it, saying why; auto turns off.+ refusal["on"] = True+ sent, record = load("off", 1.0)+ assert sent == {} and len(calls) == 2 and record["experts"]["applied"] is None+ assert record["experts"]["reason"].startswith("Unsloth: offload reads")+ sent, record = load(None, None)+ assert sent == {} and not (record["experts"]["applied"] or record["experts"]["reason"])+ # A hand-set part beside "auto" goes off with the planner unsloth_zoo never called, saying why.+ sent, record = load(1.0, None)+ assert (+ sent == {}+ and len(calls) == 1+ and record["ple"]["reason"].startswith("unsloth_zoo cannot offload this load")+ )+ refusal["on"] = False+ monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: None)+ sent, record = load("off", 1.0)+ assert (+ sent == {}+ and record["experts"]["reason"] == "the installed unsloth_zoo cannot read weights from disk"+ )+++def test_auto_offload_is_planned_as_the_load_is_priced(monkeypatch):+ from core.inference import mlx_inference as mi+ from core.inference import mlx_memory as mm++ plans = []+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: object())+ monkeypatch.setattr(mm, "mlx_offload_plan", lambda d, **kw: plans.append(kw) or (1.0, 0.5))+ monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: 1 << 40)+ monkeypatch.setattr(mi, "mlx_checkpoint_window", lambda d: 32768)+ monkeypatch.setattr(mi, "mlx_kv_quant_is_refused", lambda d, bits: True)+ priced = {"pinned": None, "load_in_4bit": False, "retains_history": True, "kv_bits": 2}+ auto = lambda at, *knobs: mi.mlx_resolve_offload(at, knobs, **priced)+ assert auto("/snap", "auto", "auto") == (1.0, 0.5) and plans[-1]["vision"] is None+ assert (plans[-1]["budget_bytes"], plans[-1]["kv_bits"]) == (1 << 40, None)+ mi.mlx_resolve_offload("/snap", ("auto", None), vision = True, **priced)+ assert plans[-1]["vision"] is True+ plans.pop()+ assert auto("/snap", 0.5, None) == (0.5, None) and auto(None, "auto", "auto") == (None, None)+ monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: None)+ assert auto("/snap", "auto", "auto") == (None, None) and len(plans) == 1+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: None)+ monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: 1 / 0)+ assert auto("/snap", "auto", 0.5) == (None, 0.5) and len(plans) == 1++[email protected]("decided_on_mlx_vlm", [False, True])+def test_a_load_falling_back_to_mlx_lm_plans_auto_offload_again(monkeypatch, decided_on_mlx_vlm):+ pytest.importorskip("mlx.core")+ from core.inference import mlx_inference as mi+ from core.inference import mlx_memory as mm+ from core.inference import mlx_speculative++ plans, applied = [], []++ class _Loader:+ @staticmethod+ def from_pretrained(name, **kwargs):+ offload = kwargs.get("offload")+ if not kwargs["text_only"]:+ if decided_on_mlx_vlm and callable(offload):+ offload("/snap")+ raise ValueError("mlx-vlm has no such architecture")+ applied.append(offload("/snap") if callable(offload) else offload)+ return SimpleNamespace(config = {}), SimpleNamespace()++ def resolve(model_dir, knobs, **kw):+ plans.append((knobs, kw["vision"]))+ # A plan for the mlx-vlm route differs from the one mlx-lm needs, so a reused plan shows.+ return (None, 0.25) if kw["vision"] else (None, 1.0)++ loader = types.ModuleType("unsloth_zoo.mlx.loader")+ loader.FastMLXModel = _Loader+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: object())+ monkeypatch.setattr(mi, "mlx_resolve_offload", resolve)+ monkeypatch.setattr(mi, "_classify_mlx_audio_type", lambda *a, **k: None)+ monkeypatch.setattr(+ mi.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (8192, 8192, 8192)+ )+ source = mlx_speculative.DrafterSource("dflash", "/drafter", False)+ monkeypatch.setattr(mlx_speculative, "speculation_refusal", lambda **_k: None)+ monkeypatch.setattr(+ mlx_speculative,+ "resolve_speculation",+ lambda mode, *_a, **_k: mlx_speculative.SpecResolution(mode, (source,)),+ )+ config = SimpleNamespace(identifier = "org/moe", is_vision = False, is_lora = False)++ mi.MLXInferenceBackend().load_model(config, speculative_type = "dflash")+ assert plans[-1] == (("auto", "auto"), None)+ assert applied == [{"ple": None, "experts": 1.0}]+++def test_offload_knobs_validate_and_reload_on_change():+ from pydantic import ValidationError++ from core.inference import orchestrator+ from models.inference import EstimateMemoryRequest, LoadRequest+ from routes.inference import _mlx_runtime_settings_match++ req = lambda **knobs: LoadRequest(model = "m", model_path = "m", **knobs)+ assert (+ req(mlx_ple_offload = 1).mlx_ple_offload,+ req(mlx_expert_offload = "off").mlx_expert_offload,+ ) == (1.0, "off")+ assert req(mlx_ple_offload = "auto").mlx_ple_offload == "auto"+ for bad in (0, 1.5, -0.5, True, "on", "0.5"):+ for model in (LoadRequest, EstimateMemoryRequest):+ with pytest.raises(ValidationError):+ model(+ model_path = "m",+ **({"model": "m"} if model is LoadRequest else {}),+ mlx_expert_offload = bad,+ )+ be = SimpleNamespace(active_model_name = "m", models = {"m": {"mlx_kv_quant_requested": "auto"}})+ assert _mlx_runtime_settings_match(be, req(mlx_ple_offload = "auto", mlx_expert_offload = None))+ assert not _mlx_runtime_settings_match(be, req(mlx_ple_offload = "off"))+ assert not _mlx_runtime_settings_match(be, req(mlx_expert_offload = 0.5))+ be.models["m"]["mlx_offload"] = {"ple": {"requested": None}, "experts": {"requested": 0.5}}+ assert _mlx_runtime_settings_match(be, req(mlx_expert_offload = 0.5))+ assert not _mlx_runtime_settings_match(be, req(mlx_expert_offload = 0.75))+ assert not _mlx_runtime_settings_match(be, req(mlx_expert_offload = 0.5, mlx_ple_offload = 1.0))+ import inspect++ from routes import inference as routes++ part = lambda applied, requested: {"applied": applied, "requested": requested}+ info = {"mlx_offload": {"ple": part(1.0, 1.0), "experts": part(0.6, None)}}+ assert "reads 60% of its MoE experts from disk" in routes._mlx_offload_warning(info)+ info["mlx_offload"]["experts"]["requested"] = 0.6+ assert routes._mlx_offload_warning(info) is None+ assert inspect.getsource(routes).count("memory_warning = _mlx_offload_warning(") == 3+ load = inspect.getsource(orchestrator.InferenceOrchestrator.load_model)+ assert "mlx_offload" in orchestrator._MLX_RUNTIME_MIRROR_FIELDS+ assert '"mlx_offload": tuple(mlx_offload)' in load # carried to the worker+++def test_the_worker_carries_the_offload_knobs_into_the_load_and_back(monkeypatch):+ from core.inference import worker++ model = SimpleNamespace(+ identifier = "org/moe",+ display_name = "moe",+ is_vision = False,+ is_lora = False,+ base_model = None,+ is_audio = False,+ audio_type = None,+ has_audio_input = False,+ )+ asked, sent = {}, []+ entry = {"mlx_offload": {"experts": {"applied": 0.75, "requested": 0.75}, "disk_bytes": 7}}+ backend = SimpleNamespace(+ device = "mlx",+ active_model_name = model.identifier,+ models = {model.identifier: entry},+ load_model = lambda **kw: asked.update(kw) or True,+ )+ monkeypatch.setattr(worker, "_build_model_config", lambda config: model)+ monkeypatch.setattr(worker, "_run_security_gates", lambda *a, **k: True)+ monkeypatch.setattr(worker, "_resolve_lora_4bit", lambda *a: False)+ monkeypatch.setattr(worker, "_needs_nemotron_trust", lambda *a, **k: False)+ xet = types.ModuleType("utils.hf_xet_fallback")+ xet.start_watchdog = lambda **k: SimpleNamespace(set = lambda: None)+ monkeypatch.setitem(sys.modules, "utils.hf_xet_fallback", xet)+ queue = SimpleNamespace(put = lambda item, *a, **k: sent.append(item))++ worker._handle_load(+ backend,+ {"model_name": model.identifier, "mlx_offload": ("off", 0.75)},+ queue,+ )+ assert asked["offload"] == ("off", 0.75)+ (loaded,) = [item for item in sent if item.get("type") == "loaded"]+ assert {k: loaded["model_info"][k] for k in entry} == entry++@pytest.mark.parametrize("is_lora", [False, True])def test_mlx_base_fusion_lifetime_and_lora_request_scope(monkeypatch, mlx_moe, is_lora):backend = _install_fake_text_stack(monkeypatch, {"p": [1, 2], "generated": [7, 8]}, [])diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 9e04e304df5..a5c191dd66f 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -7607,6 +7607,19 @@ def test_the_retired_snapshot_path_entry_hands_over_its_launch_flags(override_stassert f"{_LEGACY_SNAPSHOT}:Q4_K_M" not in resp.overrides+def test_the_retired_snapshot_path_entry_hands_over_its_mlx_offload(override_store):+ # A client that predates the offload knobs omits them, so retiring the row a load read keeps them.+ settings.set_model_override(f"{_LEGACY_SNAPSHOT}:Q4_K_M", mlx_ple_offload = 1.0)++ resp = _put("unsloth/B-GGUF:Q4_K_M", max_seq_length = 32768)+ entry = resp.overrides["unsloth/B-GGUF:Q4_K_M"]+ assert entry["mlx_ple_offload"] == 1.0 and entry["max_seq_length"] == 32768+ assert f"{_LEGACY_SNAPSHOT}:Q4_K_M" not in resp.overrides+ # An explicit null still resets it.+ resp = _put("unsloth/B-GGUF:Q4_K_M", max_seq_length = 32768, mlx_ple_offload = None)+ assert "mlx_ple_offload" not in resp.overrides["unsloth/B-GGUF:Q4_K_M"]++def test_a_standalone_gguf_save_keeps_its_filename_label_launch_flags(override_store):path = "/models/Qwen3-8B-Q4_K_M.gguf"settings.set_model_override(@@ -9027,6 +9040,15 @@ def test_mlx_int8_prefill_is_stored_only_when_on_and_reaches_the_load(monkeypatcassert "org/m" not in settings.get_model_overrides()+def test_mlx_offload_is_stored_unless_auto_and_survives_a_client_that_omits_it(monkeypatch):+ _mock_override_store(monkeypatch)+ knobs = {"mlx_ple_offload": "off", "mlx_expert_offload": 0.5}+ row = _put("m", **knobs).overrides["m"]+ assert settings.model_override_load_kwargs(row, is_gguf = False) == row == knobs+ assert _put("m", mlx_expert_offload = None).overrides["m"] == {"mlx_ple_offload": "off"}+ assert "m" not in _put("m", mlx_ple_offload = "auto").overrides++def _idle_backend(kw, monkeypatch, *, user_loaded):"""A loaded, long-idle GGUF backend wired into the idle loop."""_reset_keepwarm()diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.pyindex 90c4ce9f1dc..341c0399c17 100644--- a/studio/backend/utils/openai_auto_switch_settings.py+++ b/studio/backend/utils/openai_auto_switch_settings.py@@ -550,6 +550,13 @@ def normalize_model_override(if _coerce_bool(payload.get("mlx_int8_prefill")):entry["mlx_int8_prefill"] = True+ for key in ("mlx_ple_offload", "mlx_expert_offload"):+ value = payload.get(key)+ if value == "off" or (+ isinstance(value, (int, float)) and not isinstance(value, bool) and 0 < value <= 1+ ):+ entry[key] = value if value == "off" else float(value)+# Stored only when off: absent is the 4-bit default, so an untouched model follows it.if payload.get("load_in_4bit") is False:entry["load_in_4bit"] = False@@ -699,6 +706,8 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di("disable_vision", "disable_vision"),("chat_template_override", "chat_template_override"),("mlx_int8_prefill", "mlx_int8_prefill"),+ ("mlx_ple_offload", "mlx_ple_offload"),+ ("mlx_expert_offload", "mlx_expert_offload"),):if override.get(source) is not None and source not in gguf_drops:kwargs[target] = override[source]diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.tsindex 9e2f53490df..ea39bba36de 100644--- a/studio/frontend/scripts/check-bundle-budget.ts+++ b/studio/frontend/scripts/check-bundle-budget.ts@@ -385,8 +385,10 @@ export const BUDGET = {// transfer (Node 26, 98 chunks). Raised to that plus about 200 B.// Desktop Chat only upgrade (#14593), incl. the Train row opening the full install dialog, merged with main// at c5a9fe021: 6,912,687 B raw / 2,082,221 B transfer (Node 24, 97 chunks). Raised to that plus about 200 B.- transferBytes: 2_082_421,- rawBytes: 6_912_887,+ // The MLX PLE and expert offload controls (#14326): +7,406 B raw / +2,073 B transfer, plus about 500 B raw+ // (a CI build measured about 100 B over a local one). Raised by that.+ transferBytes: 2_084_696,+ rawBytes: 6_920_821,};// The chunk count is reported but not budgeted. Splitting a page out of the entry raises it while lowering thediff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex 4a235e9e3e8..49f3f424216 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -2777,6 +2777,15 @@ const VISIBLE_MODEL_RUNTIME_KEYS = ["mlxKvQuantNote","mlxInt8Prefill","loadedMlxInt8PrefillRequested",+ "mlxPleOffload",+ "mlxExpertOffload",+ "loadedMlxPleOffloadRequested",+ "loadedMlxExpertOffloadRequested",+ "loadedMlxPleOffload",+ "loadedMlxExpertOffload",+ "mlxPleOffloadReason",+ "mlxExpertOffloadReason",+ "loadedMlxOffloadDiskBytes","loadedContextBudget","loadedIsMultimodal","loadedIsDiffusion",@@ -3859,6 +3868,8 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{cache_type_kv: config.kvCacheDtype,mlx_kv_quant: config.mlxKvQuant ?? null,mlx_int8_prefill: config.mlxInt8Prefill ?? false,+ mlx_ple_offload: config.mlxPleOffload ?? null,+ mlx_expert_offload: config.mlxExpertOffload ?? null,speculative_type: effectiveSpeculativeType,spec_draft_n_max: effectiveSpecDraftNMax,...mlxDrafter,diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex 7743b1b50f1..15e1594d3f7 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -2117,6 +2117,12 @@ export function useChatModelRuntime() {const previousMlxInt8Prefill = rollbackConfig? (rollbackConfig.mlxInt8Prefill ?? false): useChatRuntimeStore.getState().mlxInt8Prefill;+ const previousMlxPleOffload = rollbackConfig+ ? (rollbackConfig.mlxPleOffload ?? null)+ : useChatRuntimeStore.getState().mlxPleOffload;+ const previousMlxExpertOffload = rollbackConfig+ ? (rollbackConfig.mlxExpertOffload ?? null)+ : useChatRuntimeStore.getState().mlxExpertOffload;if (isGguf && isDiffusion === undefined) {// Prepare the token exactly as validateModel/loadModel do: the Hub rejects an invalid// Authorization header with 401 even for a public repo, so sending the raw stored token here would@@ -2219,6 +2225,12 @@ export function useChatModelRuntime() {let loadMlxInt8Prefill = pendingLoadConfig? (pendingLoadConfig.mlxInt8Prefill ?? false): stateBeforeUnload.mlxInt8Prefill;+ let loadMlxPleOffload = pendingLoadConfig+ ? (pendingLoadConfig.mlxPleOffload ?? null)+ : stateBeforeUnload.mlxPleOffload;+ let loadMlxExpertOffload = pendingLoadConfig+ ? (pendingLoadConfig.mlxExpertOffload ?? null)+ : stateBeforeUnload.mlxExpertOffload;// gpuMemoryMode is a standing preference; the rest are per-model knobs the reset below clears, so// they are re-baselined there in lock-step with the store. A GGUF native context can exceed// maxSeqLength, so sizing on raw maxSeqLength could pass, unload, then have /load refuse it. A@@ -2727,6 +2739,8 @@ export function useChatModelRuntime() {};loadMlxKvQuant = pendingLoadConfig?.mlxKvQuant ?? null;loadMlxInt8Prefill = pendingLoadConfig?.mlxInt8Prefill ?? false;+ loadMlxPleOffload = pendingLoadConfig?.mlxPleOffload ?? null;+ loadMlxExpertOffload = pendingLoadConfig?.mlxExpertOffload ?? null;loadChatTemplateOverride =pendingLoadConfig?.chatTemplateOverride?.trim()? pendingLoadConfig.chatTemplateOverride@@ -2831,6 +2845,8 @@ export function useChatModelRuntime() {cache_type_kv: loadKvCacheDtype,mlx_kv_quant: loadMlxKvQuant ?? null,mlx_int8_prefill: loadMlxInt8Prefill,+ mlx_ple_offload: loadMlxPleOffload,+ mlx_expert_offload: loadMlxExpertOffload,speculative_type: loadSpeculativeType,spec_draft_n_max: loadSpecDraftNMax,...(targetIsMlx ? { spec_draft_model: loadSpecDraftModel } : {}),@@ -3228,6 +3244,8 @@ export function useChatModelRuntime() {cache_type_kv: rollbackState.loadedKvCacheDtype,mlx_kv_quant: rollbackState.loadedMlxKvQuantRequested,mlx_int8_prefill: rollbackState.loadedMlxInt8PrefillRequested,+ mlx_ple_offload: rollbackState.loadedMlxPleOffloadRequested,+ mlx_expert_offload: rollbackState.loadedMlxExpertOffloadRequested,speculative_type:rollbackState.loadedSpeculativeType,spec_draft_n_max:@@ -3337,6 +3355,8 @@ export function useChatModelRuntime() {// nParallel above.mlxKvQuant: previousMlxKvQuant,mlxInt8Prefill: previousMlxInt8Prefill,+ mlxPleOffload: previousMlxPleOffload,+ mlxExpertOffload: previousMlxExpertOffload,loadedChatTemplateOverride:rollbackState.loadedChatTemplateOverride,...loadedGpuMemoryFields(rollbackResponse),diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.tsindex 16f26e6ef66..02e38f13655 100644--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts@@ -45,6 +45,7 @@ import { resolveBatchSizeSeed } from "./resolve-batch-size-seed";import { resolveChatTemplateSeed } from "./resolve-chat-template-seed";import { resolveCtxPinSeed } from "./resolve-ctx-pin-seed";import { engineLaunchFieldsFrom } from "./engine-launch-fields";+import { mlxOffloadStateFrom } from "./mlx-runtime-state";import { resolveLlamaExtraArgsSeed } from "./resolve-llama-extra-args-seed";import { shouldSeedVisionSwitch } from "./resolve-vision-switch-seed";@@ -245,6 +246,7 @@ export function applyActiveModelStatusToStore(const currentSpecType = normalizeSpeculativeType(status.speculative_type);const incomingMlxKvQuant = normalizeMlxKvQuant(status.mlx_kv_quant_requested);const incomingMlxInt8Prefill = status.mlx_int8_prefill_requested === true;+ const incomingMlxOffload = mlxOffloadStateFrom(status);const prevState = useChatRuntimeStore.getState();// The chat's own level when the model this replaces was running a per-model pin's: the clamp// narrows the chat's level to this model's ladder, and a pin is one model's, not the chat's.@@ -584,6 +586,17 @@ export function applyActiveModelStatusToStore(status.chat_template_override_reason ?? null,mlxKvQuantNote: status.mlx_kv_quant_note ?? null,loadedMlxInt8PrefillRequested: incomingMlxInt8Prefill,+ ...incomingMlxOffload,+ ...((hydratingExistingModel ||+ (prevState.mlxPleOffload ?? null) ===+ (prevState.loadedMlxPleOffloadRequested ?? null)) && {+ mlxPleOffload: incomingMlxOffload.loadedMlxPleOffloadRequested,+ }),+ ...((hydratingExistingModel ||+ (prevState.mlxExpertOffload ?? null) ===+ (prevState.loadedMlxExpertOffloadRequested ?? null)) && {+ mlxExpertOffload: incomingMlxOffload.loadedMlxExpertOffloadRequested,+ }),...((hydratingExistingModel ||(prevState.mlxKvQuant ?? null) ===(prevState.loadedMlxKvQuantRequested ?? null)) && {@@ -602,6 +615,7 @@ export function applyActiveModelStatusToStore(chatTemplateOverrideReason: null,mlxKvQuantNote: null,loadedMlxInt8PrefillRequested: false,+ ...incomingMlxOffload,})),// Baseline only, never the control: the echo is the RESOLVED count and would pin a blank...(seedLoadParams &&diff --git a/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts b/studio/frontend/src/features/chat/lib/mlx-runtime-state.tsindex 5dee433b4fb..19e7f48eaa9 100644--- a/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts+++ b/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts@@ -3,7 +3,9 @@import {normalizeMlxKvQuant,+ normalizeMlxOffload,type MlxKvQuant,+ type MlxOffload,} from "@/features/model-picker/model-config/per-model-config";import type { LoadModelResponse } from "../types/api";@@ -15,10 +17,39 @@ type MlxRuntimeResponse = Pick<| "chat_template_override_reason"| "mlx_kv_quant_note"| "mlx_int8_prefill_requested"+ | "mlx_offload">;-/** MLX runtime state a load response establishes (KV quantization, template verdict, int8 prefill). A- * non-MLX response retires the verdicts but omits mlxKvQuant and mlxInt8Prefill: the settings are+function appliedFraction(value: unknown): number | null {+ const fraction = normalizeMlxOffload(value);+ return typeof fraction === "number" ? fraction : null;+}++export function mlxOffloadStateFrom(resp: Partial<MlxRuntimeResponse>): {+ loadedMlxPleOffloadRequested: MlxOffload;+ loadedMlxExpertOffloadRequested: MlxOffload;+ loadedMlxPleOffload: number | null;+ loadedMlxExpertOffload: number | null;+ mlxPleOffloadReason: string | null;+ mlxExpertOffloadReason: string | null;+ loadedMlxOffloadDiskBytes: number | null;+} {+ const offload = resp.is_mlx === true ? resp.mlx_offload : null;+ const disk = offload?.disk_bytes;+ return {+ loadedMlxPleOffloadRequested: normalizeMlxOffload(offload?.ple?.requested),+ loadedMlxExpertOffloadRequested: normalizeMlxOffload(offload?.experts?.requested),+ loadedMlxPleOffload: appliedFraction(offload?.ple?.applied),+ loadedMlxExpertOffload: appliedFraction(offload?.experts?.applied),+ mlxPleOffloadReason: offload?.ple?.reason ?? null,+ mlxExpertOffloadReason: offload?.experts?.reason ?? null,+ loadedMlxOffloadDiskBytes:+ typeof disk === "number" && Number.isFinite(disk) && disk > 0 ? disk : null,+ };+}++/** MLX runtime state a load response establishes (KV quantization, template verdict, int8 prefill, offload). A+ * non-MLX response retires the verdicts but omits the chosen settings: they are* dormant there, not wrong, and a preset carrying the width must survive the round-trip. */export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {mlxKvQuant?: MlxKvQuant | null;@@ -28,7 +59,10 @@ export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {mlxKvQuantNote: string | null;mlxInt8Prefill?: boolean;loadedMlxInt8PrefillRequested: boolean;-} {+ mlxPleOffload?: MlxOffload;+ mlxExpertOffload?: MlxOffload;+} & ReturnType<typeof mlxOffloadStateFrom> {+ const offload = mlxOffloadStateFrom(resp);if (resp.is_mlx !== true) {return {loadedMlxKvQuantRequested: null,@@ -36,6 +70,7 @@ export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {chatTemplateOverrideReason: null,mlxKvQuantNote: null,loadedMlxInt8PrefillRequested: false,+ ...offload,};}const requested = normalizeMlxKvQuant(resp.mlx_kv_quant_requested);@@ -47,5 +82,8 @@ export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {mlxKvQuantNote: resp.mlx_kv_quant_note ?? null,mlxInt8Prefill: resp.mlx_int8_prefill_requested === true,loadedMlxInt8PrefillRequested: resp.mlx_int8_prefill_requested === true,+ mlxPleOffload: offload.loadedMlxPleOffloadRequested,+ mlxExpertOffload: offload.loadedMlxExpertOffloadRequested,+ ...offload,};}diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.tsindex 46aea025d7a..e2b23d7247e 100644--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts@@ -2,7 +2,10 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import type { PerModelConfig } from "@/features/model-picker";-import { normalizeMlxKvQuant } from "@/features/model-picker/model-config/per-model-config";+import {+ normalizeMlxKvQuant,+ normalizeMlxOffload,+} from "@/features/model-picker/model-config/per-model-config";import {parseGpuLayersOverride,@@ -33,6 +36,7 @@ type ResidentRuntime = Pick<| "cache_type_kv"| "mlx_kv_quant_requested"| "mlx_int8_prefill_requested"+ | "mlx_offload"| "speculative_type"| "spec_draft_n_max"| "spec_draft_model"@@ -291,6 +295,19 @@ const SETTING_CHECKS: SettingCheck[] = [agrees: (c, s) =>Boolean(c.mlxInt8Prefill) === (s.mlx_int8_prefill_requested === true),},+ {+ // The request, not the applied fraction: Auto stays Auto whatever it picked.+ mlxComparable: true,+ pinned: () => true,+ agrees: (c, s) =>+ (c.mlxPleOffload ?? null) === normalizeMlxOffload(s.mlx_offload?.ple?.requested),+ },+ {+ mlxComparable: true,+ pinned: () => true,+ agrees: (c, s) =>+ (c.mlxExpertOffload ?? null) === normalizeMlxOffload(s.mlx_offload?.experts?.requested),+ },{// Always pinned: an unset mode resolves to the standing preference and the load sends it. Reading// it as silence let a pick asking for "off" adopt a resident MTP runtime.diff --git a/studio/frontend/src/features/chat/presets/preset-load-config.ts b/studio/frontend/src/features/chat/presets/preset-load-config.tsindex 9d87b950499..8d4fcd1fde0 100644--- a/studio/frontend/src/features/chat/presets/preset-load-config.ts+++ b/studio/frontend/src/features/chat/presets/preset-load-config.ts@@ -343,6 +343,8 @@ export function applyPresetLoadConfig(config?: PresetLoadConfig | null): void {chatTemplateOverride: null,// Not carried by presets: enabling it always goes through its confirmation.mlxInt8Prefill: store.mlxInt8Prefill,+ mlxPleOffload: store.mlxPleOffload,+ mlxExpertOffload: store.mlxExpertOffload,gpuMemoryMode: config.gpuMemoryMode,gpuLayers: config.gpuLayers,nCpuMoe: config.nCpuMoe,diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsxindex 828f8448640..a39cccbe0be 100644--- a/studio/frontend/src/features/chat/shared-composer.tsx+++ b/studio/frontend/src/features/chat/shared-composer.tsx@@ -2013,6 +2013,8 @@ export function SharedComposer({cache_type_kv: ownConfig.kvCacheDtype ?? null,mlx_kv_quant: ownConfig.mlxKvQuant ?? null,mlx_int8_prefill: ownConfig.mlxInt8Prefill ?? false,+ mlx_ple_offload: ownConfig.mlxPleOffload ?? null,+ mlx_expert_offload: ownConfig.mlxExpertOffload ?? null,speculative_type: effectiveSpeculativeType,spec_draft_n_max: effectiveSpecDraftNMax,...(targetIsMlx ? { spec_draft_model: ownConfig.specDraftModel ?? null } : {}),diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.tsindex 7788edde1c1..82dc4aaab36 100644--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts@@ -16,6 +16,7 @@ import {import {loadedContextFields,type MlxKvQuant,+ type MlxOffload,} from "@/features/model-picker/model-config/per-model-config";import {cachedPinnableGpuIndexKind,@@ -2660,6 +2661,17 @@ type ChatRuntimeStore = {mlxKvQuantNote: string | null;mlxInt8Prefill: boolean;loadedMlxInt8PrefillRequested: boolean;+ /** MLX offload knobs as chosen (null = Auto), as the resident load asked for them, the+ * fraction it applied (null = none), and why a request was not applied. */+ mlxPleOffload: MlxOffload;+ mlxExpertOffload: MlxOffload;+ loadedMlxPleOffloadRequested: MlxOffload;+ loadedMlxExpertOffloadRequested: MlxOffload;+ loadedMlxPleOffload: number | null;+ loadedMlxExpertOffload: number | null;+ mlxPleOffloadReason: string | null;+ mlxExpertOffloadReason: string | null;+ loadedMlxOffloadDiskBytes: number | null;loadedKvCacheDtype: string | null;speculativeType: string | null;loadedSpeculativeType: string | null;@@ -4621,6 +4633,15 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({mlxKvQuantNote: null,mlxInt8Prefill: false,loadedMlxInt8PrefillRequested: false,+ mlxPleOffload: null,+ mlxExpertOffload: null,+ loadedMlxPleOffloadRequested: null,+ loadedMlxExpertOffloadRequested: null,+ loadedMlxPleOffload: null,+ loadedMlxExpertOffload: null,+ mlxPleOffloadReason: null,+ mlxExpertOffloadReason: null,+ loadedMlxOffloadDiskBytes: null,loadedKvCacheDtype: null,speculativeType: readPersistedSpeculativeType(),loadedSpeculativeType: null,@@ -5614,6 +5635,15 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({mlxKvQuantNote: null,mlxInt8Prefill: false,loadedMlxInt8PrefillRequested: false,+ mlxPleOffload: null,+ mlxExpertOffload: null,+ loadedMlxPleOffloadRequested: null,+ loadedMlxExpertOffloadRequested: null,+ loadedMlxPleOffload: null,+ loadedMlxExpertOffload: null,+ mlxPleOffloadReason: null,+ mlxExpertOffloadReason: null,+ loadedMlxOffloadDiskBytes: null,loadedKvCacheDtype: null,speculativeType: readPersistedSpeculativeType(),loadedSpeculativeType: null,diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex 5cfdc3914cd..aa9dad0285f 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -12,6 +12,20 @@ export type MmprojFallbackReason =| "projector_incompatible"| "projector_startup_failure";+/** One MLX offload knob on the resident load: the fraction read from disk (null = none), as asked, and why a+ * request is not applied. */+export interface MlxOffloadPart {+ applied: number | null;+ requested: "auto" | "off" | number | null;+ reason: string | null;+}++export interface MlxOffloadReport {+ ple: MlxOffloadPart;+ experts: MlxOffloadPart;+ disk_bytes: number | null;+}+export interface BackendModelDetails {id: string;name?: string | null;@@ -78,6 +92,9 @@ export interface LoadModelRequest {cache_type_kv?: string | null;mlx_kv_quant?: string | null;mlx_int8_prefill?: boolean;+ /** MLX only: share of the PLE tables / routed experts read from disk, "off", or null for Auto. */+ mlx_ple_offload?: "off" | number | null;+ mlx_expert_offload?: "off" | number | null;/** Speculative decoding mode for GGUF models: "auto" (platform-aware DSpark/DFlash when the model* ships that sidecar, else MTP on MTP GGUFs, ngram-mod for sub-3B), "mtp", "dspark",* "dflash", "ngram", "mtp+ngram", "off". The legacy spellings are still accepted. */@@ -299,6 +316,7 @@ export interface LoadModelResponse {mlx_int8_prefill?: boolean | null;mlx_int8_prefill_requested?: boolean | null;mlx_int8_prefill_reason?: string | null;+ mlx_offload?: MlxOffloadReport | null;chat_template?: string | null;/** Canonical UI-facing mode the load request resolved to. See LoadModelRequest. */speculative_type?: string | null;@@ -457,6 +475,7 @@ export interface InferenceStatusResponse {mlx_int8_prefill?: boolean | null;mlx_int8_prefill_requested?: boolean | null;mlx_int8_prefill_reason?: string | null;+ mlx_offload?: MlxOffloadReport | null;chat_template_override?: string | null;/** Canonical UI-facing mode currently active. See LoadModelRequest. */speculative_type?: string | null;diff --git a/studio/frontend/src/features/model-picker/api/memory-estimate.ts b/studio/frontend/src/features/model-picker/api/memory-estimate.tsindex f3b823a30af..3c74b69457a 100644--- a/studio/frontend/src/features/model-picker/api/memory-estimate.ts+++ b/studio/frontend/src/features/model-picker/api/memory-estimate.ts@@ -63,8 +63,14 @@ export interface MemoryEstimate {gpuLayers: number | null;/** `--n-cpu-moe` is set, so the GPU figure ignores it and reads high. */moeOffloadUnmodelled: boolean;+ /** MLX: share of the PLE tables / routed experts read from disk, priced in `weightsBytes`. */+ mlxPleOffload: number | null;+ mlxExpertOffload: number | null;+ mlxOffloadable: MlxOffloadablePart[];}+export type MlxOffloadablePart = "ple" | "experts";+/** The load settings that move the estimate. Mirrors the fields /load takes. */export interface MemoryEstimateRequest {modelPath: string;@@ -75,6 +81,8 @@ export interface MemoryEstimateRequest {cacheTypeKv?: string | null;maxSeqLength?: number | null;mlxKvQuant?: string | null;+ mlxPleOffload?: "off" | number | null;+ mlxExpertOffload?: "off" | number | null;nParallel?: number | null;nBatch?: number | null;nUbatch?: number | null;@@ -118,6 +126,9 @@ const UNAVAILABLE: MemoryEstimate = {layerCount: null,gpuLayers: null,moeOffloadUnmodelled: false,+ mlxPleOffload: null,+ mlxExpertOffload: null,+ mlxOffloadable: [],};interface ApiEstimateResponse {@@ -146,6 +157,7 @@ interface ApiEstimateResponse {layer_count: number | null;gpu_layers: number | null;moe_offload_unmodelled: boolean;+ mlx_offload?: { ple?: unknown; experts?: unknown; offloadable?: unknown } | null;}function estimateRequestBody(@@ -161,6 +173,8 @@ function estimateRequestBody(cache_type_kv: payload.cacheTypeKv ?? null,max_seq_length: payload.maxSeqLength ?? null,mlx_kv_quant: payload.mlxKvQuant ?? null,+ mlx_ple_offload: payload.mlxPleOffload ?? null,+ mlx_expert_offload: payload.mlxExpertOffload ?? null,n_parallel: payload.nParallel ?? null,n_batch: payload.nBatch ?? null,n_ubatch: payload.nUbatch ?? null,@@ -210,6 +224,18 @@ function flag(value: unknown, fallback: boolean): boolean {return typeof value === "boolean" ? value : fallback;}+function offloadFraction(value: unknown): number | null {+ return typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1+ ? value+ : null;+}++function offloadableParts(value: unknown): MlxOffloadablePart[] {+ return Array.isArray(value)+ ? value.filter((part): part is MlxOffloadablePart => part === "ple" || part === "experts")+ : [];+}+const ESTIMATE_REASONS: readonly MemoryEstimateReason[] = ["not_gguf","not_downloaded",@@ -266,6 +292,9 @@ function toMemoryEstimate(body: ApiEstimateResponse): MemoryEstimate {layerCount: nullableCount(body.layer_count),gpuLayers: nullableCount(body.gpu_layers),moeOffloadUnmodelled: flag(body.moe_offload_unmodelled, false),+ mlxPleOffload: offloadFraction(body.mlx_offload?.ple),+ mlxExpertOffload: offloadFraction(body.mlx_offload?.experts),+ mlxOffloadable: offloadableParts(body.mlx_offload?.offloadable),};}diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.tsindex 79fb5c01582..73d67531a2e 100644--- a/studio/frontend/src/features/model-picker/api/model-overrides.ts+++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts@@ -45,6 +45,10 @@ export interface ApiModelOverride {mlx_kv_bits?: number;// biome-ignore lint/style/useNamingConvention: API schemamlx_int8_prefill?: boolean;+ // biome-ignore lint/style/useNamingConvention: API schema+ mlx_ple_offload?: "off" | number | null;+ // biome-ignore lint/style/useNamingConvention: API schema+ mlx_expert_offload?: "off" | number | null;// Stored only when false: absent is the 4-bit default.// biome-ignore lint/style/useNamingConvention: API schemaload_in_4bit?: boolean;@@ -391,6 +395,8 @@ export function fromApiOverride(tensorParallel: override.tensor_parallel ?? local.tensorParallel,disableVision: override.disable_vision ?? local.disableVision,mlxInt8Prefill: override.mlx_int8_prefill ?? local.mlxInt8Prefill,+ mlxPleOffload: override.mlx_ple_offload ?? local.mlxPleOffload,+ mlxExpertOffload: override.mlx_expert_offload ?? local.mlxExpertOffload,loadIn4bit: override.load_in_4bit ?? local.loadIn4bit,chatTemplateOverride:override.chat_template_override ?? local.chatTemplateOverride,@@ -430,6 +436,9 @@ export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {engine_kv_cache_dtype: config.engineKvCacheDtype ?? "auto",engine_seed: config.engineSeed ?? null,engine_gpu_memory_utilization: config.engineGpuMemoryUtilization ?? null,+ // Sent even at Auto: the server keeps a stored knob the payload omits.+ mlx_ple_offload: config.mlxPleOffload ?? null,+ mlx_expert_offload: config.mlxExpertOffload ?? null,};if (config.maxSeqLength && config.maxSeqLength > 0) {payload.max_seq_length = config.maxSeqLength;diff --git a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsxindex b0185ae3c82..703a1d38ebf 100644--- a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx+++ b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx@@ -11,9 +11,11 @@ import {import type { MemoryEstimate } from "../api/memory-estimate";import {type MemoryFitVerdict,+ NOTE_SEPARATOR,formatMemoryGb,glueNoteItems,memoryFigureCandidates,+ mlxOffloadPercent,resolveDraftCacheNote,resolveKvNote,resolveMemoryFit,@@ -225,6 +227,17 @@ export function MemoryEstimateRow({},);const kvNote = resolveKvNote(estimate);+ const weightsNote = [+ estimate.gpuLayers != null && estimate.layerCount != null+ ? `${estimate.gpuLayers} of ${estimate.layerCount + 1} layers on GPU`+ : null,+ estimate.mlxPleOffload ? `PLE ${mlxOffloadPercent(estimate.mlxPleOffload)} on disk` : null,+ estimate.mlxExpertOffload+ ? `experts ${mlxOffloadPercent(estimate.mlxExpertOffload)} on disk`+ : null,+ ]+ .filter(Boolean)+ .join(NOTE_SEPARATOR);const draftCacheNote = resolveDraftCacheNote(estimate.drafterRuntimeGpuBytes,estimate.drafterRuntimeBytes,@@ -290,11 +303,7 @@ export function MemoryEstimateRow({<MemoryBreakdownLinelabel="Weights"value={formatMemoryGb(estimate.weightsBytes)}- note={- estimate.gpuLayers != null && estimate.layerCount != null- ? `${estimate.gpuLayers} of ${estimate.layerCount + 1} layers on GPU`- : undefined- }+ note={weightsNote || undefined}/><MemoryBreakdownLinelabel="KV cache"diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex 49351de3509..9dfcde89709 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -102,7 +102,7 @@ import {loadManagedLlamaFlags,subscribeLlamaFlagCatalog,} from "../api/llama-flags";-import { type MemoryEstimate } from "../api/memory-estimate";+import { type MemoryEstimate, type MlxOffloadablePart } from "../api/memory-estimate";import { fetchMlxDrafters } from "../api/mlx-drafters";import {resolveEstimateContext,@@ -110,7 +110,14 @@ import {resolveMlxServedWindow,shouldRequestMemoryEstimate,} from "../model-config/estimate-context";-import { resolveReclaimableMemoryCredit } from "../model-config/memory-fit";+import {+ MLX_OFFLOAD_SLIDER_AUTO,+ mlxOffloadDecided,+ mlxOffloadFromSlider,+ mlxOffloadModePatch,+ mlxOffloadSliderValue,+ resolveReclaimableMemoryCredit,+} from "../model-config/memory-fit";import {resolveResidentEstimateRequest,selectResidentEstimateSettings,@@ -178,6 +185,7 @@ import {MLX_KV_QUANTS,mlxKvQuantLabel,type MlxKvQuant,+ type MlxOffload,N_BATCH_LLAMA_DEFAULT,N_BATCH_MAX,N_BATCH_MIN,@@ -1216,6 +1224,115 @@ function GpuMemorySettings({const MLX_KV_QUANT_AUTO = "auto";+const MLX_OFFLOAD_SLIDERS = [+ {+ part: "experts",+ key: "mlxExpertOffload",+ label: "Experts on Disk",+ hint: "Share of the MoE experts read from disk as they are needed instead of held in memory. Saves memory but slows generation a lot. 0% keeps every expert in memory; Auto leaves it to Unsloth.",+ },+ {+ part: "ple",+ key: "mlxPleOffload",+ label: "PLE on Disk",+ hint: "Share of the per-layer embedding tables read from disk instead of held in memory. Saves memory at a small cost to generation speed. 0% keeps them all in memory; Auto leaves it to Unsloth.",+ },+] as const satisfies readonly {+ part: MlxOffloadablePart;+ key: "mlxPleOffload" | "mlxExpertOffload";+ label: string;+ hint: string;+}[];++function MlxOffloadSettings({+ config,+ update,+ offloadable,+ decided,+ notes,+}: {+ config: PerModelConfig;+ update: (patch: Partial<PerModelConfig>) => void;+ offloadable: readonly MlxOffloadablePart[];+ decided: Record<MlxOffloadablePart, number | null> | null;+ notes: Record<MlxOffloadablePart, string | null>;+}) {+ const sliders = MLX_OFFLOAD_SLIDERS.filter(+ ({ part, key }) => offloadable.includes(part) || config[key] != null,+ );+ if (sliders.length === 0) return null;+ const isManual = sliders.some(({ key }) => config[key] != null);+ const shownNotes = sliders.map(({ part }) => notes[part]).filter(Boolean);+ return (+ <>+ <div className={ROW_CLASS}>+ <div className="flex min-w-0 items-center gap-1.5">+ <span className={LABEL_CLASS}>Disk Offload</span>+ <InfoHint>+ <div className="flex flex-col gap-1.5">+ <div>+ <span className="font-medium">Default:</span> Unsloth reads+ parts of the model from disk only when it would not otherwise+ fit in memory.+ </div>+ <div>+ <span className="font-medium">Manual:</span> set how much of+ each part stays on disk.+ </div>+ </div>+ </InfoHint>+ </div>+ <Select+ value={isManual ? "manual" : "auto"}+ onValueChange={(v) => {+ if (v === "manual" && !decided) return;+ update(+ mlxOffloadModePatch(+ v === "manual",+ sliders.map(({ part, key }) => ({ key, decided: decided?.[part] ?? null })),+ ),+ );+ }}+ >+ <SelectTrigger+ animateRadius={false}+ icon={ChevronDownStandardIcon}+ iconClassName="size-3.5"+ className={SELECT_TRIGGER_CLASS}+ >+ <SelectValue />+ </SelectTrigger>+ <SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">+ <SelectItem value="auto">Default</SelectItem>+ <SelectItem value="manual" disabled={!isManual && !decided}>+ Manual+ </SelectItem>+ </SelectContent>+ </Select>+ </div>+ {isManual &&+ sliders.map(({ part, key, label, hint }) => (+ <AdvancedGpuSlider+ key={part}+ label={label}+ value={mlxOffloadSliderValue(config[key])}+ min={MLX_OFFLOAD_SLIDER_AUTO}+ max={100}+ step={5}+ onChange={(v) => update({ [key]: mlxOffloadFromSlider(v) })}+ format={(v) => (v < 0 ? "Auto" : `${v}%`)}+ info={hint}+ />+ ))}+ {shownNotes.map((note) => (+ <p key={note} className="text-ui-11 text-muted-foreground">+ {note}+ </p>+ ))}+ </>+ );+}+function AdvancedSettingsToggle({checked,onCheckedChange,@@ -1511,6 +1628,9 @@ function MlxAdvancedSettings({servedByMlx,int8PrefillAvailable,onInt8PrefillChange,+ offloadable,+ offloadDecided,+ offloadNotes,speculativeFallback,modelPath,onEditTemplate,@@ -1524,6 +1644,9 @@ function MlxAdvancedSettings({servedByMlx: boolean;int8PrefillAvailable: boolean;onInt8PrefillChange: (checked: boolean) => void;+ offloadable: readonly MlxOffloadablePart[];+ offloadDecided: Record<MlxOffloadablePart, number | null> | null;+ offloadNotes: Record<MlxOffloadablePart, string | null>;speculativeFallback: string;modelPath: string;onEditTemplate: () => void;@@ -1606,6 +1729,15 @@ function MlxAdvancedSettings({{servedByMlx && (<ParallelSlotsRow config={config} update={update} hint={MLX_PARALLEL_HINT} />)}+ {servedByMlx && (+ <MlxOffloadSettings+ config={config}+ update={update}+ offloadable={offloadable}+ decided={offloadDecided}+ notes={offloadNotes}+ />+ )}<div className="space-y-1"><ChatTemplateSettingconfig={config}@@ -2443,6 +2575,16 @@ export function ModelConfigPage({useShallow(selectResidentEstimateSettings),);const mlxKvQuantNote = useChatRuntimeStore((s) => s.mlxKvQuantNote);+ const loadedMlxPleOffloadRequested = useChatRuntimeStore(+ (s) => s.loadedMlxPleOffloadRequested,+ );+ const loadedMlxExpertOffloadRequested = useChatRuntimeStore(+ (s) => s.loadedMlxExpertOffloadRequested,+ );+ const loadedMlxPleOffload = useChatRuntimeStore((s) => s.loadedMlxPleOffload);+ const loadedMlxExpertOffload = useChatRuntimeStore((s) => s.loadedMlxExpertOffload);+ const mlxPleOffloadReason = useChatRuntimeStore((s) => s.mlxPleOffloadReason);+ const mlxExpertOffloadReason = useChatRuntimeStore((s) => s.mlxExpertOffloadReason);const loadedMlxKvQuantRequested = useChatRuntimeStore((s) => s.loadedMlxKvQuantRequested,);@@ -2630,16 +2772,23 @@ export function ModelConfigPage({const [initialMlxInt8Prefill] = useState(() => configState.mlxInt8Prefill ?? false,);+ const [initialMlxOffload] = useState(+ () => configState.mlxPleOffload != null || configState.mlxExpertOffload != null,+ );// Applicability stays live, unlike the snapshot above: MLX can become available after mount,// and a width that starts applying then has to surface.const autoOpenForMlxKvQuant = servedByMlx && initialMlxKvQuant != null;const autoOpenForMlxInt8Prefill = servedByMlx && initialMlxInt8Prefill;+ const autoOpenForMlxOffload = servedByMlx && initialMlxOffload;// Opened to change one field: show its section for this visit, leaving the saved switch alone.const [revealForFocus, setRevealForFocus] = useState(focusField != null);const showAdvanced =revealForFocus ||(advancedPreference ??- (autoOpenAdvanced || autoOpenForMlxKvQuant || autoOpenForMlxInt8Prefill));+ (autoOpenAdvanced ||+ autoOpenForMlxKvQuant ||+ autoOpenForMlxInt8Prefill ||+ autoOpenForMlxOffload));const toggleAdvanced = (open: boolean) => {setRevealForFocus(false);saveAdvancedSettingsOpen(open);@@ -3257,6 +3406,8 @@ export function ModelConfigPage({? null: resolveMlxEstimateContext(savedContextPin(config)),mlxKvQuant: runtimeConfig.mlxKvQuant ?? null,+ mlxPleOffload: targetIsMlx ? (runtimeConfig.mlxPleOffload ?? null) : null,+ mlxExpertOffload: targetIsMlx ? (runtimeConfig.mlxExpertOffload ?? null) : null,nParallel: runtimeConfig.nParallel,nBatch: runtimeConfig.nBatch,nUbatch: runtimeConfig.nUbatch,@@ -3287,6 +3438,24 @@ export function ModelConfigPage({}: null;const memoryEstimate = useMemoryEstimate(memoryEstimateRequest);+ // The loaded model's reason answers only the request it was given.+ const mlxOffloadNote = (+ value: MlxOffload | undefined,+ loadedRequested: MlxOffload,+ reason: string | null,+ ): string | null =>+ isActiveModel && reason && (value ?? null) === (loadedRequested ?? null)+ ? reason+ : null;+ const mlxOffloadNotes = {+ ple: mlxOffloadNote(config.mlxPleOffload, loadedMlxPleOffloadRequested, mlxPleOffloadReason),+ experts: mlxOffloadNote(+ config.mlxExpertOffload,+ loadedMlxExpertOffloadRequested,+ mlxExpertOffloadReason,+ ),+ };+ const mlxOffloadDecidedNow = mlxOffloadDecided(memoryEstimate);const mlxFittedWindow = targetIsMlx? servedWindow(memoryEstimate.estimate?.contextFitted): null;@@ -3318,7 +3487,11 @@ export function ModelConfigPage({residentEstimateSettings,residentContext,targetIsMlx && !residentModelLoading- ? { kvQuant: loadedMlxKvQuantRequested ?? null }+ ? {+ kvQuant: loadedMlxKvQuantRequested ?? null,+ pleOffload: loadedMlxPleOffload,+ expertOffload: loadedMlxExpertOffload,+ }: undefined,);const residentEstimate = useMemoryEstimate(residentEstimateRequest, {@@ -3958,6 +4131,9 @@ export function ModelConfigPage({? setInt8PrefillConfirmOpen(true): update({ mlxInt8Prefill: false })}+ offloadable={memoryEstimate.estimate?.mlxOffloadable ?? []}+ offloadDecided={mlxOffloadDecidedNow}+ offloadNotes={mlxOffloadNotes}speculativeFallback={speculativeFallback}modelPath={target.id}onEditTemplate={() => setTemplateOpen(true)}diff --git a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.tsindex 3b166a93919..27920cc88aa 100644--- a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts+++ b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts@@ -41,6 +41,8 @@ export function useActiveModelConfig(): ActiveModelConfigState {const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype);const mlxKvQuant = useChatRuntimeStore((s) => s.mlxKvQuant);const mlxInt8Prefill = useChatRuntimeStore((s) => s.mlxInt8Prefill);+ const mlxPleOffload = useChatRuntimeStore((s) => s.mlxPleOffload);+ const mlxExpertOffload = useChatRuntimeStore((s) => s.mlxExpertOffload);const speculativeType = useChatRuntimeStore((s) => s.speculativeType);const specDraftNMax = useChatRuntimeStore((s) => s.specDraftNMax);const nParallel = useChatRuntimeStore((s) => s.nParallel);@@ -100,6 +102,8 @@ export function useActiveModelConfig(): ActiveModelConfigState {// over a field it cannot show.const effectiveMlxKvQuant = isMlx ? (mlxKvQuant ?? null) : null;const effectiveMlxInt8Prefill = isMlx && mlxInt8Prefill;+ const effectiveMlxPleOffload = isMlx ? (mlxPleOffload ?? null) : null;+ const effectiveMlxExpertOffload = isMlx ? (mlxExpertOffload ?? null) : null;const config = useMemo<PerModelConfig | null>(() => {if (!checkpoint || isExternalModelId(checkpoint)) {@@ -117,6 +121,8 @@ export function useActiveModelConfig(): ActiveModelConfigState {kvCacheDtype: kvCacheDtype ?? null,mlxKvQuant: effectiveMlxKvQuant,mlxInt8Prefill: effectiveMlxInt8Prefill,+ mlxPleOffload: effectiveMlxPleOffload,+ mlxExpertOffload: effectiveMlxExpertOffload,speculativeType: speculativeType ?? "auto",specDraftNMax: specDraftNMax ?? null,nParallel: nParallel ?? null,@@ -174,6 +180,8 @@ export function useActiveModelConfig(): ActiveModelConfigState {kvCacheDtype,effectiveMlxKvQuant,effectiveMlxInt8Prefill,+ effectiveMlxPleOffload,+ effectiveMlxExpertOffload,speculativeType,specDraftNMax,nParallel,diff --git a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.tsindex 96915398db7..dc69770a909 100644--- a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts+++ b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts@@ -38,6 +38,8 @@ function estimateKey(request: MemoryEstimateRequest | null): string | null {request.cacheTypeKv ?? null,request.maxSeqLength ?? null,request.mlxKvQuant ?? null,+ request.mlxPleOffload ?? null,+ request.mlxExpertOffload ?? null,request.nParallel ?? null,request.nBatch ?? null,request.nUbatch ?? null,diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.tsindex b26bc55db48..ccf4ceaeb2a 100644--- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts@@ -78,6 +78,8 @@ export function applyPerModelConfigToRuntime(customContextLength: config.customContextLength ?? null,mlxKvQuant: config.mlxKvQuant ?? null,mlxInt8Prefill: config.mlxInt8Prefill ?? false,+ mlxPleOffload: config.mlxPleOffload ?? null,+ mlxExpertOffload: config.mlxExpertOffload ?? null,kvCacheDtype: config.kvCacheDtype ?? null,speculativeType:normalizeSpeculativeType(config.speculativeType) ??@@ -154,6 +156,8 @@ export function currentRuntimePerModelConfig(kvCacheDtype: s.kvCacheDtype ?? null,mlxKvQuant: s.mlxKvQuant ?? null,mlxInt8Prefill: s.mlxInt8Prefill ?? false,+ mlxPleOffload: s.mlxPleOffload ?? null,+ mlxExpertOffload: s.mlxExpertOffload ?? null,speculativeType: normalizeSpeculativeType(s.speculativeType),specDraftNMax: s.specDraftNMax ?? null,specDraftCacheDtype: s.specDraftCacheDtype ?? null,@@ -211,6 +215,8 @@ export function perModelConfigsEqual((a.kvCacheDtype ?? null) === (b.kvCacheDtype ?? null) &&(a.mlxKvQuant ?? null) === (b.mlxKvQuant ?? null) &&Boolean(a.mlxInt8Prefill) === Boolean(b.mlxInt8Prefill) &&+ (a.mlxPleOffload ?? null) === (b.mlxPleOffload ?? null) &&+ (a.mlxExpertOffload ?? null) === (b.mlxExpertOffload ?? null) &&speculative(a.speculativeType) === speculative(b.speculativeType) &&(a.specDraftNMax ?? null) === (b.specDraftNMax ?? null) &&(a.specDraftCacheDtype ?? null) === (b.specDraftCacheDtype ?? null) &&diff --git a/studio/frontend/src/features/model-picker/model-config/config-signature.ts b/studio/frontend/src/features/model-picker/model-config/config-signature.tsindex b97d354ab80..d5b62963259 100644--- a/studio/frontend/src/features/model-picker/model-config/config-signature.ts+++ b/studio/frontend/src/features/model-picker/model-config/config-signature.ts@@ -60,6 +60,8 @@ export function loadedConfigSignature(config.kvCacheDtype ?? "",config.mlxKvQuant ?? "",config.mlxInt8Prefill ? "1" : "0",+ config.mlxPleOffload ?? "",+ config.mlxExpertOffload ?? "",config.speculativeType ?? "",config.specDraftNMax ?? "",config.specDraftCacheDtype ?? "",diff --git a/studio/frontend/src/features/model-picker/model-config/memory-fit.ts b/studio/frontend/src/features/model-picker/model-config/memory-fit.tsindex 28620b2c674..6f627cb2f1e 100644--- a/studio/frontend/src/features/model-picker/model-config/memory-fit.ts+++ b/studio/frontend/src/features/model-picker/model-config/memory-fit.ts@@ -28,6 +28,7 @@ import type {SystemGpuDevice,} from "../../../hooks/gpu-selection.ts";import { gpuMemoryTotalsGb, sharesHostMemory } from "../../../hooks/gpu-vram.ts";+import type { MlxOffload } from "./per-model-config.ts";/** A memory figure in bytes, to two decimals. @deprecated Prefer `formatBytesGiB` from* `@/lib/memory/format`, whose name says which unit it takes. This alias exists because a@@ -560,6 +561,45 @@ export function resolveMemoryAdvisory(/** What `resolveKvNote` joins its items with, and what `glueNoteItems` splits on. */export const NOTE_SEPARATOR = " · ";+/** What choosing Default or Manual disk offload stores: Default leaves each part to Auto, Manual+ * pins the fraction Auto would pick, or off where Auto keeps the part resident. */+export function mlxOffloadModePatch(+ manual: boolean,+ parts: readonly { key: "mlxPleOffload" | "mlxExpertOffload"; decided: number | null }[],+): Record<string, MlxOffload> {+ return Object.fromEntries(+ parts.map(({ key, decided }) => [key, manual ? decided || "off" : null]),+ );+}++/** Disk offload slider position for Auto, one step below 0% (off), as GPU Layers has. */+export const MLX_OFFLOAD_SLIDER_AUTO = -5;++export function mlxOffloadSliderValue(value: MlxOffload | undefined): number {+ if (value == null) return MLX_OFFLOAD_SLIDER_AUTO;+ return value === "off" ? 0 : Math.round(value * 100);+}++export function mlxOffloadFromSlider(position: number): MlxOffload {+ if (position < 0) return null;+ return position === 0 ? "off" : position / 100;+}++/** The fractions Auto picks for the current settings, or null until an estimate of them settles: a stale or+ * loading estimate answers earlier ones, and reading its parts as resident would pin them off. */+export function mlxOffloadDecided(state: {+ estimate: { mlxPleOffload?: number | null; mlxExpertOffload?: number | null } | null;+ stale: boolean;+ loading: boolean;+}): { ple: number | null; experts: number | null } | null {+ if (state.stale || state.loading || !state.estimate) return null;+ return { ple: state.estimate.mlxPleOffload ?? null, experts: state.estimate.mlxExpertOffload ?? null };+}++export function mlxOffloadPercent(fraction: number): string {+ return `${Math.round(fraction * 100)}%`;+}+/** A separated caption, breakable only between its items. A caption like "f16 - 262,144 tokens* - 4 slots" does not fit a narrow panel, and the browser breaks at the last space that* fits, orphaning "slots". Gluing each item with U+00A0 leaves one break opportunity perdiff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.tsindex b5438115cfc..1e37b14c4c9 100644--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts@@ -36,6 +36,9 @@ export interface PerModelConfig {mlxKvQuant?: MlxKvQuant | null;/** MLX only: run quantized projections with int8 activations, where the model supports it. */mlxInt8Prefill?: boolean;+ /** MLX only: share of the PLE tables / routed experts read from disk, "off", or null for Auto. */+ mlxPleOffload?: MlxOffload;+ mlxExpertOffload?: MlxOffload;speculativeType: string | null;specDraftNMax: number | null;/** KV cache dtype for the DRAFT context, sized and quantized independently of kvCacheDtype.@@ -88,6 +91,8 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = {kvCacheDtype: null,mlxKvQuant: null,mlxInt8Prefill: false,+ mlxPleOffload: null,+ mlxExpertOffload: null,speculativeType: null,specDraftNMax: null,specDraftCacheDtype: null,@@ -388,6 +393,15 @@ export function normalizeMlxKvQuant(}const VALID_KV_CACHE_DTYPES = new Set<string>(KV_CACHE_DTYPES);+export type MlxOffload = "off" | number | null;++export function normalizeMlxOffload(value: unknown): MlxOffload {+ if (value === "off") return "off";+ return typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1+ ? value+ : null;+}+// llama-server's --load-mode enum in --help order. "auto" is the default: the UI shows it, storage keeps null and// the backend emits no flag. Never sent verbatim; builds like b10360 reject "auto" as a value.export const LOAD_MODES = [@@ -427,9 +441,10 @@ const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";// would normalize the unknown field straight back out of the record.// v2 added nBatch/nUbatch, v3 llamaExtraArgs, v4 disableVision, v5 the llama-server tuning group// (loadMode / specDraftCacheDtype / ctxCheckpoints / cacheRam), v6 the reasoning budget pair,-// v7 mlxKvQuant, v9 mlxInt8Prefill, v10 the per-GPU split, v11 the MLX drafter. v8 is skipped: nightly builds stamped it for the reverted custom+// v7 mlxKvQuant, v9 mlxInt8Prefill, v10 the per-GPU split, v11 the MLX drafter, v12 the MLX offload pair. v8 is skipped: nightly builds stamped it for the reverted custom// llama.cpp config (#12725), so a v8 client must not claim to understand an int8 prefill record.-const STORAGE_SCHEMA_VERSION = 11;+const STORAGE_SCHEMA_VERSION = 12;+const PRE_MLX_OFFLOAD_SCHEMA_VERSION = 11;const PRE_MLX_DRAFTER_SCHEMA_VERSION = 10;const PRE_TENSOR_SPLIT_SCHEMA_VERSION = 9;const PRE_MLX_INT8_PREFILL_SCHEMA_VERSION = 7;@@ -475,6 +490,8 @@ const STORED_CONFIG_FIELDS = new Set(["kvCacheDtype","mlxKvQuant","mlxInt8Prefill",+ "mlxPleOffload",+ "mlxExpertOffload","loadIn4bit","speculativeType","specDraftNMax",@@ -1127,6 +1144,8 @@ function normalizeV1(partial: RawConfig): PerModelConfig {maxSeqLength: normalizeMaxSeqLength(partial.maxSeqLength),mlxKvQuant: normalizeMlxKvQuant(partial.mlxKvQuant, partial.mlxKvBits),mlxInt8Prefill: partial.mlxInt8Prefill === true,+ mlxPleOffload: normalizeMlxOffload(partial.mlxPleOffload),+ mlxExpertOffload: normalizeMlxOffload(partial.mlxExpertOffload),kvCacheDtype:typeof partial.kvCacheDtype === "string" &&VALID_KV_CACHE_DTYPES.has(partial.kvCacheDtype)@@ -1213,9 +1232,12 @@ function storedSchemaVersion(normalized: PerModelConfig): number {const mode = normalized.speculativeType;// A kept Auto and the MLX-only modes are values an older client folds back to unset.const mlxMode = mode === "auto" || (MLX_ONLY_SPEC_TYPES as readonly string[]).includes(mode ?? "");- if (normalized.specDraftModel != null || mlxMode) {+ if (normalized.mlxPleOffload != null || normalized.mlxExpertOffload != null) {return STORAGE_SCHEMA_VERSION;}+ if (normalized.specDraftModel != null || mlxMode) {+ return PRE_MLX_OFFLOAD_SCHEMA_VERSION;+ }if (normalized.tensorSplit != null) {return PRE_MLX_DRAFTER_SCHEMA_VERSION;}@@ -1405,6 +1427,8 @@ export function isDefaultConfig(config: PerModelConfig): boolean {(config.kvCacheDtype ?? null) === DEFAULT_PER_MODEL_CONFIG.kvCacheDtype &&(config.mlxKvQuant ?? null) === DEFAULT_PER_MODEL_CONFIG.mlxKvQuant &&!config.mlxInt8Prefill &&+ config.mlxPleOffload == null &&+ config.mlxExpertOffload == null &&config.speculativeType === DEFAULT_PER_MODEL_CONFIG.speculativeType &&config.specDraftNMax == null &&config.nParallel == null &&diff --git a/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts b/studio/frontend/src/features/model-picker/model-config/resident-memory-request.tsindex 45bcf8a1e01..ed594526b8c 100644--- a/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts+++ b/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts@@ -66,12 +66,17 @@ export function selectResidentEstimateSettings(state: ResidentState) {}/** Copy only source identity from the pending request. `mlx`: the resident is an MLX load,- * which reads its window from maxSeqLength and its cache width from mlxKvQuant, not nCtx. */+ * which reads its window from maxSeqLength and its cache width from mlxKvQuant, not nCtx, and+ * is priced at the offload it applied rather than what Auto would pick now. */export function resolveResidentEstimateRequest(source: MemoryEstimateRequest | null,settings: ReturnType<typeof selectResidentEstimateSettings>,context: number | null,- mlx?: { kvQuant: string | null },+ mlx?: {+ kvQuant: string | null;+ pleOffload?: number | null;+ expertOffload?: number | null;+ },): MemoryEstimateRequest | null {if (!source ||@@ -90,7 +95,12 @@ export function resolveResidentEstimateRequest(...(settings ?? {}),nCtx: Math.floor(context),...(mlx- ? { maxSeqLength: Math.floor(context), mlxKvQuant: mlx.kvQuant }+ ? {+ maxSeqLength: Math.floor(context),+ mlxKvQuant: mlx.kvQuant,+ mlxPleOffload: mlx.pleOffload ?? "off",+ mlxExpertOffload: mlx.expertOffload ?? "off",+ }: {}),};}diff --git a/studio/frontend/src/features/model-picker/sharing/fields.ts b/studio/frontend/src/features/model-picker/sharing/fields.tsindex eb0ab5de32a..09b1e7b8a87 100644--- a/studio/frontend/src/features/model-picker/sharing/fields.ts+++ b/studio/frontend/src/features/model-picker/sharing/fields.ts@@ -47,6 +47,8 @@ export type SharedConfigKey = Exclude<| "maxSeqLength"| "mlxKvQuant"| "mlxInt8Prefill"+ | "mlxPleOffload"+ | "mlxExpertOffload"| "engine"| "enginePrecision"| "engineParallelism"diff --git a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts b/studio/frontend/tests/hydration-clears-stale-local-record.test.tsindex 9108601737e..5d6cbb50580 100644--- a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts+++ b/studio/frontend/tests/hydration-clears-stale-local-record.test.ts@@ -38,6 +38,15 @@ test("int8 prefill reaches the server row only when on", () => {assert.equal(fromApiOverride({}, on).mlxInt8Prefill, true);});+test("MLX offload is always sent, null clearing to Auto, and hydrates from the row else the local value", () => {+ const set = { ...DEFAULT_PER_MODEL_CONFIG, mlxPleOffload: 0.5, mlxExpertOffload: "off" as const };+ assert.deepEqual([toApiOverride(set).mlx_ple_offload, toApiOverride(set).mlx_expert_offload], [0.5, "off"]);+ const auto = toApiOverride(DEFAULT_PER_MODEL_CONFIG);+ assert.deepEqual([auto.mlx_ple_offload, auto.mlx_expert_offload], [null, null]);+ assert.equal(fromApiOverride({ mlx_expert_offload: 0.25 }, set).mlxExpertOffload, 0.25);+ assert.equal(fromApiOverride({}, set).mlxPleOffload, 0.5);+});+test("a server row holding the superseded cache width hydrates as the single choice", () => {assert.equal(fromApiOverride({ mlx_kv_bits: 8 }).mlxKvQuant, "8");assert.equal(fromApiOverride({ mlx_kv_quant: "tq-4", mlx_kv_bits: 8 }).mlxKvQuant, "tq-4");diff --git a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts b/studio/frontend/tests/memory-estimate-narrow-panel.test.tsindex 7535d704d13..8af1111b40c 100644--- a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts+++ b/studio/frontend/tests/memory-estimate-narrow-panel.test.ts@@ -59,6 +59,9 @@ const props: Props = {layerCount: 27,gpuLayers: 12,moeOffloadUnmodelled: false,+ mlxPleOffload: null,+ mlxExpertOffload: null,+ mlxOffloadable: [],},loading: false,stale: false,@@ -126,6 +129,12 @@ test("context checkpoints get their own host RAM row instead of inflating the KVassert.doesNotMatch(render({ expanded: true }), /Context checkpoints/);});+test("MLX offload is noted on the Weights line", () => {+ const html = render({ expanded: true, estimate: { ...props.estimate!, mlxExpertOffload: 0.5 } });+ assert.match(html, /Weights[\s\S]*?experts\u00a050%\u00a0on\u00a0disk/);+ assert.doesNotMatch(render({ expanded: true }), /on\u00a0disk/);+});+test("a shared pool shows the total without a duplicate GPU figure", () => {const html = render({ singleMemoryPool: true, isUnifiedMemory: true });assert.match(html, />Unified<\/span>/);diff --git a/studio/frontend/tests/memory-estimate-skew.test.ts b/studio/frontend/tests/memory-estimate-skew.test.tsindex ca57999b92e..c27607c009f 100644--- a/studio/frontend/tests/memory-estimate-skew.test.ts+++ b/studio/frontend/tests/memory-estimate-skew.test.ts@@ -274,6 +274,16 @@ test("a field arriving as a string or a non-finite number does not become a figuassert.equal(classifyMemoryFit(estimate.totalBytes, 24), "unknown");});+test("the MLX offload sends the knobs as stored and reads the nested answer", async () => {+ answer(() => json({ ...FULL, mlx_offload: { ple: null, experts: 0.5, offloadable: ["experts", "x"] } }));+ const estimate = await fetchMemoryEstimate({ ...REQUEST, mlxPleOffload: "off", mlxExpertOffload: 0.25 });+ const body = requests.at(-1)?.body as Record<string, unknown>;+ assert.deepEqual([body.mlx_ple_offload, body.mlx_expert_offload], ["off", 0.25]);+ assert.deepEqual([estimate.mlxPleOffload, estimate.mlxExpertOffload, estimate.mlxOffloadable], [null, 0.5, ["experts"]]);+ answer(() => json({ ...FULL, mlx_offload: null }));+ assert.deepEqual((await fetchMemoryEstimate(REQUEST)).mlxOffloadable, []);+});+test("the checkpoint share defaults to none and never exceeds the cache", async () => {answer(() => json(FULL));assert.equal((await fetchMemoryEstimate(REQUEST)).kvCheckpointBytes, 0);diff --git a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts b/studio/frontend/tests/mlx-context-pin-record-compat.test.tsindex 5e15bc55b3a..96dc28c44c5 100644--- a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts+++ b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts@@ -189,8 +189,8 @@ const ROWS: Row[] = ["and reads normally.",},{- name: "version 12 (genuinely future)",- raw: { version: 12, customContextLength: 32768 },+ name: "version 13 (genuinely future)",+ raw: { version: 13, customContextLength: 32768 },normalizedPin: null,rawPin: 32768,isDefault: true,@@ -335,6 +335,10 @@ test("both pin shapes are stamped version 1, so neither is distinguishable by ve);assert.equal(stage({ version: 12, customContextLength: 32768 }).remembered,+ true,+ );+ assert.equal(+ stage({ version: 13, customContextLength: 32768 }).remembered,false,);});@@ -358,6 +362,18 @@ test("int8 prefill defaults off, and only an enabled one stamps v9", () => {assert.equal(stage({ mlxInt8Prefill: true }).config.mlxInt8Prefill, true);});+test("MLX offload keeps off or a fraction in (0, 1], and only a set one stamps v12", () => {+ const offload = (value: unknown) => normalizePerModelConfig({ mlxExpertOffload: value }).mlxExpertOffload;+ assert.deepEqual([0.5, 1, "off"].map(offload), [0.5, 1, "off"]);+ assert.deepEqual([0, 1.5, -0.2, Number.NaN, "auto", "0.5", true].map(offload), Array(7).fill(null));+ assert.equal(isDefaultConfig({ ...DEFAULT_PER_MODEL_CONFIG, mlxPleOffload: "off" }), false);+ assert.equal(stampedVersion({ specDraftModel: "org/drafter", speculativeType: "dflash" }), 11);+ assert.equal(stampedVersion({ mlxPleOffload: 0.25 }), 12);+ assert.equal(stampedVersion({ mlxExpertOffload: "off" }), 12);+ const config = stage({ version: 12, mlxPleOffload: 0.75, mlxExpertOffload: "off" }).config;+ assert.deepEqual([config.mlxPleOffload, config.mlxExpertOffload], [0.75, "off"]);+});+/** Old config page, non-GGUF: the number reads maxSeqLength; customContextLength only sets atDefault. */function oldClientConfigPage(diff --git a/studio/frontend/tests/mlx-offload-mode.test.ts b/studio/frontend/tests/mlx-offload-mode.test.tsnew file mode 100644index 00000000000..ba9e387666e--- /dev/null+++ b/studio/frontend/tests/mlx-offload-mode.test.ts@@ -0,0 +1,50 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import {+ MLX_OFFLOAD_SLIDER_AUTO,+ mlxOffloadDecided,+ mlxOffloadFromSlider,+ mlxOffloadModePatch,+ mlxOffloadSliderValue,+} from "../src/features/model-picker/model-config/memory-fit.ts";++test("Manual pins what Auto would pick, and off where it keeps a part resident", () => {+ const parts = [+ { key: "mlxExpertOffload", decided: 0.6 },+ { key: "mlxPleOffload", decided: null },+ ] as const;+ assert.deepEqual(mlxOffloadModePatch(true, parts), { mlxExpertOffload: 0.6, mlxPleOffload: "off" });+ assert.deepEqual(mlxOffloadModePatch(true, [{ key: "mlxPleOffload", decided: 0 }]), {+ mlxPleOffload: "off",+ });+});++test("Default leaves every shown part to Auto", () => {+ assert.deepEqual(+ mlxOffloadModePatch(false, [+ { key: "mlxExpertOffload", decided: 0.6 },+ { key: "mlxPleOffload", decided: 1 },+ ]),+ { mlxExpertOffload: null, mlxPleOffload: null },+ );+});++test("the slider has an Auto step below off, so a part on Auto is never shown as off", () => {+ assert.equal(mlxOffloadSliderValue(null), MLX_OFFLOAD_SLIDER_AUTO);+ assert.equal(mlxOffloadSliderValue("off"), 0);+ assert.equal(mlxOffloadSliderValue(0.25), 25);+ assert.deepEqual([MLX_OFFLOAD_SLIDER_AUTO, 0, 60].map(mlxOffloadFromSlider), [null, "off", 0.6]);+});++test("Auto's pick is read only from an estimate of the current settings", () => {+ const estimate = { mlxPleOffload: 1, mlxExpertOffload: 0.5 };+ assert.deepEqual(mlxOffloadDecided({ estimate, stale: false, loading: false }), { ple: 1, experts: 0.5 });+ // Unknown, not resident: Manual would otherwise pin every part off while the estimate refreshes.+ assert.equal(mlxOffloadDecided({ estimate, stale: true, loading: false }), null);+ assert.equal(mlxOffloadDecided({ estimate, stale: false, loading: true }), null);+ assert.equal(mlxOffloadDecided({ estimate: null, stale: false, loading: false }), null);+});diff --git a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.tsindex 26955e9f1dc..6fcdc318fdf 100644--- a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts+++ b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts@@ -91,8 +91,13 @@ type FieldCase = {different: unknown;/** The backend compares offload knobs only under Manual, MoE count only with a layer pin. */live?: { config: Record<string, unknown>; status: Record<string, unknown> };+ wrap?: (value: unknown) => unknown;};+const offloadRequest = (part: "ple" | "experts") => (requested: unknown) => ({+ [part]: { applied: null, requested, reason: null },+});+const MANUAL_MODE = {config: { gpuMemoryMode: "manual" },status: { gpu_memory_mode: "manual" },@@ -249,8 +254,29 @@ const FIELDS: FieldCase[] = [same: true,different: false,},+ { key: "mlxPleOffload", statusKey: "mlx_offload", same: 0.5, different: "off", wrap: offloadRequest("ple") },+ {+ key: "mlxExpertOffload",+ statusKey: "mlx_offload",+ same: "off",+ different: 0.25,+ wrap: offloadRequest("experts"),+ },];+test("an Auto offload adopts a load that asked for auto, whatever fraction it applied", () => {+ const status = {+ ...ACCELERATORS["apple-mlx"],+ mlx_offload: {+ ple: { applied: null, requested: null, reason: null },+ experts: { applied: 0.5, requested: "auto" as const, reason: null },+ disk_bytes: null,+ },+ };+ assert.equal(residentRuntimeMatchesConfig(status, BLANK), true);+ assert.equal(residentRuntimeMatchesConfig(status, { ...BLANK, mlxExpertOffload: 0.5 }), false);+});+for (const [accelerator, base] of Object.entries(ACCELERATORS)) {test(`[${accelerator}] a config pinning nothing adopts the resident load`, () => {assert.equal(residentRuntimeMatchesConfig(base, BLANK), true);@@ -261,7 +287,7 @@ for (const [accelerator, base] of Object.entries(ACCELERATORS)) {test(`[${accelerator}] ${field.key} the resident load already runs is adopted`, () => {assert.equal(residentRuntimeMatchesConfig(- { ...base, ...field.live?.status, [field.statusKey]: field.same },+ { ...base, ...field.live?.status, [field.statusKey]: (field.wrap ?? ((v) => v))(field.same) },{ ...BLANK, ...field.live?.config, [field.key]: field.same },),true,@@ -274,7 +300,7 @@ for (const [accelerator, base] of Object.entries(ACCELERATORS)) {{...base,...field.live?.status,- [field.statusKey]: field.different,+ [field.statusKey]: (field.wrap ?? ((v) => v))(field.different),},{ ...BLANK, ...field.live?.config, [field.key]: field.same },),diff --git a/studio/frontend/tests/resident-memory-request.test.ts b/studio/frontend/tests/resident-memory-request.test.tsindex 643205a70b0..51dce6aa312 100644--- a/studio/frontend/tests/resident-memory-request.test.ts+++ b/studio/frontend/tests/resident-memory-request.test.ts@@ -147,6 +147,15 @@ test("an MLX resident is priced at its served window and loaded cache width", ()});assert.equal(request?.maxSeqLength, 8192);assert.equal(request?.mlxKvQuant, "4");+ assert.equal(request?.mlxPleOffload, "off");+ assert.equal(request?.mlxExpertOffload, "off");+ const offloaded = resolveResidentEstimateRequest(pending, settings, 8192, {+ kvQuant: null,+ pleOffload: 1,+ expertOffload: 0.5,+ });+ assert.equal(offloaded?.mlxPleOffload, 1);+ assert.equal(offloaded?.mlxExpertOffload, 0.5);const auto = resolveResidentEstimateRequest(pending, settings, 8192, {kvQuant: null,});diff --git a/studio/frontend/tests/same-model-reload-load-settings.test.ts b/studio/frontend/tests/same-model-reload-load-settings.test.tsindex 39dbacf5bab..246a8b229a5 100644--- a/studio/frontend/tests/same-model-reload-load-settings.test.ts+++ b/studio/frontend/tests/same-model-reload-load-settings.test.ts@@ -105,6 +105,11 @@ test("a same-model MLX reload elsewhere moves the KV width, int8 prefill and vermlx_kv_quant_reason: "Quantized KV cache to 8 bits",mlx_kv_quant_note: "from token 256",mlx_int8_prefill_requested: true,+ mlx_offload: {+ ple: { applied: 0.5, requested: "auto", reason: null },+ experts: { applied: null, requested: 0.25, reason: "no_quantized_experts" },+ disk_bytes: 1024,+ },}),MLX,);@@ -115,6 +120,10 @@ test("a same-model MLX reload elsewhere moves the KV width, int8 prefill and verassert.equal(s.mlxKvQuantNote, "from token 256");assert.equal(s.mlxInt8Prefill, true);assert.equal(s.loadedMlxInt8PrefillRequested, true);+ assert.deepEqual(+ [s.mlxPleOffload, s.loadedMlxPleOffload, s.mlxExpertOffload, s.loadedMlxExpertOffload, s.mlxExpertOffloadReason],+ [null, 0.5, 0.25, null, "no_quantized_experts"],+ );});test("a same-model MLX reload elsewhere moves a width that was already seeded", () => {diff --git a/studio/frontend/tests/server-tuning-settings.test.ts b/studio/frontend/tests/server-tuning-settings.test.tsindex 5363ee9abc6..5c8ab0a7d25 100644--- a/studio/frontend/tests/server-tuning-settings.test.ts+++ b/studio/frontend/tests/server-tuning-settings.test.ts@@ -102,8 +102,9 @@ test("the four take part in the editor's identity", () => {test("a record only claims the new schema version when it carries one", () => {// toStoredConfig stamps the oldest version that understands every field present.const source = readSrc("features/model-picker/model-config/per-model-config.ts");- assert.match(source, /const STORAGE_SCHEMA_VERSION = 11;\nconst PRE_MLX_DRAFTER_SCHEMA_VERSION = 10;/);- assert.match(source, /specDraftModel != null \|\| mlxMode\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);+ assert.match(source, /const STORAGE_SCHEMA_VERSION = 12;\nconst PRE_MLX_OFFLOAD_SCHEMA_VERSION = 11;\nconst PRE_MLX_DRAFTER_SCHEMA_VERSION = 10;/);+ assert.match(source, /mlxExpertOffload != null\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);+ assert.match(source, /specDraftModel != null \|\| mlxMode\)\s*\{\s*return PRE_MLX_OFFLOAD_SCHEMA_VERSION;/);assert.match(source, /tensorSplit != null\)\s*\{\s*return PRE_MLX_DRAFTER_SCHEMA_VERSION;/);assert.match(source, /const PRE_TENSOR_SPLIT_SCHEMA_VERSION = 9;/);assert.match(source, /const PRE_MLX_INT8_PREFILL_SCHEMA_VERSION = 7;/);diff --git a/tests/studio/test_cli_attach_load_knobs.py b/tests/studio/test_cli_attach_load_knobs.pyindex b90532416a1..dad22f4e5d3 100644--- a/tests/studio/test_cli_attach_load_knobs.py+++ b/tests/studio/test_cli_attach_load_knobs.py@@ -945,6 +945,31 @@ def test_a_turboquant_resident_reloads_as_turboquant(monkeypatch):assert server.loads[0]["mlx_kv_quant"] == "tq-4"+def test_a_resident_keeps_its_mlx_offload_on_reload(monkeypatch):+ server = FakeServer(+ [dict(RESIDENT)],+ {+ "is_gguf": False,+ "active_model": RESIDENT["id"],+ "model_identifier": RESIDENT["id"],+ "requested_context_length": 4096,+ "load_in_4bit": True,+ "mlx_offload": {+ "ple": {"requested": "off", "applied": None},+ # applied, which a reload must not send back as the request+ "experts": {"requested": 0.75, "applied": None},+ },+ },+ ).install(monkeypatch)++ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))++ assert (server.loads[0]["mlx_ple_offload"], server.loads[0]["mlx_expert_offload"]) == (+ "off",+ 0.75,+ )++def test_a_proven_no_op_skips_the_preload_gate(monkeypatch):"""A proven no-op skips the gate."""server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.pyindex 666ca7a6d6c..b2c74e70679 100644--- a/unsloth_cli/commands/start.py+++ b/unsloth_cli/commands/start.py@@ -2167,6 +2167,11 @@ def _resident_runtime_payload(status: dict, payload: dict) -> dict:if value is None:continuecarried[load_field] = value+ offload = status.get("mlx_offload") or {}+ for part, load_field in (("ple", "mlx_ple_offload"), ("experts", "mlx_expert_offload")):+ requested = (offload.get(part) or {}).get("requested")+ if load_field not in payload and requested is not None:+ carried[load_field] = requestedreturn carried