Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

feat(studio): MLX PLE and expert offload in Unsloth Studio (#14326)

Long Yixing9880bc0af996
diff --git a/studio/backend/core/inference/memory_contract.py b/studio/backend/core/inference/memory_contract.py
index 20f8b3937f9..f43d3acdc8b 100644
--- a/studio/backend/core/inference/memory_contract.py
+++ b/studio/backend/core/inference/memory_contract.py
@@ -172,6 +172,15 @@ def build_memory_estimate(
layer_count = getattr(breakdown, "layer_count", None),
gpu_layers = getattr(breakdown, "gpu_layers", None),
moe_offload_unmodelled = bool(moe_offload_unmodelled),
+ mlx_offload = (
+ None
+ if getattr(breakdown, "offloadable", None) is None
+ else {
+ "ple": breakdown.ple_offload,
+ "experts": breakdown.expert_offload,
+ "offloadable": list(breakdown.offloadable),
+ }
+ ),
context_is_pinned = bool(context_is_pinned),
inherited_device_pin = bool(inherited_device_pin),
spec_unpriced = bool(spec_unpriced),
@@ -211,6 +220,7 @@ def project_estimate_memory_response(estimate: MemoryEstimate) -> dict:
"layer_count": estimate.layer_count,
"gpu_layers": estimate.gpu_layers,
"moe_offload_unmodelled": estimate.moe_offload_unmodelled,
+ "mlx_offload": estimate.mlx_offload,
}
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index b4d8f782286..9d4af942017 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -1015,6 +1015,147 @@ def _int8_prefill_status(requested, model):
return status
+_MLX_OFFLOAD_PARTS = (("ple", "PLE tables"), ("experts", "quantized routed experts"))
+
+
+def normalize_mlx_offload(value):
+ """An offload knob as a load reads it: ``"auto"`` (also null), the fraction read from disk, or
+ None for off."""
+ if value in (None, "auto"):
+ return "auto"
+ return None if value == "off" else float(value)
+
+
+def mlx_resolve_offload(
+ model_dir,
+ knobs,
+ *,
+ pinned,
+ load_in_4bit: bool,
+ retains_history: bool,
+ kv_bits = None,
+ vision = None,
+) -> tuple:
+ """The ``(ple, expert)`` fractions a load of *model_dir* reads from disk (None for off), from
+ knobs as ``normalize_mlx_offload`` gives them: an ``"auto"`` knob decided by unsloth_zoo's 50/50
+ rule on this load's pricing, and off where that cannot be planned; any other knob as given.
+ ``vision=True`` prices the mlx-vlm route a speculative text load takes."""
+ fixed = tuple(None if knob == "auto" else knob for knob in knobs)
+ if "auto" not in knobs or not model_dir:
+ return fixed
+ from core.inference.mlx_memory import mlx_offload_plan, mlx_offload_planner
+
+ if mlx_offload_planner() is None:
+ return fixed
+ budget = mlx_memory_budget(retains_history = retains_history)
+ ceiling = pinned or mlx_checkpoint_window(model_dir)
+ if budget is None or not ceiling:
+ return fixed
+ if kv_bits is not None and mlx_kv_quant_is_refused(model_dir, kv_bits):
+ kv_bits = None
+ with mlx_rng_preserved():
+ try:
+ plan = mlx_offload_plan(
+ model_dir,
+ budget_bytes = budget,
+ max_ctx = int(ceiling),
+ context = pinned or None,
+ ple = knobs[0],
+ experts = knobs[1],
+ kv_bits = kv_bits,
+ load_in_4bit = load_in_4bit,
+ vision = vision,
+ )
+ except Exception as exc:
+ logger.info("MLX auto offload not planned for %s: %s", model_dir, exc)
+ return fixed
+ return fixed if plan is None else plan
+
+
+def mlx_checkpoint_window(model_dir) -> Optional[int]:
+ from types import SimpleNamespace
+
+ from core.inference.mlx_memory import _snapshot_config
+
+ try:
+ config = _snapshot_config(model_dir)
+ except Exception:
+ return None
+ if config is None:
+ return None
+ native = mlx_native_context_length(SimpleNamespace(config = config))
+ return None if native is None else min(int(native), MAX_REQUESTABLE_CONTEXT)
+
+
+def _mlx_offload_request(knobs, plan):
+ """The offload a load asks for: the ``(ple, experts)`` knobs as sent, the fraction each wants,
+ the ``FastMLXModel.from_pretrained`` kwargs, and why none is passed when unsloth_zoo cannot
+ offload. unsloth_zoo hands an ``"auto"`` knob to *plan* once it has the checkpoint (the one it
+ resolved and downloaded), and never does for a load it cannot offload; with no *plan* it is off."""
+ from core.inference.mlx_memory import mlx_offload_planner
+
+ requested = dict(zip(("ple", "experts"), knobs))
+ wanted = {part: normalize_mlx_offload(value) for part, value in requested.items()}
+ if plan is None:
+ wanted = {part: None if value == "auto" else value for part, value in wanted.items()}
+ offload = {"requested": requested, "wanted": wanted, "reason": None, "kwargs": {}}
+ if not any(wanted.values()):
+ return offload
+ if mlx_offload_planner() is None:
+ offload["reason"] = "the installed unsloth_zoo cannot read weights from disk"
+ elif "auto" in wanted.values():
+
+ def decide(model_dir):
+ wanted.update(zip(wanted, plan(model_dir, tuple(wanted.values()))))
+ offload["decided"] = True
+ return dict(wanted)
+
+ offload["decided"] = False
+ offload["kwargs"]["offload"] = decide
+ else:
+ offload["kwargs"]["offload"] = dict(wanted)
+ return offload
+
+
+def _load_offloading(load, source, load_kwargs, offload):
+ """``load(source, **load_kwargs)``, again without offload when unsloth_zoo refuses it for this load
+ (an adapter, a bitsandbytes checkpoint, runtime quantization); ``offload`` keeps the reason."""
+ try:
+ return load(source, **load_kwargs)
+ except ValueError as exc:
+ # unsloth_zoo's offload refusals all open so.
+ if not offload["kwargs"] or not str(exc).startswith("Unsloth: offload "):
+ raise
+ logger.info("MLX offload not applied to %s: %s", source, exc)
+ offload["reason"] = str(exc)
+ load_kwargs.pop("offload", None)
+ offload["kwargs"] = {}
+ return load(source, **load_kwargs)
+
+
+def _mlx_offload_status(offload, model):
+ """Per knob the fraction read from disk, the request and why it is not applied; and the bytes
+ the loaded model reads from disk."""
+ reports = getattr(model, "_unsloth_offload", None) or {}
+ status = {"disk_bytes": sum(report["on_disk_bytes"] for report in reports.values()) or None}
+ for part, what in _MLX_OFFLOAD_PARTS:
+ applied = offload["wanted"][part] if part in reports else None
+ reason = None
+ if offload["wanted"][part] not in (None, "auto") and applied is None:
+ reason = offload["reason"] or (
+ "unsloth_zoo cannot offload this load (an adapter, a bitsandbytes checkpoint or "
+ "runtime quantization)"
+ if offload.get("decided") is False
+ else f"the model has no {what} unsloth_zoo can read from disk"
+ )
+ status[part] = {
+ "applied": applied,
+ "requested": offload["requested"][part],
+ "reason": reason,
+ }
+ return status
+
+
# Keyed by checkpoint fingerprint: the panel asks again on every model switch.
_int8_prefill_checkpoint_cache: dict = {}
_int8_prefill_checkpoint_lock = threading.Lock()
@@ -2691,6 +2832,7 @@ def mlx_drafter_fit(
*,
load_in_4bit: bool,
costless = False,
+ offload = None,
):
"""``(attaches, fitted context or None)`` for *source*'s drafter on an mlx-vlm load of *model_dir*:
only a priced fit reserves it, a pinned context must fit whole, and a ``costless`` one must fit
@@ -2710,6 +2852,7 @@ def mlx_drafter_fit(
max_ctx = int(ceiling),
load_in_4bit = load_in_4bit,
vision = True,
+ offload = offload,
)
outcome, fitted = fit(drafter = (source.path, source.builtin))
if outcome == "unsizable":
@@ -4883,6 +5026,7 @@ class MLXInferenceBackend:
spec_draft_n_max = None,
spec_draft_model = None,
spec_drafters_allowed = None,
+ offload = (None, None),
) -> bool:
import mlx.core as mx
@@ -5004,7 +5148,6 @@ class MLXInferenceBackend:
load_kwargs["pipeline_group"] = distributed_group
else:
load_kwargs["tensor_group"] = distributed_group
-
precision_folder = (
None if getattr(config, "is_gguf", False) else getattr(config, "gguf_variant", None)
)
@@ -5013,14 +5156,34 @@ class MLXInferenceBackend:
from hub.utils.mlx_variants import download_precision_folder
load_source = download_precision_folder(model_name, precision_folder, hf_token)
+ knobs = offload
+
+ # Auto is planned only where the load is priced: not distributed, an adapter or an explicit dtype.
+ def offload_request():
+ return _mlx_offload_request(
+ knobs,
+ None
+ if is_distributed or is_lora or dtype is not None
+ else functools.partial(
+ mlx_resolve_offload,
+ pinned = _positive_int(max_seq_length),
+ load_in_4bit = load_in_4bit,
+ retains_history = not use_vlm or mlx_vlm_snapshot_store_available(),
+ kv_bits = None if self._turboquant else _normalize_mlx_kv_bits(kv_bits),
+ vision = True if speculates and not is_vision else None,
+ ),
+ )
+
+ offload = offload_request()
+ load_kwargs.update(offload["kwargs"])
+
# Freed before the replacement weights are allocated, for headroom.
self._model_fusion.close()
self._clear_prompt_cache()
model = None
try:
- model, tokenizer_or_processor = FastMLXModel.from_pretrained(
- load_source,
- **load_kwargs,
+ model, tokenizer_or_processor = _load_offloading(
+ FastMLXModel.from_pretrained, load_source, load_kwargs, offload
)
except Exception as exc:
if is_vision or not use_vlm or is_metal_queue_dead(exc):
@@ -5067,9 +5230,11 @@ class MLXInferenceBackend:
_drain_generation_streams(mx)
mx.clear_cache()
load_kwargs["text_only"] = True
- model, tokenizer_or_processor = FastMLXModel.from_pretrained(
- load_source,
- **load_kwargs,
+ # Auto planned on the mlx-vlm route, and decide() consumed its knobs: plan this route afresh.
+ offload = offload_request()
+ load_kwargs.update(offload["kwargs"])
+ model, tokenizer_or_processor = _load_offloading(
+ FastMLXModel.from_pretrained, load_source, load_kwargs, offload
)
self._reads_vision = bool(is_vision)
@@ -5103,6 +5268,9 @@ class MLXInferenceBackend:
_served_ctx, _native_ctx, _max_ctx = self._resolve_context_lengths(
self._model, max_seq_length
)
+ _offload = _mlx_offload_status(offload, self._model)
+ _offloaded = (_offload["ple"]["applied"], _offload["experts"]["applied"])
+ _offloaded = {"offload": _offloaded} if any(_offloaded) else {}
_priceable = not (is_distributed or is_lora or dtype is not None)
spec_kind = _drafter_ctx = None
_drafter_fitted = False
@@ -5135,6 +5303,7 @@ class MLXInferenceBackend:
source,
load_in_4bit = load_in_4bit,
costless = spec_mode == "auto",
+ **_offloaded,
)
(
@@ -5171,6 +5340,7 @@ class MLXInferenceBackend:
kv_bits = _requested_bits,
is_vlm = use_vlm,
eligibility = _eligibility,
+ **_offloaded,
**({"vision": True} if _vlm_route else {}),
)
)
@@ -5288,6 +5458,7 @@ class MLXInferenceBackend:
"mlx_int8_prefill": self._int8_prefill["active"],
"mlx_int8_prefill_requested": self._int8_prefill["requested"],
"mlx_int8_prefill_reason": self._int8_prefill["reason"],
+ "mlx_offload": _offload,
"chat_template_override_requested": self._template_override["requested"],
"chat_template_override_reason": self._template_override["reason"],
"speculative_type": spec_mode,
diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.py
index 037c65af123..2909688cb2d 100644
--- a/studio/backend/core/inference/mlx_memory.py
+++ b/studio/backend/core/inference/mlx_memory.py
@@ -29,6 +29,7 @@ __all__ = [
"mlx_fit_context",
"mlx_fit_outcome",
"mlx_memory_breakdown",
+ "mlx_offload_plan",
"mlx_shard_files",
"mlx_weight_bytes",
]
@@ -75,6 +76,10 @@ class MlxMemoryBreakdown:
kv_on_gpu: bool = True
n_parallel: int = 1
gpu_layers: Optional[int] = None
+ # The fractions of the PLE tables / routed experts the load reads from disk; None for none.
+ ple_offload: Optional[float] = None
+ expert_offload: Optional[float] = None
+ offloadable: tuple = ()
def _snapshot_config(model_dir: str) -> Optional[dict]:
@@ -182,13 +187,7 @@ def _checkpoint_tensors(model_dir: str, config: Optional[dict], dtype):
tensors = {}
for shard in mlx_shard_files(model_dir, config):
- with open(shard, "rb") as handle:
- length = int.from_bytes(handle.read(8), "little")
- limit = min(_MAX_SAFETENSORS_HEADER, os.fstat(handle.fileno()).st_size - 8)
- if not 0 < length <= limit:
- raise ValueError(f"{shard} declares a {length}-byte safetensors header")
- header = json.loads(handle.read(length))
- for name, meta in header.items():
+ for name, meta in _safetensors_header(shard).items():
if name == "__metadata__":
continue
stored = getattr(mx, _SAFETENSORS_DTYPES.get(meta["dtype"], "float32"))
@@ -256,7 +255,9 @@ def _resident_bytes(model_dir: str, config: dict, quantize: bool) -> int:
is_vlm = _builds_on_mlx_vlm(config),
)
return sum(
- value.nbytes for _, value in tree_flatten(model.parameters()) if hasattr(value, "nbytes")
+ value.nbytes
+ for name, value in tree_flatten(model.parameters())
+ if hasattr(value, "nbytes") and not _mapped(name, config, quantize)
)
@@ -270,12 +271,55 @@ def mlx_weight_bytes(
if not config:
return shards
if config.get("quantization") or config.get("quantization_config"):
- return shards
+ return shards - _mapped_bytes(model_dir, config)
try:
return _resident_bytes(model_dir, config, quantize = load_in_4bit)
except Exception as exc:
logger.debug("MLX estimate priced %s as stored: %s", model_dir, exc)
- return shards
+ return shards - _mapped_bytes(model_dir, config, load_in_4bit)
+
+
+def _mapped(
+ name: str,
+ config: dict,
+ quantize: bool = False,
+) -> bool:
+ """A table mlx-vlm reads row by row from the checkpoint rather than holding it: DeepSeek
+ V4.1's Engram, unless the load quantizes it."""
+ return not quantize and _maps_engram(config) and ".engram.embed." in name
+
+
+def _maps_engram(config: dict) -> bool:
+ # mlx-vlm offloads Engram for every deepseek_v41 load, whatever the config says of its layers.
+ return "deepseek_v41" in (
+ config.get("model_type"),
+ (config.get("text_config") or {}).get("model_type"),
+ )
+
+
+def _mapped_bytes(
+ model_dir: str,
+ config: dict,
+ quantize: bool = False,
+) -> int:
+ if quantize or not _maps_engram(config):
+ return 0
+ total = 0
+ for shard in mlx_shard_files(model_dir, config):
+ for name, meta in _safetensors_header(shard).items():
+ if name != "__metadata__" and _mapped(name, config):
+ start, end = meta["data_offsets"]
+ total += end - start
+ return total
+
+
+def _safetensors_header(shard: str) -> dict:
+ with open(shard, "rb") as handle:
+ length = int.from_bytes(handle.read(8), "little")
+ limit = min(_MAX_SAFETENSORS_HEADER, os.fstat(handle.fileno()).st_size - 8)
+ if not 0 < length <= limit:
+ raise ValueError(f"{shard} declares a {length}-byte safetensors header")
+ return json.loads(handle.read(length))
def _runtime_dtype():
@@ -320,6 +364,15 @@ def _loads_as_vision(config: dict) -> bool:
return bool(_is_vlm(SimpleNamespace(**config)))
+def _on_route(vision: Optional[bool]):
+ token = _ROUTE_VISION.set(vision)
+ try:
+ yield
+ finally:
+ _ROUTE_VISION.reset(token)
+
+
def _builds_on_mlx_vlm(config: dict) -> bool:
"""Whether mlx-vlm builds the weights: on a vision route, or for a text model unsloth_zoo builds
there because mlx-lm cannot; that one is still served as text."""
@@ -1065,6 +1118,87 @@ def _refused_by_its_own_tensors(model_dir: str, config: dict) -> Optional[str]:
return None
[email protected]_cache(maxsize = 1)
+def mlx_offload_planner():
+ """unsloth_zoo's offload planner, or None when the installed unsloth_zoo cannot offload as
+ Studio prices it."""
+ import inspect
+
+ try:
+ from unsloth_zoo.mlx import offload
+ from unsloth_zoo.mlx.loader import FastMLXModel
+ hook = "offload" in inspect.signature(FastMLXModel.from_pretrained).parameters
+ except Exception:
+ return None
+ if not (hook and hasattr(offload, "plan_offload") and hasattr(offload, "offloadable")):
+ return None
+ return offload
+
+
+def _lazy_checkpoint(model_dir: str, config: dict):
+ """``model_dir``'s model on the route a load takes, its weights mapped but never read."""
+ from pathlib import Path
+
+ # mlx-vlm imports a checkpoint's own module unasked; pricing must never run it.
+ if config.get("model_file"):
+ raise ValueError("this checkpoint carries its own model module")
+ # Refuses an index naming a shard outside the snapshot, as every other read here does.
+ _indexed_shards(model_dir)
+ # Not strict: unsloth_zoo loads past known unused tensors (Gemma 4's). Weightless: mlx-vlm's
+ # checkpoint conversion (ModelOpt FP8) evaluates its dense weights even on a lazy load.
+ with weightless_forward():
+ if _builds_on_mlx_vlm(config):
+ from mlx_vlm.utils import load_model
+ return load_model(Path(model_dir), lazy = True, strict = False)
+ from mlx_lm.utils import load_model
+ return load_model(Path(model_dir), lazy = True, strict = False)[0]
+
+
[email protected]_cache(maxsize = 4)
+def _offloadable_cached(model_dir: str, fingerprint: tuple, cast_dtype, on_mlx_vlm: bool):
+ from unsloth_zoo.mlx.offload import offloadable
+ return offloadable(
+ _lazy_checkpoint(model_dir, _snapshot_config(model_dir)), model_dir, cast_dtype = cast_dtype
+ )
+
+
+def _offloadable(model_dir: str, config: dict) -> dict:
+ """What unsloth_zoo's offload can take from this checkpoint, sized as the rest of the weights
+ are: shards as stored, else at the runtime width."""
+ quantized = config.get("quantization") or config.get("quantization_config")
+ return _offloadable_cached(
+ os.path.realpath(model_dir),
+ _checkpoint_fingerprint(model_dir),
+ None if quantized else _runtime_dtype(),
+ _builds_on_mlx_vlm(config),
+ )
+
+
+def mlx_offloadable_parts(model_dir: str, vision: Optional[bool] = None) -> tuple:
+ if mlx_offload_planner() is None:
+ return ()
+ with _on_route(vision):
+ found = _offloadable(model_dir, _snapshot_config(model_dir) or {})
+ return tuple(key for key in ("ple", "experts") if found[key] is None)
+
+
+def _offloaded(model_dir: str, config: dict, offload: Optional[tuple]) -> tuple:
+ """``(weights change, (ple, experts) applied)`` of reading ``offload``'s fractions from disk: what
+ unsloth_zoo's offload can take swaps its size for what it holds offloaded, and a knob with
+ nothing to take applies nothing."""
+ planner = mlx_offload_planner()
+ if not offload or not any(offload) or planner is None:
+ return 0, (None, None)
+ found = _offloadable(model_dir, config)
+ applied = tuple(
+ value if found[key] is None else None for value, key in zip(offload, ("ple", "experts"))
+ )
+ delta = (
+ int(planner.offload_footprint(found, *applied)) - found["ple_bytes"] - found["expert_bytes"]
+ )
+ return delta, applied
+
+
@dataclass
class _MlxSizing:
"""What a load costs before a context is named, so a search over contexts probes once."""
@@ -1081,6 +1215,7 @@ class _MlxSizing:
drafter_weights: int = 0
drafter_plan: tuple = ()
capture_per_token: int = 0
+ offload: tuple = (None, None)
_SIZING_CACHE_MAX = 16
@@ -1109,6 +1244,7 @@ def _size_load(
load_in_4bit: bool,
vision: Optional[bool] = None,
drafter: Optional[tuple] = None,
+ offload: Optional[tuple] = None,
) -> Optional[_MlxSizing]:
"""Cached ``_size_load_uncached``: every estimate prices the fit and the breakdown, and every
slider step prices again, each otherwise rebuilding the same weightless architecture."""
@@ -1123,14 +1259,19 @@ def _size_load(
vision,
drafter,
drafter and _checkpoint_fingerprint(drafter[0]),
+ offload,
)
except Exception:
- return _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)
+ return _size_load_uncached(
+ model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload
+ )
with _sizing_lock:
if key in _sizing_cache:
_sizing_cache.move_to_end(key)
return _sizing_cache[key]
- sizing = _size_load_uncached(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)
+ sizing = _size_load_uncached(
+ model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload
+ )
with _sizing_lock:
_sizing_cache[key] = sizing
while len(_sizing_cache) > _SIZING_CACHE_MAX:
@@ -1141,6 +1282,7 @@ def _size_load(
def _clear_sizing_cache() -> None:
with _sizing_lock:
_sizing_cache.clear()
+ _offloadable_cached.cache_clear()
def _size_load_uncached(
@@ -1150,10 +1292,11 @@ def _size_load_uncached(
load_in_4bit: bool,
vision: Optional[bool] = None,
drafter: Optional[tuple] = None,
+ offload: Optional[tuple] = None,
) -> Optional[_MlxSizing]:
token = _ROUTE_VISION.set(vision)
try:
- return _sized(model_dir, kv_bits, prefill_chunk, load_in_4bit, drafter)
+ return _sized(model_dir, kv_bits, prefill_chunk, load_in_4bit, drafter, offload)
finally:
_ROUTE_VISION.reset(token)
@@ -1164,8 +1307,10 @@ def _sized(
prefill_chunk: Optional[int],
load_in_4bit: bool,
drafter: Optional[tuple],
+ offload: Optional[tuple] = None,
) -> Optional[_MlxSizing]:
- """Everything about a load that does not move with the context, or None if it cannot be sized."""
+ """Everything about a load that does not move with the context, or None if it cannot be sized.
+ ``offload`` is the ``(ple, experts)`` fractions the load reads from disk."""
config = _snapshot_config(model_dir)
if config is None:
return None
@@ -1180,6 +1325,12 @@ def _sized(
return None
if weights <= 0:
return None
+ try:
+ delta, offload = _offloaded(model_dir, config, offload)
+ weights += delta
+ except Exception as exc:
+ logger.debug("MLX estimate cannot size the offload of %s: %s", model_dir, exc)
+ return None
try:
dtype = _runtime_dtype()
loaded_chunk, loaded_group = _generation_settings(config)
@@ -1208,6 +1359,7 @@ def _sized(
drafter_weights = drafted[0],
drafter_plan = tuple(drafted[1]),
capture_per_token = drafted[2],
+ offload = offload,
)
@@ -1249,6 +1401,8 @@ def _priced_at(sizing: _MlxSizing, n_ctx: int) -> Optional[MlxMemoryBreakdown]:
context,
chunk,
),
+ ple_offload = sizing.offload[0],
+ expert_offload = sizing.offload[1],
)
@@ -1261,10 +1415,12 @@ def mlx_memory_breakdown(
load_in_4bit: bool = False,
vision: Optional[bool] = None,
drafter: Optional[tuple] = None,
+ offload: Optional[tuple] = None,
) -> Optional[MlxMemoryBreakdown]:
"""Price an MLX load, or None when it cannot honestly be sized. ``vision`` overrides the loader route
- the config implies; ``drafter`` is the ``(path, builtin)`` drafter a speculative load attaches."""
- sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)
+ the config implies; ``drafter`` is the ``(path, builtin)`` drafter a speculative load attaches;
+ ``offload`` the ``(ple, experts)`` fractions it reads from disk."""
+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload)
return None if sizing is None else _priced_at(sizing, n_ctx)
@@ -1279,12 +1435,17 @@ def mlx_fit_outcome(
load_in_4bit: bool = False,
vision: Optional[bool] = None,
drafter: Optional[tuple] = None,
+ offload: Optional[tuple] = None,
) -> tuple:
"""``("unsizable" | "fits" | "fitted" | "no_fit", context)``: the largest context inside
- ``budget_bytes`` for ``"fitted"``, else None."""
- sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter)
+ ``budget_bytes`` for ``"fitted"``, else None. An offloaded load keeps unsloth_zoo's offload
+ safety margin of the budget free, as its offload planner does."""
+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision, drafter, offload)
if sizing is None:
return "unsizable", None
+ if any(sizing.offload):
+ least, share = mlx_offload_planner().SAFETY_MARGIN
+ budget_bytes = min(budget_bytes - least, budget_bytes / (1 + share))
ceiling = max(int(max_ctx or 0), MLX_KV_BLOCK)
at_ceiling = _priced_at(sizing, ceiling)
if at_ceiling is None:
@@ -1306,6 +1467,54 @@ def mlx_fit_outcome(
return ("no_fit", None) if best is None else ("fitted", (best // MLX_KV_BLOCK) * MLX_KV_BLOCK)
+def mlx_offload_plan(
+ model_dir: str,
+ *,
+ budget_bytes: int,
+ max_ctx: int,
+ context: Optional[int] = None,
+ ple = "auto",
+ experts = "auto",
+ kv_bits: Optional[int] = None,
+ prefill_chunk: Optional[int] = None,
+ load_in_4bit: bool = False,
+ vision: Optional[bool] = None,
+) -> Optional[tuple]:
+ """The ``(ple, experts)`` fractions unsloth_zoo's 50/50 rule reads from disk for this load, priced
+ as Studio prices it; a knob not ``"auto"`` and a pinned ``context`` are kept. When nothing fits,
+ everything offloadable goes to disk. None when the load cannot be planned."""
+ planner = mlx_offload_planner()
+ sizing = _size_load(model_dir, kv_bits, prefill_chunk, load_in_4bit, vision)
+ if planner is None or sizing is None:
+ return None
+ with _on_route(vision):
+ found = _offloadable(model_dir, _snapshot_config(model_dir))
+
+ def context_cost(tokens, _step):
+ priced = _priced_at(sizing, tokens)
+ if priced is None:
+ raise ValueError(f"cannot price {tokens} tokens")
+ return priced.total_bytes - sizing.weights
+
+ plan = planner.plan_offload(
+ budget = budget_bytes,
+ resident = sizing.weights - found["ple_bytes"] - found["expert_bytes"],
+ inventory = found,
+ context_cost = context_cost,
+ ceiling = max_ctx,
+ minimum = MLX_FIT_MIN_CONTEXT,
+ context = context,
+ prefill_step = sizing.chunk,
+ offload = {"ple": ple, "experts": experts},
+ )
+ if plan["context"] is None:
+ return tuple(
+ (1.0 if found[key] is None else None) if value == "auto" else value
+ for value, key in ((ple, "ple"), (experts, "experts"))
+ )
+ return plan["ple"], plan["experts"]
+
+
def mlx_fit_context(model_dir: str, **kwargs) -> Optional[int]:
"""Largest context whose estimated footprint stays inside ``budget_bytes``; None when the ceiling
already fits or nothing can be said."""
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index a7cd3e9f660..ea8f2bae513 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -275,6 +275,7 @@ _MLX_RUNTIME_MIRROR_FIELDS = (
"mlx_int8_prefill",
"mlx_int8_prefill_requested",
"mlx_int8_prefill_reason",
+ "mlx_offload",
"chat_template_override_requested",
"chat_template_override_reason",
"speculative_type",
@@ -2082,6 +2083,7 @@ class InferenceOrchestrator:
mlx_distributed: bool = False,
mlx_kv_quant: Optional[str] = None,
mlx_int8_prefill: bool = False,
+ mlx_offload: tuple = (None, None),
speculative_type: Optional[str] = None,
spec_draft_n_max: Optional[int] = None,
spec_draft_model: Optional[str] = None,
@@ -2159,6 +2161,7 @@ class InferenceOrchestrator:
else None,
"mlx_kv_quant": mlx_kv_quant,
"mlx_int8_prefill": bool(mlx_int8_prefill),
+ "mlx_offload": tuple(mlx_offload),
"speculative_type": speculative_type,
"spec_draft_n_max": spec_draft_n_max,
"spec_draft_model": spec_draft_model,
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index c86e18aa34e..0f4328e6cbb 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -815,6 +815,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
load_kwargs["distributed_group"] = config.get("_mlx_distributed_group")
load_kwargs["kv_quant"] = config.get("mlx_kv_quant")
load_kwargs["int8_prefill"] = bool(config.get("mlx_int8_prefill"))
+ load_kwargs["offload"] = tuple(config.get("mlx_offload") or (None, None))
load_kwargs["speculative_type"] = config.get("speculative_type")
load_kwargs["spec_draft_n_max"] = config.get("spec_draft_n_max")
load_kwargs["spec_draft_model"] = config.get("spec_draft_model")
@@ -922,6 +923,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
"mlx_int8_prefill",
"mlx_int8_prefill_requested",
"mlx_int8_prefill_reason",
+ "mlx_offload",
"chat_template_override_requested",
"chat_template_override_reason",
"speculative_type",
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index afceaa2f9ef..b86c9207c6e 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -60,6 +60,19 @@ def resolve_inventory_handle(value: str) -> str:
return resolved
+MlxOffload = Optional[Union[Literal["auto", "off"], float]]
+
+
+def _mlx_offload_amount(value: Any) -> Any:
+ """``None`` (the default, as ``"auto"``), ``"auto"``, ``"off"``, or the fraction to read from
+ disk, in (0, 1]."""
+ if value is None or value in ("auto", "off"):
+ return value
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 < value <= 1:
+ raise ValueError("must be 'auto', 'off' or the fraction to keep on disk, in (0, 1]")
+ return float(value)
+
+
class LoadRequest(BaseModel):
"""Request to load a model for inference"""
@@ -205,6 +218,28 @@ class LoadRequest(BaseModel):
"reason otherwise."
),
)
+ mlx_ple_offload: MlxOffload = Field(
+ None,
+ description = (
+ "MLX only: the fraction of the per-layer embedding (PLE) tables read from the "
+ "checkpoint on disk instead of held in memory, 'off', or 'auto' (also null): read "
+ "from disk only when the load would otherwise leave less than half the memory past "
+ "the weights for context. Applied only to a model with PLE tables; the status "
+ "reports the reason otherwise."
+ ),
+ )
+ mlx_expert_offload: MlxOffload = Field(
+ None,
+ description = (
+ "MLX only: the fraction of each MoE layer's routed experts read from the checkpoint "
+ "on disk on demand instead of held in memory, 'off', or 'auto' (also null): as "
+ "mlx_ple_offload, after the PLE tables. Slower decode; applied only to quantized "
+ "routed experts."
+ ),
+ )
+ _mlx_offload_amounts = field_validator("mlx_ple_offload", "mlx_expert_offload", mode = "before")(
+ _mlx_offload_amount
+ )
gpu_ids: Optional[List[int]] = Field(
None,
@@ -1125,6 +1160,11 @@ class EstimateMemoryRequest(BaseModel):
"a model that never used it, and reading it here would quantize an estimate "
"for a load whose cache stays full width.",
)
+ mlx_ple_offload: MlxOffload = Field(None, description = "As /load's mlx_ple_offload.")
+ mlx_expert_offload: MlxOffload = Field(None, description = "As /load's mlx_expert_offload.")
+ _mlx_offload_amounts = field_validator("mlx_ple_offload", "mlx_expert_offload", mode = "before")(
+ _mlx_offload_amount
+ )
_no_booleans = field_validator(
"n_batch", "n_ubatch", "ctx_checkpoints", "n_ctx", "max_seq_length", mode = "before"
@@ -1177,6 +1217,20 @@ class MlxDraftersResponse(BaseModel):
drafters: list[MlxDrafter] = Field(default_factory = list)
+class MlxOffloadEstimate(BaseModel):
+ """What an MLX load reads from disk instead of holding in memory, "auto" decided."""
+
+ ple: Optional[float] = Field(
+ None, description = "Fraction of the PLE tables read from disk; null for none"
+ )
+ experts: Optional[float] = Field(
+ None, description = "Fraction of the routed experts read from disk; null for none"
+ )
+ offloadable: list[Literal["ple", "experts"]] = Field(
+ default_factory = list, description = "What the load can read from disk at all"
+ )
+
+
class EstimateMemoryResponse(BaseModel):
"""Itemized memory an inference load would occupy, or why it could not be sized."""
@@ -1282,6 +1336,11 @@ class EstimateMemoryResponse(BaseModel):
description = "True when --n-cpu-moe is set: experts move per-tensor, not "
"per-block, so the GPU/host split above ignores it and reads high.",
)
+ mlx_offload: Optional[MlxOffloadEstimate] = Field(
+ None,
+ description = "MLX: what the load reads from disk instead of memory, priced in "
+ "weights_bytes; null when not MLX.",
+ )
class MemoryEstimate(BaseModel):
@@ -1389,6 +1448,9 @@ class MemoryEstimate(BaseModel):
moe_offload_unmodelled: bool = Field(
False, description = "--n-cpu-moe is set, so the GPU figure reads high"
)
+ mlx_offload: Optional[MlxOffloadEstimate] = Field(
+ None, description = "MLX: what the load reads from disk; null when not MLX"
+ )
context_fitted: Optional[int] = Field(
None,
description = "The window MLX would fit to this machine's memory for a load that "
@@ -1467,6 +1529,22 @@ class GenerateRequest(BaseModel):
image_base64: Optional[str] = Field(None, description = "Base64 encoded image for vision models")
+class MlxOffloadPart(BaseModel):
+ applied: Optional[float] = Field(None, description = "Fraction read from disk; null for none")
+ requested: Optional[Union[str, float]] = Field(None, description = "As the load asked for it")
+ reason: Optional[str] = Field(None, description = "Why a requested offload is not applied")
+
+
+class MlxOffloadStatus(BaseModel):
+ """The PLE tables and routed experts an MLX load reads from disk instead of memory."""
+
+ ple: MlxOffloadPart
+ experts: MlxOffloadPart
+ disk_bytes: Optional[int] = Field(
+ None, description = "Weight bytes the loaded model reads from the checkpoint on demand"
+ )
+
+
class _InferenceRuntimeFields(BaseModel):
"""Runtime fields shared by load and status responses."""
@@ -1749,6 +1827,7 @@ class _InferenceRuntimeFields(BaseModel):
description = "Why requested int8 prefill is not active, as unsloth_zoo reports it "
"(for example nax_unavailable or no_eligible_projections)",
)
+ mlx_offload: Optional[MlxOffloadStatus] = Field(None, description = "MLX: weights read from disk")
chat_template: Optional[str] = Field(
None,
description = "Jinja2 chat template string (from GGUF metadata or tokenizer)",
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 0867209d61a..a523da9a57e 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -41,6 +41,7 @@ from typing import (
TYPE_CHECKING,
Union,
)
+import dataclasses
import functools
import json
import re
@@ -8793,6 +8794,7 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
mlx_int8_prefill = None,
mlx_int8_prefill_requested = None,
mlx_int8_prefill_reason = None,
+ mlx_offload = None,
mlx_context_budget = None,
spec_draft_model = None,
chat_template_override_reason = None,
@@ -15981,20 +15983,8 @@ def _local_gguf_main_path(config: ModelConfig) -> Optional[str]:
def _mlx_estimate_ceiling(model_dir: str) -> Optional[int]:
"""The window this checkpoint declares, held to what a load may be asked for."""
- from types import SimpleNamespace
-
- from core.inference.mlx_inference import mlx_native_context_length
- from core.inference.mlx_memory import _snapshot_config
- from core.inference.runtime_context import MAX_REQUESTABLE_CONTEXT
-
- try:
- config = _snapshot_config(model_dir)
- except Exception:
- return None
- if config is None:
- return None
- native = mlx_native_context_length(SimpleNamespace(config = config))
- return None if native is None else min(int(native), MAX_REQUESTABLE_CONTEXT)
+ from core.inference.mlx_inference import mlx_checkpoint_window
+ return mlx_checkpoint_window(model_dir)
def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_bits, **route):
@@ -16019,18 +16009,98 @@ def _mlx_estimate_fitted_context(config, model_dir: str, load_in_4bit: bool, kv_
)
-def _mlx_estimate_drafter(request, model_identifier, model_dir, ceiling, load_in_4bit: bool):
- """``((path, builtin), fitted context, speculates)`` of an MLX load of *request*: the drafter it
- would attach, chosen as the load chooses it, and whether it speculates at all (so loads through mlx-vlm)."""
+def _mlx_estimate_offload(
+ request, config, model_dir: str, load_in_4bit: bool, kv_bits: Optional[int]
+) -> tuple[dict, tuple]:
+ """``({"offload": (ple, experts)} or {}, offloadable parts)`` for an MLX load of *request*,
+ "auto" decided as the load does, on its route; neither if the load quantizes, which unsloth_zoo
+ refuses."""
+ from core.inference.mlx_inference import (
+ mlx_resolve_offload,
+ mlx_vlm_snapshot_store_available,
+ normalize_mlx_offload,
+ )
+ from core.inference.mlx_memory import _snapshot_config, mlx_offloadable_parts
+
+ checkpoint = _snapshot_config(model_dir) or {}
+ if load_in_4bit and not (
+ checkpoint.get("quantization") or checkpoint.get("quantization_config")
+ ):
+ return {}, ()
+ vision = True if _mlx_estimate_speculates(request, model_dir) else None
+ try:
+ parts = mlx_offloadable_parts(model_dir, vision)
+ except Exception as exc:
+ logger.info("MLX offload eligibility unknown for %s: %s", model_dir, exc)
+ parts = ()
+ offload = mlx_resolve_offload(
+ model_dir,
+ (
+ normalize_mlx_offload(request.mlx_ple_offload),
+ normalize_mlx_offload(request.mlx_expert_offload),
+ ),
+ pinned = request.max_seq_length or None,
+ load_in_4bit = load_in_4bit,
+ # As the load: any mlx-vlm route, a speculative text load included, keeps no history without the store.
+ retains_history = (
+ not (getattr(config, "is_vision", False) or vision)
+ or mlx_vlm_snapshot_store_available()
+ ),
+ kv_bits = kv_bits,
+ vision = vision,
+ )
+ return ({"offload": offload} if any(offload) else {}), parts
+
+
+def _mlx_offload_warning(info: dict) -> Optional[str]:
+ """What an MLX load tells the user when its automatic offload put weights on disk."""
+ offload = info.get("mlx_offload") or {}
+ parts = [
+ f"{round(applied * 100)}% of its {what}"
+ for part, what in (("ple", "per-layer embedding tables"), ("experts", "MoE experts"))
+ if (applied := (offload.get(part) or {}).get("applied"))
+ and offload[part]["requested"] in (None, "auto")
+ ]
+ if not parts:
+ return None
+ return (
+ f"To fit in memory, this model reads {' and '.join(parts)} from disk (automatic offload), "
+ "so it generates more slowly. Set offload to Off to keep them in memory."
+ )
+
+
+def _mlx_estimate_speculates(request, model_dir) -> bool:
from core.inference import mlx_speculative
- from core.inference.mlx_inference import mlx_drafter_fit, parse_mlx_kv_quant
+ from core.inference.mlx_inference import parse_mlx_kv_quant
from core.inference.mlx_memory import _loads_as_vision, _snapshot_config
mode = mlx_speculative.mlx_spec_mode(request.speculative_type)
if not mlx_speculative.speculates_on_route(
mode, _loads_as_vision(_snapshot_config(model_dir) or {}), request.spec_draft_model
):
+ return False
+ bits, turboquant = parse_mlx_kv_quant(request.mlx_kv_quant)
+ legacy_bits = request.mlx_kv_quant is None and request.mlx_kv_bits is not None
+ kv_quant = bits is not None or turboquant or legacy_bits
+ return not mlx_speculative.speculation_refusal(kv_quant = kv_quant, distributed = False, lora = False)
+
+
+def _mlx_estimate_drafter(
+ request,
+ model_identifier,
+ model_dir,
+ ceiling,
+ load_in_4bit: bool,
+ offload_route: Optional[dict] = None,
+):
+ """``((path, builtin), fitted context, speculates)`` of an MLX load of *request*: the drafter it
+ would attach, chosen as the load chooses it, and whether it speculates at all (so loads through mlx-vlm)."""
+ from core.inference import mlx_speculative
+ from core.inference.mlx_inference import mlx_drafter_fit
+
+ if not _mlx_estimate_speculates(request, model_dir):
return None, None, False
+ mode = mlx_speculative.mlx_spec_mode(request.speculative_type)
resolution = mlx_speculative.resolve_speculation(
request.speculative_type,
request.spec_draft_model,
@@ -16038,11 +16108,6 @@ def _mlx_estimate_drafter(request, model_identifier, model_dir, ceiling, load_in
target_name = model_identifier,
allowed = _discoverable_drafters(model_identifier),
)
- bits, turboquant = parse_mlx_kv_quant(request.mlx_kv_quant)
- legacy_bits = request.mlx_kv_quant is None and request.mlx_kv_bits is not None
- kv_quant = bits is not None or turboquant or legacy_bits
- if mlx_speculative.speculation_refusal(kv_quant = kv_quant, distributed = False, lora = False):
- return None, None, False
for source in resolution.sources:
attaches, fitted = mlx_drafter_fit(
model_dir,
@@ -16051,6 +16116,7 @@ def _mlx_estimate_drafter(request, model_identifier, model_dir, ceiling, load_in
source,
load_in_4bit = load_in_4bit,
costless = mode == "auto",
+ **(offload_route or {}),
)
if attaches:
return (source.path, source.builtin), fitted, True
@@ -18409,7 +18475,11 @@ def _mlx_runtime_settings_match(backend, request) -> bool:
entry = backend.models.get(backend.active_model_name, {}) or {}
if "mlx_kv_quant_requested" not in entry:
return True
- from core.inference.mlx_inference import encode_mlx_kv_quant, parse_mlx_kv_quant
+ from core.inference.mlx_inference import (
+ encode_mlx_kv_quant,
+ normalize_mlx_offload,
+ parse_mlx_kv_quant,
+ )
from core.inference.mlx_speculative import mlx_spec_mode
@@ -18420,6 +18490,13 @@ def _mlx_runtime_settings_match(backend, request) -> bool:
== (request.chat_template_override or None)
and bool(entry.get("mlx_int8_prefill_requested"))
== bool(getattr(request, "mlx_int8_prefill", False))
+ and all(
+ normalize_mlx_offload(
+ ((entry.get("mlx_offload") or {}).get(part) or {}).get("requested")
+ )
+ == normalize_mlx_offload(getattr(request, f"mlx_{knob}_offload", None))
+ for part, knob in (("ple", "ple"), ("experts", "expert"))
+ )
and entry.get("speculative_type", "auto") == mlx_spec_mode(request.speculative_type)
and entry.get("spec_draft_model") == (getattr(request, "spec_draft_model", None) or None)
and entry.get("spec_draft_n_max") == _positive_int_or_none(request.spec_draft_n_max)
@@ -20196,6 +20273,8 @@ async def _load_model_impl(
mlx_int8_prefill = _model_info.get("mlx_int8_prefill"),
mlx_int8_prefill_requested = _model_info.get("mlx_int8_prefill_requested"),
mlx_int8_prefill_reason = _model_info.get("mlx_int8_prefill_reason"),
+ mlx_offload = _model_info.get("mlx_offload"),
+ memory_warning = _mlx_offload_warning(_model_info),
# Requested, as /status reports it: a null override would read
# as "using the default".
chat_template_override = _model_info.get("chat_template_override_requested"),
@@ -20936,6 +21015,7 @@ async def _load_model_impl(
subject = current_subject,
mlx_kv_quant = request.mlx_kv_quant,
mlx_int8_prefill = request.mlx_int8_prefill,
+ mlx_offload = (request.mlx_ple_offload, request.mlx_expert_offload),
speculative_type = request.speculative_type,
spec_draft_n_max = request.spec_draft_n_max,
spec_draft_model = request.spec_draft_model,
@@ -21115,6 +21195,8 @@ async def _load_model_impl(
mlx_int8_prefill = _model_info.get("mlx_int8_prefill"),
mlx_int8_prefill_requested = _model_info.get("mlx_int8_prefill_requested"),
mlx_int8_prefill_reason = _model_info.get("mlx_int8_prefill_reason"),
+ mlx_offload = _model_info.get("mlx_offload"),
+ memory_warning = _mlx_offload_warning(_model_info),
# Requested, as /status reports it: a null override would read as
# "using the default".
chat_template_override = _model_info.get("chat_template_override_requested"),
@@ -22621,14 +22703,18 @@ async def estimate_memory(
if mlx_kv_bits is not None and mlx_kv_quant_is_refused(model_dir, mlx_kv_bits):
mlx_kv_bits = None
mlx_named_ctx = request.max_seq_length or 0
+ mlx_offload, mlx_offloadable = _mlx_estimate_offload(
+ request, config, model_dir, mlx_load_in_4bit, mlx_kv_bits
+ )
mlx_drafter, mlx_fitted_ctx, mlx_speculates = _mlx_estimate_drafter(
request,
model_identifier,
model_dir,
_mlx_estimate_ceiling(model_dir),
mlx_load_in_4bit,
+ mlx_offload,
)
- mlx_route = {"vision": True} if mlx_speculates else {}
+ mlx_route = {**({"vision": True} if mlx_speculates else {}), **mlx_offload}
if not mlx_named_ctx and mlx_drafter is None:
mlx_fitted_ctx = _mlx_estimate_fitted_context(
config, model_dir, mlx_load_in_4bit, mlx_kv_bits, **mlx_route
@@ -22649,7 +22735,7 @@ async def estimate_memory(
return EstimateMemoryResponse(
**project_estimate_memory_response(
build_memory_estimate(
- mlx_breakdown,
+ dataclasses.replace(mlx_breakdown, offloadable = mlx_offloadable),
quant_file_bytes = 0,
context_fitted = mlx_fitted_ctx,
)
@@ -24020,6 +24106,8 @@ async def _slot_status(current_subject: str):
mlx_int8_prefill = model_info.get("mlx_int8_prefill"),
mlx_int8_prefill_requested = model_info.get("mlx_int8_prefill_requested"),
mlx_int8_prefill_reason = model_info.get("mlx_int8_prefill_reason"),
+ mlx_offload = model_info.get("mlx_offload"),
+ memory_warning = _mlx_offload_warning(model_info),
chat_template_override = model_info.get("chat_template_override_requested"),
chat_template_override_reason = model_info.get("chat_template_override_reason"),
loading = _loading_models,
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 370586a1660..c24aaf5471d 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -11,7 +11,7 @@ import re
import threading
import time
from contextvars import ContextVar
-from typing import Annotated, Any, Literal, Optional, get_args
+from typing import Annotated, Any, Literal, Optional, Union, get_args
from urllib.parse import unquote, urlsplit
from fastapi import APIRouter, BackgroundTasks, Depends, Header, HTTPException, Request
@@ -1083,6 +1083,16 @@ class ModelOverridePayload(BaseModel):
tensor_parallel: bool = False
disable_vision: bool = False
mlx_int8_prefill: bool = False
+ # Auto (null or "auto") is stored as absence; omitted keeps the stored value.
+ mlx_ple_offload: Optional[Union[Literal["auto", "off"], float]] = None
+ mlx_expert_offload: Optional[Union[Literal["auto", "off"], float]] = None
+
+ @field_validator("mlx_ple_offload", "mlx_expert_offload", mode = "before")
+ @classmethod
+ def _mlx_offload_amount(cls, value: Any) -> Any:
+ from models.inference import _mlx_offload_amount
+ return _mlx_offload_amount(value)
+
# Default-engine 16-bit load; only False is stored.
load_in_4bit: bool = True
# Validated in bytes below: pydantic counts characters, so a multi-byte template would pass.
@@ -2728,6 +2738,8 @@ def update_openai_auto_switch_override(
and not payload.disable_vision
and not payload.mlx_int8_prefill
and payload.load_in_4bit
+ # null clears just that knob, so naming one is not a forget.
+ and not {"mlx_ple_offload", "mlx_expert_offload"} & payload.model_fields_set
and not {
key: value
for key, value in saved_fields.items()
@@ -2805,7 +2817,8 @@ def update_openai_auto_switch_override(
+ (() if payload.mirrors_reasoning_budget else _reasoning_fields)
+ (() if payload.mirrors_spec_draft_model else _drafter_fields)
)
- if _carried_fields and not is_removal:
+ _alias_row: dict = {}
+ if not is_removal:
# The same spellings the extra-args carry-over walks: a cached repo is not an ordinary folded match,
# so a save under the repo id would find nothing and retire the alias with its tuning.
_alias_ids = [payload.model_id]
@@ -2823,12 +2836,12 @@ def update_openai_auto_switch_override(
# first non-empty row rather than merging, so filling a gap in the winner from a loser would switch
# dormant tuning on.
for _alias_id in _alias_ids:
- _stored_tuning = get_model_override(_alias_id)
- if not _stored_tuning:
+ _alias_row = get_model_override(_alias_id)
+ if not _alias_row:
continue
for name in _carried_fields:
if _kept_tuning[name] is None:
- _kept_tuning[name] = _stored_tuning.get(name)
+ _kept_tuning[name] = _alias_row.get(name)
break
removed_keys: list[str] = []
if payload.remove is True:
@@ -2938,6 +2951,13 @@ def update_openai_auto_switch_override(
"engine_gpu_memory_utilization",
)
},
+ # Omitted keeps what the row a load reads holds, which may be an alias retired below.
+ **{
+ field: getattr(payload, field)
+ if field in fields_set or is_removal
+ else (_alias_row or get_model_override(target_id)).get(field)
+ for field in ("mlx_ple_offload", "mlx_expert_offload")
+ },
max_seq_length = max_seq_length,
custom_context_length = custom_context_length,
kv_cache_dtype = payload.kv_cache_dtype,
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index 5d3b63023fb..c08d1be88ac 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -1187,6 +1187,7 @@ class TestLoadHubDownloadExclusion:
"mlx_int8_prefill",
"mlx_int8_prefill_requested",
"mlx_int8_prefill_reason",
+ "mlx_offload",
"mlx_context_budget",
"spec_draft_model",
"chat_template_override_reason",
diff --git a/studio/backend/tests/test_memory_contract.py b/studio/backend/tests/test_memory_contract.py
index e38a7107279..7a8d75294e7 100644
--- a/studio/backend/tests/test_memory_contract.py
+++ b/studio/backend/tests/test_memory_contract.py
@@ -206,6 +206,19 @@ class TestTheLegacyProjections:
)
assert out["gpu_bytes"] is None
+ def test_the_mlx_offload_the_breakdown_priced_is_reported(self):
+ mlx = {"ple_offload": None, "expert_offload": 0.6, "offloadable": ("experts",)}
+ offloaded = SimpleNamespace(**{**_BREAKDOWN.__dict__, **mlx})
+ panel = project_estimate_memory_response(
+ build_memory_estimate(offloaded, quant_file_bytes = 0)
+ )
+ assert panel["mlx_offload"].model_dump() == {
+ "ple": None,
+ "experts": 0.6,
+ "offloadable": ["experts"],
+ }
+ assert build_memory_estimate(_BREAKDOWN, quant_file_bytes = 0).mlx_offload is None
+
def test_the_kv_projection_does_not_borrow_the_planners_kv(self, estimate):
assert project_kv_cache_estimate(estimate)["kv_bytes"] is None
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index 8313a247f19..c5db7aab990 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -1105,6 +1105,82 @@ class TestEstimateMemoryRoute:
)
assert ri._local_mlx_model_dir(config) == str(tmp_path)
+ def test_an_mlx_estimate_prices_the_offload_the_load_applies(self, monkeypatch, tmp_path):
+ import core.inference.mlx_inference as mlx_inference
+ import core.inference.mlx_memory as mlx_memory
+
+ self._mlx_target(monkeypatch, str(tmp_path))
+ seen, four_bit, planned = [], {"on": False}, []
+ monkeypatch.setattr(ri, "_mlx_estimate_load_in_4bit", lambda *a: four_bit["on"])
+ monkeypatch.setattr(
+ mlx_memory, "mlx_offloadable_parts", lambda model_dir, vision = None: ("experts",)
+ )
+ monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: object())
+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda retains_history: 1 << 40)
+ monkeypatch.setattr(
+ mlx_memory, "mlx_offload_plan", lambda d, **kw: planned.append(kw) or (kw["ple"], 0.6)
+ )
+ monkeypatch.setattr(
+ mlx_memory,
+ "mlx_memory_breakdown",
+ lambda model_dir, *, n_ctx, **kw: seen.append(kw.get("offload"))
+ or mlx_memory.MlxMemoryBreakdown(1, 1, 1, 3, 3, n_ctx),
+ )
+ estimate = lambda **knobs: _estimate(model_path = "org/model", max_seq_length = 4096, **knobs)
+ price = lambda **knobs: estimate(**knobs) and seen[-1]
+ (tmp_path / "config.json").write_text(json.dumps({"quantization": {"bits": 4}}))
+ assert price(mlx_ple_offload = "off", mlx_expert_offload = 0.5) == (None, 0.5)
+ off = estimate(mlx_ple_offload = "off", mlx_expert_offload = "off")
+ assert off.mlx_offload.offloadable == ["experts"]
+ assert price(mlx_ple_offload = "off") == (None, 0.6) and planned[-1]["context"] == 4096
+ assert planned[-1]["vision"] is None
+ with monkeypatch.context() as patch:
+ patch.setattr(ri, "_mlx_estimate_speculates", lambda request, model_dir: True)
+ estimate(mlx_ple_offload = "off")
+ assert planned[-1]["vision"] is True and len(planned) == 2
+ four_bit["on"] = True # already quantized: nothing for the load to quantize, so it offloads
+ assert price(mlx_ple_offload = 1.0) == (1.0, 0.6)
+ (tmp_path / "config.json").write_text("{}")
+ assert price(mlx_ple_offload = 1.0) is None and len(planned) == 3
+ off = estimate(mlx_ple_offload = "off", mlx_expert_offload = "off")
+ assert off.mlx_offload.offloadable == []
+ four_bit["on"] = False
+ assert price(mlx_ple_offload = 1.0) == (1.0, 0.6)
+
+ def test_a_speculative_mlx_estimate_plans_offload_without_the_history_reserve(
+ self, monkeypatch, tmp_path
+ ):
+ import core.inference.mlx_inference as mlx_inference
+ import core.inference.mlx_memory as mlx_memory
+
+ self._mlx_target(monkeypatch, str(tmp_path))
+ (tmp_path / "config.json").write_text(json.dumps({"quantization": {"bits": 4}}))
+ reserves = []
+ monkeypatch.setattr(ri, "_mlx_estimate_load_in_4bit", lambda *a: False)
+ monkeypatch.setattr(
+ mlx_memory, "mlx_offloadable_parts", lambda d, vision = None: ("experts",)
+ )
+ monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: object())
+ monkeypatch.setattr(mlx_inference, "mlx_vlm_snapshot_store_available", lambda: False)
+ monkeypatch.setattr(
+ mlx_inference,
+ "mlx_memory_budget",
+ lambda retains_history: reserves.append(retains_history) or 1 << 40,
+ )
+ monkeypatch.setattr(mlx_memory, "mlx_offload_plan", lambda d, **kw: (None, 0.5))
+ monkeypatch.setattr(
+ mlx_memory,
+ "mlx_memory_breakdown",
+ lambda model_dir, *, n_ctx, **kw: mlx_memory.MlxMemoryBreakdown(1, 1, 1, 3, 3, n_ctx),
+ )
+ estimate = lambda: _estimate(model_path = "org/model", max_seq_length = 4096)
+ estimate()
+ assert reserves[-1] is True
+ # The load speculates through mlx-vlm, which keeps no history without the snapshot store.
+ monkeypatch.setattr(ri, "_mlx_estimate_speculates", lambda request, model_dir: True)
+ estimate()
+ assert reserves[-1] is False
+
def test_an_mlx_estimate_prices_max_seq_length_not_n_ctx(self, monkeypatch):
self._mlx_target(monkeypatch, "/models/thing")
seen = self._record_breakdown(
@@ -3507,6 +3583,209 @@ def test_a_load_is_sized_once_until_its_files_change(monkeypatch, tmp_path):
first = sized()
(drafter / "config.json").write_text("{}")
assert sized() != first # a changed local drafter is re-sized too
+ offloaded = lambda fraction: mm._size_load(
+ str(tmp_path), 4, None, True, None, None, (None, fraction)
+ )
+ assert offloaded(0.5) is offloaded(0.5) and offloaded(0.5) != offloaded(0.75)
+
+
+def test_offload_is_priced_by_what_unsloth_zoos_offload_takes(tmp_path, monkeypatch):
+ offload = pytest.importorskip("unsloth_zoo.mlx.offload")
+ if not _HAVE_MLX or not hasattr(offload, "plan_offload"):
+ pytest.skip("needs MLX and an unsloth_zoo that offloads")
+ mm.mlx_offload_planner.cache_clear()
+ planner = mm.mlx_offload_planner()
+ assert planner is offload
+ import mlx.core as mx
+ import mlx.nn as nn
+ from mlx.utils import tree_flatten
+ from mlx_lm.utils import _get_classes
+
+ config = {
+ "model_type": "qwen3_moe",
+ "hidden_size": 64,
+ "num_hidden_layers": 1,
+ "intermediate_size": 64,
+ "moe_intermediate_size": 64,
+ "num_attention_heads": 2,
+ "num_key_value_heads": 1,
+ "head_dim": 32,
+ "vocab_size": 64,
+ "num_experts": 16,
+ "num_experts_per_tok": 2,
+ "decoder_sparse_step": 1,
+ "mlp_only_layers": [],
+ "rms_norm_eps": 1e-6,
+ "rope_theta": 10000.0,
+ "tie_word_embeddings": True,
+ "max_position_embeddings": 512,
+ "norm_topk_prob": True,
+ }
+ model_class, args_class = _get_classes(config)
+ model = model_class(args_class.from_dict(config))
+ nn.quantize(
+ model,
+ group_size = 32,
+ bits = 4,
+ class_predicate = lambda path, m: hasattr(m, "to_quantized") and "switch_mlp" in path,
+ )
+ weights = dict(tree_flatten(model.parameters()))
+ # A tensor the model does not instantiate, as Gemma 4 checkpoints carry and unsloth_zoo loads past.
+ mx.save_safetensors(
+ str(tmp_path / "model.safetensors"), {**weights, "model.unused.scales": mx.zeros(4)}
+ )
+ config["quantization"] = {"group_size": 32, "bits": 4}
+ (tmp_path / "config.json").write_text(json.dumps(config))
+ experts = sum(v.nbytes for k, v in weights.items() if ".switch_mlp." in k)
+ found = offload.offloadable(mm._lazy_checkpoint(str(tmp_path), config), str(tmp_path))
+ assert (found["expert_bytes"], found["experts"]) == (experts, None)
+ assert mm.mlx_offloadable_parts(str(tmp_path)) == ("experts",)
+ for fractions in ((None, 0.5), (1.0, 1.0)): # no PLE table: its knob applies nothing
+ applied = (None, fractions[1])
+ delta = planner.offload_footprint(found, *applied) - experts
+ assert mm._offloaded(str(tmp_path), config, fractions) == (delta, applied)
+
+ # Auto: the 50/50 rule on this load's own pricing; when nothing fits, all of it goes to disk.
+ plan = lambda **kw: mm.mlx_offload_plan(str(tmp_path), max_ctx = 512, **kw)
+ asked, decide = {}, planner.plan_offload
+ monkeypatch.setattr(planner, "plan_offload", lambda **kw: asked.update(kw) or decide(**kw))
+ assert plan(budget_bytes = 1 << 40) == (None, None)
+ priced = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 512).total_bytes
+ assert asked["resident"] + experts + asked["context_cost"](512, None) == priced
+ assert plan(budget_bytes = 1) == (None, 1.0)
+ assert plan(budget_bytes = 1 << 40, experts = 0.5) == (None, 0.5)
+
+ mm.mlx_offload_planner.cache_clear()
+ with monkeypatch.context() as patch: # an unsloth_zoo that would not hand "auto" to Studio
+ patch.setattr(sys.modules["unsloth_zoo.mlx.loader"].FastMLXModel, "from_pretrained", print)
+ assert mm.mlx_offload_planner() is None
+ mm.mlx_offload_planner.cache_clear()
+
+ mm._clear_sizing_cache()
+ seen = []
+ kept = {
+ **found,
+ "expert_bytes": 0,
+ "expert_floor_bytes": 0,
+ "experts": "its routing top-k is unknown",
+ }
+ monkeypatch.setattr(
+ offload, "offloadable", lambda model, d, cast_dtype: seen.append(cast_dtype) or kept
+ )
+ assert mm._offloaded(str(tmp_path), config, (None, 1.0)) == (0, (None, None)) and seen == [None]
+ mm._offloaded(str(tmp_path), {**config, "quantization": None}, (None, 1.0))
+ assert seen[-1] == mm._runtime_dtype()
+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: None)
+ assert mm._offloaded(str(tmp_path), config, (1.0, 1.0)) == (0, (None, None))
+ assert plan(budget_bytes = 1) is None
+ (tmp_path / "own.py").write_text(f"open({str(tmp_path / 'ran')!r}, 'w')")
+ (tmp_path / "config.json").write_text(json.dumps({**config, "model_file": "own.py"}))
+ monkeypatch.setattr(mm, "_loads_as_vision", lambda config: True)
+ with pytest.raises(ValueError):
+ mm._lazy_checkpoint(str(tmp_path), mm._snapshot_config(str(tmp_path)))
+ assert not (tmp_path / "ran").exists()
+
+
+def test_auto_offload_is_planned_on_the_route_the_load_takes(monkeypatch):
+ routes = []
+ decide = lambda inventory, **kw: {
+ "context": 512,
+ "ple": None if inventory["ple"] else 1.0,
+ "experts": 0.5,
+ }
+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: SimpleNamespace(plan_offload = decide))
+ monkeypatch.setattr(
+ mm, "_snapshot_config", lambda d: {"model_type": "qwen3_moe", "quantization": {"bits": 4}}
+ )
+ monkeypatch.setattr(mm, "_checkpoint_fingerprint", lambda d: ())
+ monkeypatch.setattr(
+ mm,
+ "_size_load",
+ lambda d, kv, chunk, four, vision = None: routes.append(("sized", vision))
+ or SimpleNamespace(weights = 10, chunk = 512),
+ )
+ monkeypatch.setattr(mm, "_lazy_checkpoint", lambda d, config: mm._builds_on_mlx_vlm(config))
+ found = lambda tree: {
+ "ple_bytes": 0,
+ "expert_bytes": 4,
+ "ple": None if tree else "no",
+ "experts": None,
+ }
+ offload = SimpleNamespace(
+ offloadable = lambda tree, d, cast_dtype: routes.append(("inventory", tree)) or found(tree)
+ )
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.offload", offload)
+ mm._clear_sizing_cache()
+ for vision in (None, True): # a speculative text load goes through mlx-vlm
+ planned = mm.mlx_offload_plan("/x", budget_bytes = 1, max_ctx = 512, vision = vision)
+ assert planned == ((1.0, 0.5) if vision else (None, 0.5))
+ parts = mm.mlx_offloadable_parts("/x", vision)
+ assert parts == (("ple", "experts") if vision else ("experts",))
+ assert routes == [("sized", None), ("inventory", False), ("sized", True), ("inventory", True)]
+
+
+def test_the_offload_probe_maps_a_text_model_on_the_route_unsloth_zoo_builds_it(monkeypatch):
+ if not _HAVE_MLX:
+ pytest.skip("resolves the loaders")
+ import mlx_lm.utils
+ import mlx_vlm.utils
+
+ monkeypatch.setattr(
+ mm, "_builds_on_mlx_vlm", lambda config: config["model_type"] == "deepseek_v4"
+ )
+ monkeypatch.setattr(mlx_vlm.utils, "load_model", lambda path, **kw: "mlx-vlm")
+ monkeypatch.setattr(mlx_lm.utils, "load_model", lambda path, **kw: ("mlx-lm", None))
+ assert mm._lazy_checkpoint("/x", {"model_type": "deepseek_v4"}) == "mlx-vlm"
+ assert mm._lazy_checkpoint("/x", {"model_type": "llama"}) == "mlx-lm"
+
+
+def test_the_offload_probe_never_evaluates_a_converted_checkpoint(tmp_path, monkeypatch):
+ if not _HAVE_MLX:
+ pytest.skip("needs MLX to build and load a checkpoint")
+ checkpoint = pytest.importorskip("mlx_vlm.quantization.checkpoint")
+ if not hasattr(checkpoint, "transform_quantized_weights"):
+ pytest.skip("this mlx-vlm converts no ModelOpt checkpoint")
+ import mlx.core as mx
+ from mlx.utils import tree_flatten
+ from mlx_lm.models import qwen3
+
+ config = {
+ "model_type": "qwen3",
+ "hidden_size": 128,
+ "intermediate_size": 256,
+ "num_hidden_layers": 1,
+ "num_attention_heads": 2,
+ "num_key_value_heads": 1,
+ "head_dim": 64,
+ "vocab_size": 64,
+ "rms_norm_eps": 1e-6,
+ "rope_theta": 1e6,
+ "max_position_embeddings": 512,
+ "tie_word_embeddings": True,
+ # mlx-vlm decodes ModelOpt's lone-scale FP8 linears to dense weights, evaluating them.
+ "quantization_config": {"quant_method": "modelopt", "quant_algo": "MIXED_PRECISION"},
+ }
+ (tmp_path / "config.json").write_text(json.dumps(config))
+ weights = {}
+ for name, value in tree_flatten(qwen3.Model(qwen3.ModelArgs.from_dict(config)).parameters()):
+ if name.endswith("proj.weight"):
+ weights[name] = mx.zeros(value.shape, dtype = mx.uint8)
+ weights[name[: -len("weight")] + "weight_scale"] = mx.array(0.5)
+ else:
+ weights[name] = value
+ mx.save_safetensors(str(tmp_path / "model.safetensors"), weights)
+ evaluated = []
+ real_eval = mx.eval
+ monkeypatch.setattr(mx, "eval", lambda *a, **k: evaluated.append(1) or real_eval(*a, **k))
+ with mm._on_route(True):
+ mm._lazy_checkpoint(str(tmp_path), json.loads((tmp_path / "config.json").read_text()))
+ assert evaluated == []
+
+ (tmp_path / "model.safetensors.index.json").write_text(
+ json.dumps({"weight_map": {"lm_head.weight": "../outside.safetensors"}})
+ )
+ with pytest.raises(ValueError, match = "outside the snapshot"):
+ mm._lazy_checkpoint(str(tmp_path), config)
try:
@@ -4206,6 +4485,25 @@ class TestPricingALoad:
config = json.loads((Path(snapshot) / "config.json").read_text())
assert mm.mlx_weight_bytes(snapshot, config, load_in_4bit = False) == 2_064_544
+ def test_an_offloaded_load_prices_what_the_offload_holds(self, tmp_path, monkeypatch):
+ self._checkpoint(tmp_path)
+ resident = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 256)
+ monkeypatch.setattr(
+ mm, "_offloaded", lambda model_dir, config, offload: (-1000 * bool(offload), offload)
+ )
+ offloaded = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 256, offload = (None, 0.5))
+ assert offloaded.weights_bytes == resident.weights_bytes - 1000
+ assert offloaded.total_bytes == resident.total_bytes - 1000
+ assert (offloaded.ple_offload, offloaded.expert_offload) == (None, 0.5)
+ monkeypatch.setattr(
+ mm, "mlx_offload_planner", lambda: SimpleNamespace(SAFETY_MARGIN = (2 << 30, 0.03))
+ )
+ price = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 8192, offload = (None, 0.5)).total_bytes
+ fit = lambda offload: mm.mlx_fit_outcome(
+ str(tmp_path), budget_bytes = price + (2 << 30), max_ctx = 1 << 20, offload = offload
+ )
+ assert fit((None, 0.5)) == ("fitted", 8192) != fit(None)
+
def test_a_quantizing_load_is_priced_below_the_shards(self, tmp_path):
config = self._checkpoint(tmp_path)
wide = mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = False)
@@ -4213,6 +4511,37 @@ class TestPricingALoad:
assert (wide, quantized) == (2_886_144, 1_001_984)
assert quantized < wide < mm._shard_bytes(str(tmp_path), config) == 2_888_265
+ def test_engram_mlx_vlm_reads_from_disk_is_not_priced(self, tmp_path, monkeypatch):
+ import mlx.core as mx
+ import mlx.nn as nn
+ from mlx.utils import tree_flatten
+
+ def build(*_):
+ model = nn.Module()
+ model.layer = nn.Module()
+ model.layer.engram, model.layer.gate = nn.Module(), nn.Linear(128, 64, bias = False)
+ model.layer.engram.embed = nn.Embedding(64, 128)
+ return model
+
+ monkeypatch.setattr(mm, "_whole_model", build)
+ path = str(tmp_path / "model.safetensors")
+ mx.save_safetensors(path, dict(tree_flatten(build().parameters())))
+ price = lambda q = False, **change: mm.mlx_weight_bytes(
+ str(tmp_path), dict(config, **change), q
+ )
+ config = {"model_type": "deepseek_v41"}
+ # Quantized at load, the table becomes a resident one.
+ held = 64 * 128 * mm._runtime_dtype().size
+ assert price(quantization = {"bits": 2}) == os.path.getsize(path) - 64 * 128 * 4
+ assert (price(), price(model_type = "deepseek_v4")) == (held, 2 * held)
+ assert price(True) == price(True, model_type = "deepseek_v4")
+ assert price(model_type = "deepseek_v4", text_config = {"model_type": "deepseek_v41"}) == held
+ monkeypatch.setattr(mm, "_whole_model", lambda *_: 1 / 0)
+ assert (price(), price(True)) == (
+ os.path.getsize(path) - 64 * 128 * 4,
+ os.path.getsize(path),
+ )
+
def test_shards_that_name_nothing_the_architecture_has_are_not_priced_from_it(self, tmp_path):
config = self._checkpoint(tmp_path, prefix = "draft.")
with pytest.raises(ValueError, match = "does not supply"):
@@ -4545,7 +4874,9 @@ class TestTheContextSearch:
def _priced(sizing, n_ctx):
return None if n_ctx in unpriceable else SimpleNamespace(total_bytes = n_ctx)
- monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: object())
+ monkeypatch.setattr(
+ mm, "_size_load", lambda *a, **kw: SimpleNamespace(offload = (None, None))
+ )
monkeypatch.setattr(mm, "_priced_at", _priced)
def test_the_answer_is_the_largest_whole_block_under_budget(self, monkeypatch):
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 80b46312574..fe8d2f14665 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -1033,6 +1033,289 @@ def test_mlx_inference_text_load_forwards_studio_settings(monkeypatch):
assert backend.models["fake/text"]["base_model"] is None
+def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatch):
+ from core.inference import mlx_inference, mlx_memory
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ _install_fake_mlx(monkeypatch)
+ calls, fits = [], []
+ _install_fake_fast_mlx(monkeypatch, calls)
+ loader = sys.modules["unsloth_zoo.mlx.loader"].FastMLXModel
+ stock, refusal = loader.from_pretrained, {"on": False}
+
+ def from_pretrained(
+ *args,
+ offload = None,
+ **kwargs,
+ ):
+ refused = refusal["on"] and isinstance(offload, dict)
+ if callable(offload):
+ offload = None if refusal["on"] else offload("/snapshot")
+ kwargs.update(
+ {"offload": {k: v for k, v in offload.items() if v is not None}} if offload else {}
+ )
+ model, tokenizer = stock(*args, **kwargs)
+ if refused:
+ raise ValueError("Unsloth: offload reads the checkpoint's tensors as stored")
+ if (offload or {}).get("experts"): # this model has experts, no PLE table
+ model._unsloth_offload = {"experts": {"on_disk_bytes": 7}}
+ return model, tokenizer
+
+ monkeypatch.setattr(loader, "from_pretrained", staticmethod(from_pretrained))
+ monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: object())
+ monkeypatch.setattr(
+ mlx_inference,
+ "_fitted_context",
+ lambda *a, **kw: fits.append(kw.get("offload")) or (None, None),
+ )
+ planned, history = [], [] # here auto takes 0.6 of the experts
+ budget = lambda retains_history: history.append(retains_history) or 1 << 40
+ monkeypatch.setattr(mlx_inference, "mlx_memory_budget", budget)
+ monkeypatch.setattr(mlx_inference, "mlx_checkpoint_window", lambda model_dir: 32768)
+ monkeypatch.setattr(
+ mlx_memory,
+ "mlx_offload_plan",
+ lambda model_dir, **kw: planned.append((model_dir, kw))
+ or (None if kw["ple"] == "auto" else kw["ple"], 0.6),
+ )
+ config = SimpleNamespace(identifier = "fake/moe", is_vision = False, is_lora = False)
+
+ base = {"max_seq_length": 0, "load_in_4bit": False}
+
+ def load(ple, experts, *more):
+ calls.clear()
+ backend = MLXInferenceBackend()
+ backend.load_model(config, offload = (ple, experts), **{**base, **dict(*more)})
+ record = backend.models["fake/moe"]["mlx_offload"]
+ return calls[-1][1].get("offload", {}), record
+
+ sent, record = load(0.5, 0.75)
+ assert sent == {"ple": 0.5, "experts": 0.75} and fits[-1] == (None, 0.75)
+ assert record == {
+ "ple": {
+ "applied": None,
+ "requested": 0.5,
+ "reason": "the model has no PLE tables unsloth_zoo can read from disk",
+ },
+ "experts": {"applied": 0.75, "requested": 0.75, "reason": None},
+ "disk_bytes": 7,
+ }
+ sent, record = load("off", "off")
+ assert sent == {} and fits[-1] is None and record["ple"]["reason"] is None
+ assert not planned
+ sent, record = load(None, None)
+ assert sent == {"experts": 0.6} and fits[-1] == (None, 0.6) and history == [True]
+ pick = lambda *keys: tuple(planned[-1][1][key] for key in keys)
+ assert planned[-1][0] == "/snapshot" and pick("ple", "experts") == ("auto", "auto")
+ assert pick("context", "max_ctx", "kv_bits", "load_in_4bit", "vision") == (
+ None,
+ 32768,
+ None,
+ False,
+ None,
+ )
+ assert (record["experts"]["applied"], record["experts"]["requested"]) == (0.6, None)
+ with monkeypatch.context() as patch: # a speculative text load is priced on mlx-vlm
+ from core.inference import mlx_speculative
+
+ patch.setattr(mlx_speculative, "speculates_on_route", lambda *a: True)
+ patch.setattr(mlx_speculative, "speculation_refusal", lambda **k: None)
+ patch.setattr(
+ mlx_speculative,
+ "resolve_speculation",
+ lambda mode, *a, **k: mlx_speculative.SpecResolution(mode),
+ )
+ patch.setattr(mlx_speculative, "build_draft", lambda *a, **k: (None, None, None, None))
+ load(None, None)
+ assert planned[-1][1]["vision"] is True
+ load("off", None, {"max_seq_length": 8192}) # an explicit off is planned as off
+ assert pick("ple", "context", "max_ctx") == (None, 8192, 8192)
+ sent, record = load(None, None, {"dtype": "float16"}) # not priced, so not planned: off
+ assert sent == {} and len(planned) == 3 and record["experts"]["reason"] is None
+ sent, record = load(0.5, None) # decided: the hand-set part the model lacks says so
+ assert sent == {"ple": 0.5, "experts": 0.6} and record["ple"]["reason"].startswith(
+ "the model has"
+ )
+ # Refused by unsloth_zoo for this load: loaded again without it, saying why; auto turns off.
+ refusal["on"] = True
+ sent, record = load("off", 1.0)
+ assert sent == {} and len(calls) == 2 and record["experts"]["applied"] is None
+ assert record["experts"]["reason"].startswith("Unsloth: offload reads")
+ sent, record = load(None, None)
+ assert sent == {} and not (record["experts"]["applied"] or record["experts"]["reason"])
+ # A hand-set part beside "auto" goes off with the planner unsloth_zoo never called, saying why.
+ sent, record = load(1.0, None)
+ assert (
+ sent == {}
+ and len(calls) == 1
+ and record["ple"]["reason"].startswith("unsloth_zoo cannot offload this load")
+ )
+ refusal["on"] = False
+ monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: None)
+ sent, record = load("off", 1.0)
+ assert (
+ sent == {}
+ and record["experts"]["reason"] == "the installed unsloth_zoo cannot read weights from disk"
+ )
+
+
+def test_auto_offload_is_planned_as_the_load_is_priced(monkeypatch):
+ from core.inference import mlx_inference as mi
+ from core.inference import mlx_memory as mm
+
+ plans = []
+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: object())
+ monkeypatch.setattr(mm, "mlx_offload_plan", lambda d, **kw: plans.append(kw) or (1.0, 0.5))
+ monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: 1 << 40)
+ monkeypatch.setattr(mi, "mlx_checkpoint_window", lambda d: 32768)
+ monkeypatch.setattr(mi, "mlx_kv_quant_is_refused", lambda d, bits: True)
+ priced = {"pinned": None, "load_in_4bit": False, "retains_history": True, "kv_bits": 2}
+ auto = lambda at, *knobs: mi.mlx_resolve_offload(at, knobs, **priced)
+ assert auto("/snap", "auto", "auto") == (1.0, 0.5) and plans[-1]["vision"] is None
+ assert (plans[-1]["budget_bytes"], plans[-1]["kv_bits"]) == (1 << 40, None)
+ mi.mlx_resolve_offload("/snap", ("auto", None), vision = True, **priced)
+ assert plans[-1]["vision"] is True
+ plans.pop()
+ assert auto("/snap", 0.5, None) == (0.5, None) and auto(None, "auto", "auto") == (None, None)
+ monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: None)
+ assert auto("/snap", "auto", "auto") == (None, None) and len(plans) == 1
+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: None)
+ monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: 1 / 0)
+ assert auto("/snap", "auto", 0.5) == (None, 0.5) and len(plans) == 1
+
+
[email protected]("decided_on_mlx_vlm", [False, True])
+def test_a_load_falling_back_to_mlx_lm_plans_auto_offload_again(monkeypatch, decided_on_mlx_vlm):
+ pytest.importorskip("mlx.core")
+ from core.inference import mlx_inference as mi
+ from core.inference import mlx_memory as mm
+ from core.inference import mlx_speculative
+
+ plans, applied = [], []
+
+ class _Loader:
+ @staticmethod
+ def from_pretrained(name, **kwargs):
+ offload = kwargs.get("offload")
+ if not kwargs["text_only"]:
+ if decided_on_mlx_vlm and callable(offload):
+ offload("/snap")
+ raise ValueError("mlx-vlm has no such architecture")
+ applied.append(offload("/snap") if callable(offload) else offload)
+ return SimpleNamespace(config = {}), SimpleNamespace()
+
+ def resolve(model_dir, knobs, **kw):
+ plans.append((knobs, kw["vision"]))
+ # A plan for the mlx-vlm route differs from the one mlx-lm needs, so a reused plan shows.
+ return (None, 0.25) if kw["vision"] else (None, 1.0)
+
+ loader = types.ModuleType("unsloth_zoo.mlx.loader")
+ loader.FastMLXModel = _Loader
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", loader)
+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: object())
+ monkeypatch.setattr(mi, "mlx_resolve_offload", resolve)
+ monkeypatch.setattr(mi, "_classify_mlx_audio_type", lambda *a, **k: None)
+ monkeypatch.setattr(
+ mi.MLXInferenceBackend, "_resolve_context_lengths", lambda *a: (8192, 8192, 8192)
+ )
+ source = mlx_speculative.DrafterSource("dflash", "/drafter", False)
+ monkeypatch.setattr(mlx_speculative, "speculation_refusal", lambda **_k: None)
+ monkeypatch.setattr(
+ mlx_speculative,
+ "resolve_speculation",
+ lambda mode, *_a, **_k: mlx_speculative.SpecResolution(mode, (source,)),
+ )
+ config = SimpleNamespace(identifier = "org/moe", is_vision = False, is_lora = False)
+
+ mi.MLXInferenceBackend().load_model(config, speculative_type = "dflash")
+ assert plans[-1] == (("auto", "auto"), None)
+ assert applied == [{"ple": None, "experts": 1.0}]
+
+
+def test_offload_knobs_validate_and_reload_on_change():
+ from pydantic import ValidationError
+
+ from core.inference import orchestrator
+ from models.inference import EstimateMemoryRequest, LoadRequest
+ from routes.inference import _mlx_runtime_settings_match
+
+ req = lambda **knobs: LoadRequest(model = "m", model_path = "m", **knobs)
+ assert (
+ req(mlx_ple_offload = 1).mlx_ple_offload,
+ req(mlx_expert_offload = "off").mlx_expert_offload,
+ ) == (1.0, "off")
+ assert req(mlx_ple_offload = "auto").mlx_ple_offload == "auto"
+ for bad in (0, 1.5, -0.5, True, "on", "0.5"):
+ for model in (LoadRequest, EstimateMemoryRequest):
+ with pytest.raises(ValidationError):
+ model(
+ model_path = "m",
+ **({"model": "m"} if model is LoadRequest else {}),
+ mlx_expert_offload = bad,
+ )
+ be = SimpleNamespace(active_model_name = "m", models = {"m": {"mlx_kv_quant_requested": "auto"}})
+ assert _mlx_runtime_settings_match(be, req(mlx_ple_offload = "auto", mlx_expert_offload = None))
+ assert not _mlx_runtime_settings_match(be, req(mlx_ple_offload = "off"))
+ assert not _mlx_runtime_settings_match(be, req(mlx_expert_offload = 0.5))
+ be.models["m"]["mlx_offload"] = {"ple": {"requested": None}, "experts": {"requested": 0.5}}
+ assert _mlx_runtime_settings_match(be, req(mlx_expert_offload = 0.5))
+ assert not _mlx_runtime_settings_match(be, req(mlx_expert_offload = 0.75))
+ assert not _mlx_runtime_settings_match(be, req(mlx_expert_offload = 0.5, mlx_ple_offload = 1.0))
+ import inspect
+
+ from routes import inference as routes
+
+ part = lambda applied, requested: {"applied": applied, "requested": requested}
+ info = {"mlx_offload": {"ple": part(1.0, 1.0), "experts": part(0.6, None)}}
+ assert "reads 60% of its MoE experts from disk" in routes._mlx_offload_warning(info)
+ info["mlx_offload"]["experts"]["requested"] = 0.6
+ assert routes._mlx_offload_warning(info) is None
+ assert inspect.getsource(routes).count("memory_warning = _mlx_offload_warning(") == 3
+ load = inspect.getsource(orchestrator.InferenceOrchestrator.load_model)
+ assert "mlx_offload" in orchestrator._MLX_RUNTIME_MIRROR_FIELDS
+ assert '"mlx_offload": tuple(mlx_offload)' in load # carried to the worker
+
+
+def test_the_worker_carries_the_offload_knobs_into_the_load_and_back(monkeypatch):
+ from core.inference import worker
+
+ model = SimpleNamespace(
+ identifier = "org/moe",
+ display_name = "moe",
+ is_vision = False,
+ is_lora = False,
+ base_model = None,
+ is_audio = False,
+ audio_type = None,
+ has_audio_input = False,
+ )
+ asked, sent = {}, []
+ entry = {"mlx_offload": {"experts": {"applied": 0.75, "requested": 0.75}, "disk_bytes": 7}}
+ backend = SimpleNamespace(
+ device = "mlx",
+ active_model_name = model.identifier,
+ models = {model.identifier: entry},
+ load_model = lambda **kw: asked.update(kw) or True,
+ )
+ monkeypatch.setattr(worker, "_build_model_config", lambda config: model)
+ monkeypatch.setattr(worker, "_run_security_gates", lambda *a, **k: True)
+ monkeypatch.setattr(worker, "_resolve_lora_4bit", lambda *a: False)
+ monkeypatch.setattr(worker, "_needs_nemotron_trust", lambda *a, **k: False)
+ xet = types.ModuleType("utils.hf_xet_fallback")
+ xet.start_watchdog = lambda **k: SimpleNamespace(set = lambda: None)
+ monkeypatch.setitem(sys.modules, "utils.hf_xet_fallback", xet)
+ queue = SimpleNamespace(put = lambda item, *a, **k: sent.append(item))
+
+ worker._handle_load(
+ backend,
+ {"model_name": model.identifier, "mlx_offload": ("off", 0.75)},
+ queue,
+ )
+ assert asked["offload"] == ("off", 0.75)
+ (loaded,) = [item for item in sent if item.get("type") == "loaded"]
+ assert {k: loaded["model_info"][k] for k in entry} == entry
+
+
@pytest.mark.parametrize("is_lora", [False, True])
def test_mlx_base_fusion_lifetime_and_lora_request_scope(monkeypatch, mlx_moe, is_lora):
backend = _install_fake_text_stack(monkeypatch, {"p": [1, 2], "generated": [7, 8]}, [])
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 9e04e304df5..a5c191dd66f 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -7607,6 +7607,19 @@ def test_the_retired_snapshot_path_entry_hands_over_its_launch_flags(override_st
assert f"{_LEGACY_SNAPSHOT}:Q4_K_M" not in resp.overrides
+def test_the_retired_snapshot_path_entry_hands_over_its_mlx_offload(override_store):
+ # A client that predates the offload knobs omits them, so retiring the row a load read keeps them.
+ settings.set_model_override(f"{_LEGACY_SNAPSHOT}:Q4_K_M", mlx_ple_offload = 1.0)
+
+ resp = _put("unsloth/B-GGUF:Q4_K_M", max_seq_length = 32768)
+ entry = resp.overrides["unsloth/B-GGUF:Q4_K_M"]
+ assert entry["mlx_ple_offload"] == 1.0 and entry["max_seq_length"] == 32768
+ assert f"{_LEGACY_SNAPSHOT}:Q4_K_M" not in resp.overrides
+ # An explicit null still resets it.
+ resp = _put("unsloth/B-GGUF:Q4_K_M", max_seq_length = 32768, mlx_ple_offload = None)
+ assert "mlx_ple_offload" not in resp.overrides["unsloth/B-GGUF:Q4_K_M"]
+
+
def test_a_standalone_gguf_save_keeps_its_filename_label_launch_flags(override_store):
path = "/models/Qwen3-8B-Q4_K_M.gguf"
settings.set_model_override(
@@ -9027,6 +9040,15 @@ def test_mlx_int8_prefill_is_stored_only_when_on_and_reaches_the_load(monkeypatc
assert "org/m" not in settings.get_model_overrides()
+def test_mlx_offload_is_stored_unless_auto_and_survives_a_client_that_omits_it(monkeypatch):
+ _mock_override_store(monkeypatch)
+ knobs = {"mlx_ple_offload": "off", "mlx_expert_offload": 0.5}
+ row = _put("m", **knobs).overrides["m"]
+ assert settings.model_override_load_kwargs(row, is_gguf = False) == row == knobs
+ assert _put("m", mlx_expert_offload = None).overrides["m"] == {"mlx_ple_offload": "off"}
+ assert "m" not in _put("m", mlx_ple_offload = "auto").overrides
+
+
def _idle_backend(kw, monkeypatch, *, user_loaded):
"""A loaded, long-idle GGUF backend wired into the idle loop."""
_reset_keepwarm()
diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py
index 90c4ce9f1dc..341c0399c17 100644
--- a/studio/backend/utils/openai_auto_switch_settings.py
+++ b/studio/backend/utils/openai_auto_switch_settings.py
@@ -550,6 +550,13 @@ def normalize_model_override(
if _coerce_bool(payload.get("mlx_int8_prefill")):
entry["mlx_int8_prefill"] = True
+ for key in ("mlx_ple_offload", "mlx_expert_offload"):
+ value = payload.get(key)
+ if value == "off" or (
+ isinstance(value, (int, float)) and not isinstance(value, bool) and 0 < value <= 1
+ ):
+ entry[key] = value if value == "off" else float(value)
+
# Stored only when off: absent is the 4-bit default, so an untouched model follows it.
if payload.get("load_in_4bit") is False:
entry["load_in_4bit"] = False
@@ -699,6 +706,8 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di
("disable_vision", "disable_vision"),
("chat_template_override", "chat_template_override"),
("mlx_int8_prefill", "mlx_int8_prefill"),
+ ("mlx_ple_offload", "mlx_ple_offload"),
+ ("mlx_expert_offload", "mlx_expert_offload"),
):
if override.get(source) is not None and source not in gguf_drops:
kwargs[target] = override[source]
diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.ts
index 9e2f53490df..ea39bba36de 100644
--- a/studio/frontend/scripts/check-bundle-budget.ts
+++ b/studio/frontend/scripts/check-bundle-budget.ts
@@ -385,8 +385,10 @@ export const BUDGET = {
// transfer (Node 26, 98 chunks). Raised to that plus about 200 B.
// Desktop Chat only upgrade (#14593), incl. the Train row opening the full install dialog, merged with main
// at c5a9fe021: 6,912,687 B raw / 2,082,221 B transfer (Node 24, 97 chunks). Raised to that plus about 200 B.
- transferBytes: 2_082_421,
- rawBytes: 6_912_887,
+ // The MLX PLE and expert offload controls (#14326): +7,406 B raw / +2,073 B transfer, plus about 500 B raw
+ // (a CI build measured about 100 B over a local one). Raised by that.
+ transferBytes: 2_084_696,
+ rawBytes: 6_920_821,
};
// The chunk count is reported but not budgeted. Splitting a page out of the entry raises it while lowering the
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 4a235e9e3e8..49f3f424216 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -2777,6 +2777,15 @@ const VISIBLE_MODEL_RUNTIME_KEYS = [
"mlxKvQuantNote",
"mlxInt8Prefill",
"loadedMlxInt8PrefillRequested",
+ "mlxPleOffload",
+ "mlxExpertOffload",
+ "loadedMlxPleOffloadRequested",
+ "loadedMlxExpertOffloadRequested",
+ "loadedMlxPleOffload",
+ "loadedMlxExpertOffload",
+ "mlxPleOffloadReason",
+ "mlxExpertOffloadReason",
+ "loadedMlxOffloadDiskBytes",
"loadedContextBudget",
"loadedIsMultimodal",
"loadedIsDiffusion",
@@ -3859,6 +3868,8 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
cache_type_kv: config.kvCacheDtype,
mlx_kv_quant: config.mlxKvQuant ?? null,
mlx_int8_prefill: config.mlxInt8Prefill ?? false,
+ mlx_ple_offload: config.mlxPleOffload ?? null,
+ mlx_expert_offload: config.mlxExpertOffload ?? null,
speculative_type: effectiveSpeculativeType,
spec_draft_n_max: effectiveSpecDraftNMax,
...mlxDrafter,
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index 7743b1b50f1..15e1594d3f7 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -2117,6 +2117,12 @@ export function useChatModelRuntime() {
const previousMlxInt8Prefill = rollbackConfig
? (rollbackConfig.mlxInt8Prefill ?? false)
: useChatRuntimeStore.getState().mlxInt8Prefill;
+ const previousMlxPleOffload = rollbackConfig
+ ? (rollbackConfig.mlxPleOffload ?? null)
+ : useChatRuntimeStore.getState().mlxPleOffload;
+ const previousMlxExpertOffload = rollbackConfig
+ ? (rollbackConfig.mlxExpertOffload ?? null)
+ : useChatRuntimeStore.getState().mlxExpertOffload;
if (isGguf && isDiffusion === undefined) {
// Prepare the token exactly as validateModel/loadModel do: the Hub rejects an invalid
// Authorization header with 401 even for a public repo, so sending the raw stored token here would
@@ -2219,6 +2225,12 @@ export function useChatModelRuntime() {
let loadMlxInt8Prefill = pendingLoadConfig
? (pendingLoadConfig.mlxInt8Prefill ?? false)
: stateBeforeUnload.mlxInt8Prefill;
+ let loadMlxPleOffload = pendingLoadConfig
+ ? (pendingLoadConfig.mlxPleOffload ?? null)
+ : stateBeforeUnload.mlxPleOffload;
+ let loadMlxExpertOffload = pendingLoadConfig
+ ? (pendingLoadConfig.mlxExpertOffload ?? null)
+ : stateBeforeUnload.mlxExpertOffload;
// gpuMemoryMode is a standing preference; the rest are per-model knobs the reset below clears, so
// they are re-baselined there in lock-step with the store. A GGUF native context can exceed
// maxSeqLength, so sizing on raw maxSeqLength could pass, unload, then have /load refuse it. A
@@ -2727,6 +2739,8 @@ export function useChatModelRuntime() {
};
loadMlxKvQuant = pendingLoadConfig?.mlxKvQuant ?? null;
loadMlxInt8Prefill = pendingLoadConfig?.mlxInt8Prefill ?? false;
+ loadMlxPleOffload = pendingLoadConfig?.mlxPleOffload ?? null;
+ loadMlxExpertOffload = pendingLoadConfig?.mlxExpertOffload ?? null;
loadChatTemplateOverride =
pendingLoadConfig?.chatTemplateOverride?.trim()
? pendingLoadConfig.chatTemplateOverride
@@ -2831,6 +2845,8 @@ export function useChatModelRuntime() {
cache_type_kv: loadKvCacheDtype,
mlx_kv_quant: loadMlxKvQuant ?? null,
mlx_int8_prefill: loadMlxInt8Prefill,
+ mlx_ple_offload: loadMlxPleOffload,
+ mlx_expert_offload: loadMlxExpertOffload,
speculative_type: loadSpeculativeType,
spec_draft_n_max: loadSpecDraftNMax,
...(targetIsMlx ? { spec_draft_model: loadSpecDraftModel } : {}),
@@ -3228,6 +3244,8 @@ export function useChatModelRuntime() {
cache_type_kv: rollbackState.loadedKvCacheDtype,
mlx_kv_quant: rollbackState.loadedMlxKvQuantRequested,
mlx_int8_prefill: rollbackState.loadedMlxInt8PrefillRequested,
+ mlx_ple_offload: rollbackState.loadedMlxPleOffloadRequested,
+ mlx_expert_offload: rollbackState.loadedMlxExpertOffloadRequested,
speculative_type:
rollbackState.loadedSpeculativeType,
spec_draft_n_max:
@@ -3337,6 +3355,8 @@ export function useChatModelRuntime() {
// nParallel above.
mlxKvQuant: previousMlxKvQuant,
mlxInt8Prefill: previousMlxInt8Prefill,
+ mlxPleOffload: previousMlxPleOffload,
+ mlxExpertOffload: previousMlxExpertOffload,
loadedChatTemplateOverride:
rollbackState.loadedChatTemplateOverride,
...loadedGpuMemoryFields(rollbackResponse),
diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
index 16f26e6ef66..02e38f13655 100644
--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
@@ -45,6 +45,7 @@ import { resolveBatchSizeSeed } from "./resolve-batch-size-seed";
import { resolveChatTemplateSeed } from "./resolve-chat-template-seed";
import { resolveCtxPinSeed } from "./resolve-ctx-pin-seed";
import { engineLaunchFieldsFrom } from "./engine-launch-fields";
+import { mlxOffloadStateFrom } from "./mlx-runtime-state";
import { resolveLlamaExtraArgsSeed } from "./resolve-llama-extra-args-seed";
import { shouldSeedVisionSwitch } from "./resolve-vision-switch-seed";
@@ -245,6 +246,7 @@ export function applyActiveModelStatusToStore(
const currentSpecType = normalizeSpeculativeType(status.speculative_type);
const incomingMlxKvQuant = normalizeMlxKvQuant(status.mlx_kv_quant_requested);
const incomingMlxInt8Prefill = status.mlx_int8_prefill_requested === true;
+ const incomingMlxOffload = mlxOffloadStateFrom(status);
const prevState = useChatRuntimeStore.getState();
// The chat's own level when the model this replaces was running a per-model pin's: the clamp
// narrows the chat's level to this model's ladder, and a pin is one model's, not the chat's.
@@ -584,6 +586,17 @@ export function applyActiveModelStatusToStore(
status.chat_template_override_reason ?? null,
mlxKvQuantNote: status.mlx_kv_quant_note ?? null,
loadedMlxInt8PrefillRequested: incomingMlxInt8Prefill,
+ ...incomingMlxOffload,
+ ...((hydratingExistingModel ||
+ (prevState.mlxPleOffload ?? null) ===
+ (prevState.loadedMlxPleOffloadRequested ?? null)) && {
+ mlxPleOffload: incomingMlxOffload.loadedMlxPleOffloadRequested,
+ }),
+ ...((hydratingExistingModel ||
+ (prevState.mlxExpertOffload ?? null) ===
+ (prevState.loadedMlxExpertOffloadRequested ?? null)) && {
+ mlxExpertOffload: incomingMlxOffload.loadedMlxExpertOffloadRequested,
+ }),
...((hydratingExistingModel ||
(prevState.mlxKvQuant ?? null) ===
(prevState.loadedMlxKvQuantRequested ?? null)) && {
@@ -602,6 +615,7 @@ export function applyActiveModelStatusToStore(
chatTemplateOverrideReason: null,
mlxKvQuantNote: null,
loadedMlxInt8PrefillRequested: false,
+ ...incomingMlxOffload,
})),
// Baseline only, never the control: the echo is the RESOLVED count and would pin a blank
...(seedLoadParams &&
diff --git a/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts b/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts
index 5dee433b4fb..19e7f48eaa9 100644
--- a/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts
+++ b/studio/frontend/src/features/chat/lib/mlx-runtime-state.ts
@@ -3,7 +3,9 @@
import {
normalizeMlxKvQuant,
+ normalizeMlxOffload,
type MlxKvQuant,
+ type MlxOffload,
} from "@/features/model-picker/model-config/per-model-config";
import type { LoadModelResponse } from "../types/api";
@@ -15,10 +17,39 @@ type MlxRuntimeResponse = Pick<
| "chat_template_override_reason"
| "mlx_kv_quant_note"
| "mlx_int8_prefill_requested"
+ | "mlx_offload"
>;
-/** MLX runtime state a load response establishes (KV quantization, template verdict, int8 prefill). A
- * non-MLX response retires the verdicts but omits mlxKvQuant and mlxInt8Prefill: the settings are
+function appliedFraction(value: unknown): number | null {
+ const fraction = normalizeMlxOffload(value);
+ return typeof fraction === "number" ? fraction : null;
+}
+
+export function mlxOffloadStateFrom(resp: Partial<MlxRuntimeResponse>): {
+ loadedMlxPleOffloadRequested: MlxOffload;
+ loadedMlxExpertOffloadRequested: MlxOffload;
+ loadedMlxPleOffload: number | null;
+ loadedMlxExpertOffload: number | null;
+ mlxPleOffloadReason: string | null;
+ mlxExpertOffloadReason: string | null;
+ loadedMlxOffloadDiskBytes: number | null;
+} {
+ const offload = resp.is_mlx === true ? resp.mlx_offload : null;
+ const disk = offload?.disk_bytes;
+ return {
+ loadedMlxPleOffloadRequested: normalizeMlxOffload(offload?.ple?.requested),
+ loadedMlxExpertOffloadRequested: normalizeMlxOffload(offload?.experts?.requested),
+ loadedMlxPleOffload: appliedFraction(offload?.ple?.applied),
+ loadedMlxExpertOffload: appliedFraction(offload?.experts?.applied),
+ mlxPleOffloadReason: offload?.ple?.reason ?? null,
+ mlxExpertOffloadReason: offload?.experts?.reason ?? null,
+ loadedMlxOffloadDiskBytes:
+ typeof disk === "number" && Number.isFinite(disk) && disk > 0 ? disk : null,
+ };
+}
+
+/** MLX runtime state a load response establishes (KV quantization, template verdict, int8 prefill, offload). A
+ * non-MLX response retires the verdicts but omits the chosen settings: they are
* dormant there, not wrong, and a preset carrying the width must survive the round-trip. */
export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {
mlxKvQuant?: MlxKvQuant | null;
@@ -28,7 +59,10 @@ export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {
mlxKvQuantNote: string | null;
mlxInt8Prefill?: boolean;
loadedMlxInt8PrefillRequested: boolean;
-} {
+ mlxPleOffload?: MlxOffload;
+ mlxExpertOffload?: MlxOffload;
+} & ReturnType<typeof mlxOffloadStateFrom> {
+ const offload = mlxOffloadStateFrom(resp);
if (resp.is_mlx !== true) {
return {
loadedMlxKvQuantRequested: null,
@@ -36,6 +70,7 @@ export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {
chatTemplateOverrideReason: null,
mlxKvQuantNote: null,
loadedMlxInt8PrefillRequested: false,
+ ...offload,
};
}
const requested = normalizeMlxKvQuant(resp.mlx_kv_quant_requested);
@@ -47,5 +82,8 @@ export function mlxRuntimeStateFrom(resp: MlxRuntimeResponse): {
mlxKvQuantNote: resp.mlx_kv_quant_note ?? null,
mlxInt8Prefill: resp.mlx_int8_prefill_requested === true,
loadedMlxInt8PrefillRequested: resp.mlx_int8_prefill_requested === true,
+ mlxPleOffload: offload.loadedMlxPleOffloadRequested,
+ mlxExpertOffload: offload.loadedMlxExpertOffloadRequested,
+ ...offload,
};
}
diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.ts
index 46aea025d7a..e2b23d7247e 100644
--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts
+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts
@@ -2,7 +2,10 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import type { PerModelConfig } from "@/features/model-picker";
-import { normalizeMlxKvQuant } from "@/features/model-picker/model-config/per-model-config";
+import {
+ normalizeMlxKvQuant,
+ normalizeMlxOffload,
+} from "@/features/model-picker/model-config/per-model-config";
import {
parseGpuLayersOverride,
@@ -33,6 +36,7 @@ type ResidentRuntime = Pick<
| "cache_type_kv"
| "mlx_kv_quant_requested"
| "mlx_int8_prefill_requested"
+ | "mlx_offload"
| "speculative_type"
| "spec_draft_n_max"
| "spec_draft_model"
@@ -291,6 +295,19 @@ const SETTING_CHECKS: SettingCheck[] = [
agrees: (c, s) =>
Boolean(c.mlxInt8Prefill) === (s.mlx_int8_prefill_requested === true),
},
+ {
+ // The request, not the applied fraction: Auto stays Auto whatever it picked.
+ mlxComparable: true,
+ pinned: () => true,
+ agrees: (c, s) =>
+ (c.mlxPleOffload ?? null) === normalizeMlxOffload(s.mlx_offload?.ple?.requested),
+ },
+ {
+ mlxComparable: true,
+ pinned: () => true,
+ agrees: (c, s) =>
+ (c.mlxExpertOffload ?? null) === normalizeMlxOffload(s.mlx_offload?.experts?.requested),
+ },
{
// Always pinned: an unset mode resolves to the standing preference and the load sends it. Reading
// it as silence let a pick asking for "off" adopt a resident MTP runtime.
diff --git a/studio/frontend/src/features/chat/presets/preset-load-config.ts b/studio/frontend/src/features/chat/presets/preset-load-config.ts
index 9d87b950499..8d4fcd1fde0 100644
--- a/studio/frontend/src/features/chat/presets/preset-load-config.ts
+++ b/studio/frontend/src/features/chat/presets/preset-load-config.ts
@@ -343,6 +343,8 @@ export function applyPresetLoadConfig(config?: PresetLoadConfig | null): void {
chatTemplateOverride: null,
// Not carried by presets: enabling it always goes through its confirmation.
mlxInt8Prefill: store.mlxInt8Prefill,
+ mlxPleOffload: store.mlxPleOffload,
+ mlxExpertOffload: store.mlxExpertOffload,
gpuMemoryMode: config.gpuMemoryMode,
gpuLayers: config.gpuLayers,
nCpuMoe: config.nCpuMoe,
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index 828f8448640..a39cccbe0be 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -2013,6 +2013,8 @@ export function SharedComposer({
cache_type_kv: ownConfig.kvCacheDtype ?? null,
mlx_kv_quant: ownConfig.mlxKvQuant ?? null,
mlx_int8_prefill: ownConfig.mlxInt8Prefill ?? false,
+ mlx_ple_offload: ownConfig.mlxPleOffload ?? null,
+ mlx_expert_offload: ownConfig.mlxExpertOffload ?? null,
speculative_type: effectiveSpeculativeType,
spec_draft_n_max: effectiveSpecDraftNMax,
...(targetIsMlx ? { spec_draft_model: ownConfig.specDraftModel ?? null } : {}),
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index 7788edde1c1..82dc4aaab36 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -16,6 +16,7 @@ import {
import {
loadedContextFields,
type MlxKvQuant,
+ type MlxOffload,
} from "@/features/model-picker/model-config/per-model-config";
import {
cachedPinnableGpuIndexKind,
@@ -2660,6 +2661,17 @@ type ChatRuntimeStore = {
mlxKvQuantNote: string | null;
mlxInt8Prefill: boolean;
loadedMlxInt8PrefillRequested: boolean;
+ /** MLX offload knobs as chosen (null = Auto), as the resident load asked for them, the
+ * fraction it applied (null = none), and why a request was not applied. */
+ mlxPleOffload: MlxOffload;
+ mlxExpertOffload: MlxOffload;
+ loadedMlxPleOffloadRequested: MlxOffload;
+ loadedMlxExpertOffloadRequested: MlxOffload;
+ loadedMlxPleOffload: number | null;
+ loadedMlxExpertOffload: number | null;
+ mlxPleOffloadReason: string | null;
+ mlxExpertOffloadReason: string | null;
+ loadedMlxOffloadDiskBytes: number | null;
loadedKvCacheDtype: string | null;
speculativeType: string | null;
loadedSpeculativeType: string | null;
@@ -4621,6 +4633,15 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
mlxKvQuantNote: null,
mlxInt8Prefill: false,
loadedMlxInt8PrefillRequested: false,
+ mlxPleOffload: null,
+ mlxExpertOffload: null,
+ loadedMlxPleOffloadRequested: null,
+ loadedMlxExpertOffloadRequested: null,
+ loadedMlxPleOffload: null,
+ loadedMlxExpertOffload: null,
+ mlxPleOffloadReason: null,
+ mlxExpertOffloadReason: null,
+ loadedMlxOffloadDiskBytes: null,
loadedKvCacheDtype: null,
speculativeType: readPersistedSpeculativeType(),
loadedSpeculativeType: null,
@@ -5614,6 +5635,15 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
mlxKvQuantNote: null,
mlxInt8Prefill: false,
loadedMlxInt8PrefillRequested: false,
+ mlxPleOffload: null,
+ mlxExpertOffload: null,
+ loadedMlxPleOffloadRequested: null,
+ loadedMlxExpertOffloadRequested: null,
+ loadedMlxPleOffload: null,
+ loadedMlxExpertOffload: null,
+ mlxPleOffloadReason: null,
+ mlxExpertOffloadReason: null,
+ loadedMlxOffloadDiskBytes: null,
loadedKvCacheDtype: null,
speculativeType: readPersistedSpeculativeType(),
loadedSpeculativeType: null,
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 5cfdc3914cd..aa9dad0285f 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -12,6 +12,20 @@ export type MmprojFallbackReason =
| "projector_incompatible"
| "projector_startup_failure";
+/** One MLX offload knob on the resident load: the fraction read from disk (null = none), as asked, and why a
+ * request is not applied. */
+export interface MlxOffloadPart {
+ applied: number | null;
+ requested: "auto" | "off" | number | null;
+ reason: string | null;
+}
+
+export interface MlxOffloadReport {
+ ple: MlxOffloadPart;
+ experts: MlxOffloadPart;
+ disk_bytes: number | null;
+}
+
export interface BackendModelDetails {
id: string;
name?: string | null;
@@ -78,6 +92,9 @@ export interface LoadModelRequest {
cache_type_kv?: string | null;
mlx_kv_quant?: string | null;
mlx_int8_prefill?: boolean;
+ /** MLX only: share of the PLE tables / routed experts read from disk, "off", or null for Auto. */
+ mlx_ple_offload?: "off" | number | null;
+ mlx_expert_offload?: "off" | number | null;
/** Speculative decoding mode for GGUF models: "auto" (platform-aware DSpark/DFlash when the model
* ships that sidecar, else MTP on MTP GGUFs, ngram-mod for sub-3B), "mtp", "dspark",
* "dflash", "ngram", "mtp+ngram", "off". The legacy spellings are still accepted. */
@@ -299,6 +316,7 @@ export interface LoadModelResponse {
mlx_int8_prefill?: boolean | null;
mlx_int8_prefill_requested?: boolean | null;
mlx_int8_prefill_reason?: string | null;
+ mlx_offload?: MlxOffloadReport | null;
chat_template?: string | null;
/** Canonical UI-facing mode the load request resolved to. See LoadModelRequest. */
speculative_type?: string | null;
@@ -457,6 +475,7 @@ export interface InferenceStatusResponse {
mlx_int8_prefill?: boolean | null;
mlx_int8_prefill_requested?: boolean | null;
mlx_int8_prefill_reason?: string | null;
+ mlx_offload?: MlxOffloadReport | null;
chat_template_override?: string | null;
/** Canonical UI-facing mode currently active. See LoadModelRequest. */
speculative_type?: string | null;
diff --git a/studio/frontend/src/features/model-picker/api/memory-estimate.ts b/studio/frontend/src/features/model-picker/api/memory-estimate.ts
index f3b823a30af..3c74b69457a 100644
--- a/studio/frontend/src/features/model-picker/api/memory-estimate.ts
+++ b/studio/frontend/src/features/model-picker/api/memory-estimate.ts
@@ -63,8 +63,14 @@ export interface MemoryEstimate {
gpuLayers: number | null;
/** `--n-cpu-moe` is set, so the GPU figure ignores it and reads high. */
moeOffloadUnmodelled: boolean;
+ /** MLX: share of the PLE tables / routed experts read from disk, priced in `weightsBytes`. */
+ mlxPleOffload: number | null;
+ mlxExpertOffload: number | null;
+ mlxOffloadable: MlxOffloadablePart[];
}
+export type MlxOffloadablePart = "ple" | "experts";
+
/** The load settings that move the estimate. Mirrors the fields /load takes. */
export interface MemoryEstimateRequest {
modelPath: string;
@@ -75,6 +81,8 @@ export interface MemoryEstimateRequest {
cacheTypeKv?: string | null;
maxSeqLength?: number | null;
mlxKvQuant?: string | null;
+ mlxPleOffload?: "off" | number | null;
+ mlxExpertOffload?: "off" | number | null;
nParallel?: number | null;
nBatch?: number | null;
nUbatch?: number | null;
@@ -118,6 +126,9 @@ const UNAVAILABLE: MemoryEstimate = {
layerCount: null,
gpuLayers: null,
moeOffloadUnmodelled: false,
+ mlxPleOffload: null,
+ mlxExpertOffload: null,
+ mlxOffloadable: [],
};
interface ApiEstimateResponse {
@@ -146,6 +157,7 @@ interface ApiEstimateResponse {
layer_count: number | null;
gpu_layers: number | null;
moe_offload_unmodelled: boolean;
+ mlx_offload?: { ple?: unknown; experts?: unknown; offloadable?: unknown } | null;
}
function estimateRequestBody(
@@ -161,6 +173,8 @@ function estimateRequestBody(
cache_type_kv: payload.cacheTypeKv ?? null,
max_seq_length: payload.maxSeqLength ?? null,
mlx_kv_quant: payload.mlxKvQuant ?? null,
+ mlx_ple_offload: payload.mlxPleOffload ?? null,
+ mlx_expert_offload: payload.mlxExpertOffload ?? null,
n_parallel: payload.nParallel ?? null,
n_batch: payload.nBatch ?? null,
n_ubatch: payload.nUbatch ?? null,
@@ -210,6 +224,18 @@ function flag(value: unknown, fallback: boolean): boolean {
return typeof value === "boolean" ? value : fallback;
}
+function offloadFraction(value: unknown): number | null {
+ return typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1
+ ? value
+ : null;
+}
+
+function offloadableParts(value: unknown): MlxOffloadablePart[] {
+ return Array.isArray(value)
+ ? value.filter((part): part is MlxOffloadablePart => part === "ple" || part === "experts")
+ : [];
+}
+
const ESTIMATE_REASONS: readonly MemoryEstimateReason[] = [
"not_gguf",
"not_downloaded",
@@ -266,6 +292,9 @@ function toMemoryEstimate(body: ApiEstimateResponse): MemoryEstimate {
layerCount: nullableCount(body.layer_count),
gpuLayers: nullableCount(body.gpu_layers),
moeOffloadUnmodelled: flag(body.moe_offload_unmodelled, false),
+ mlxPleOffload: offloadFraction(body.mlx_offload?.ple),
+ mlxExpertOffload: offloadFraction(body.mlx_offload?.experts),
+ mlxOffloadable: offloadableParts(body.mlx_offload?.offloadable),
};
}
diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.ts
index 79fb5c01582..73d67531a2e 100644
--- a/studio/frontend/src/features/model-picker/api/model-overrides.ts
+++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts
@@ -45,6 +45,10 @@ export interface ApiModelOverride {
mlx_kv_bits?: number;
// biome-ignore lint/style/useNamingConvention: API schema
mlx_int8_prefill?: boolean;
+ // biome-ignore lint/style/useNamingConvention: API schema
+ mlx_ple_offload?: "off" | number | null;
+ // biome-ignore lint/style/useNamingConvention: API schema
+ mlx_expert_offload?: "off" | number | null;
// Stored only when false: absent is the 4-bit default.
// biome-ignore lint/style/useNamingConvention: API schema
load_in_4bit?: boolean;
@@ -391,6 +395,8 @@ export function fromApiOverride(
tensorParallel: override.tensor_parallel ?? local.tensorParallel,
disableVision: override.disable_vision ?? local.disableVision,
mlxInt8Prefill: override.mlx_int8_prefill ?? local.mlxInt8Prefill,
+ mlxPleOffload: override.mlx_ple_offload ?? local.mlxPleOffload,
+ mlxExpertOffload: override.mlx_expert_offload ?? local.mlxExpertOffload,
loadIn4bit: override.load_in_4bit ?? local.loadIn4bit,
chatTemplateOverride:
override.chat_template_override ?? local.chatTemplateOverride,
@@ -430,6 +436,9 @@ export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {
engine_kv_cache_dtype: config.engineKvCacheDtype ?? "auto",
engine_seed: config.engineSeed ?? null,
engine_gpu_memory_utilization: config.engineGpuMemoryUtilization ?? null,
+ // Sent even at Auto: the server keeps a stored knob the payload omits.
+ mlx_ple_offload: config.mlxPleOffload ?? null,
+ mlx_expert_offload: config.mlxExpertOffload ?? null,
};
if (config.maxSeqLength && config.maxSeqLength > 0) {
payload.max_seq_length = config.maxSeqLength;
diff --git a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx
index b0185ae3c82..703a1d38ebf 100644
--- a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx
+++ b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx
@@ -11,9 +11,11 @@ import {
import type { MemoryEstimate } from "../api/memory-estimate";
import {
type MemoryFitVerdict,
+ NOTE_SEPARATOR,
formatMemoryGb,
glueNoteItems,
memoryFigureCandidates,
+ mlxOffloadPercent,
resolveDraftCacheNote,
resolveKvNote,
resolveMemoryFit,
@@ -225,6 +227,17 @@ export function MemoryEstimateRow({
},
);
const kvNote = resolveKvNote(estimate);
+ const weightsNote = [
+ estimate.gpuLayers != null && estimate.layerCount != null
+ ? `${estimate.gpuLayers} of ${estimate.layerCount + 1} layers on GPU`
+ : null,
+ estimate.mlxPleOffload ? `PLE ${mlxOffloadPercent(estimate.mlxPleOffload)} on disk` : null,
+ estimate.mlxExpertOffload
+ ? `experts ${mlxOffloadPercent(estimate.mlxExpertOffload)} on disk`
+ : null,
+ ]
+ .filter(Boolean)
+ .join(NOTE_SEPARATOR);
const draftCacheNote = resolveDraftCacheNote(
estimate.drafterRuntimeGpuBytes,
estimate.drafterRuntimeBytes,
@@ -290,11 +303,7 @@ export function MemoryEstimateRow({
<MemoryBreakdownLine
label="Weights"
value={formatMemoryGb(estimate.weightsBytes)}
- note={
- estimate.gpuLayers != null && estimate.layerCount != null
- ? `${estimate.gpuLayers} of ${estimate.layerCount + 1} layers on GPU`
- : undefined
- }
+ note={weightsNote || undefined}
/>
<MemoryBreakdownLine
label="KV cache"
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index 49351de3509..9dfcde89709 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -102,7 +102,7 @@ import {
loadManagedLlamaFlags,
subscribeLlamaFlagCatalog,
} from "../api/llama-flags";
-import { type MemoryEstimate } from "../api/memory-estimate";
+import { type MemoryEstimate, type MlxOffloadablePart } from "../api/memory-estimate";
import { fetchMlxDrafters } from "../api/mlx-drafters";
import {
resolveEstimateContext,
@@ -110,7 +110,14 @@ import {
resolveMlxServedWindow,
shouldRequestMemoryEstimate,
} from "../model-config/estimate-context";
-import { resolveReclaimableMemoryCredit } from "../model-config/memory-fit";
+import {
+ MLX_OFFLOAD_SLIDER_AUTO,
+ mlxOffloadDecided,
+ mlxOffloadFromSlider,
+ mlxOffloadModePatch,
+ mlxOffloadSliderValue,
+ resolveReclaimableMemoryCredit,
+} from "../model-config/memory-fit";
import {
resolveResidentEstimateRequest,
selectResidentEstimateSettings,
@@ -178,6 +185,7 @@ import {
MLX_KV_QUANTS,
mlxKvQuantLabel,
type MlxKvQuant,
+ type MlxOffload,
N_BATCH_LLAMA_DEFAULT,
N_BATCH_MAX,
N_BATCH_MIN,
@@ -1216,6 +1224,115 @@ function GpuMemorySettings({
const MLX_KV_QUANT_AUTO = "auto";
+const MLX_OFFLOAD_SLIDERS = [
+ {
+ part: "experts",
+ key: "mlxExpertOffload",
+ label: "Experts on Disk",
+ hint: "Share of the MoE experts read from disk as they are needed instead of held in memory. Saves memory but slows generation a lot. 0% keeps every expert in memory; Auto leaves it to Unsloth.",
+ },
+ {
+ part: "ple",
+ key: "mlxPleOffload",
+ label: "PLE on Disk",
+ hint: "Share of the per-layer embedding tables read from disk instead of held in memory. Saves memory at a small cost to generation speed. 0% keeps them all in memory; Auto leaves it to Unsloth.",
+ },
+] as const satisfies readonly {
+ part: MlxOffloadablePart;
+ key: "mlxPleOffload" | "mlxExpertOffload";
+ label: string;
+ hint: string;
+}[];
+
+function MlxOffloadSettings({
+ config,
+ update,
+ offloadable,
+ decided,
+ notes,
+}: {
+ config: PerModelConfig;
+ update: (patch: Partial<PerModelConfig>) => void;
+ offloadable: readonly MlxOffloadablePart[];
+ decided: Record<MlxOffloadablePart, number | null> | null;
+ notes: Record<MlxOffloadablePart, string | null>;
+}) {
+ const sliders = MLX_OFFLOAD_SLIDERS.filter(
+ ({ part, key }) => offloadable.includes(part) || config[key] != null,
+ );
+ if (sliders.length === 0) return null;
+ const isManual = sliders.some(({ key }) => config[key] != null);
+ const shownNotes = sliders.map(({ part }) => notes[part]).filter(Boolean);
+ return (
+ <>
+ <div className={ROW_CLASS}>
+ <div className="flex min-w-0 items-center gap-1.5">
+ <span className={LABEL_CLASS}>Disk Offload</span>
+ <InfoHint>
+ <div className="flex flex-col gap-1.5">
+ <div>
+ <span className="font-medium">Default:</span> Unsloth reads
+ parts of the model from disk only when it would not otherwise
+ fit in memory.
+ </div>
+ <div>
+ <span className="font-medium">Manual:</span> set how much of
+ each part stays on disk.
+ </div>
+ </div>
+ </InfoHint>
+ </div>
+ <Select
+ value={isManual ? "manual" : "auto"}
+ onValueChange={(v) => {
+ if (v === "manual" && !decided) return;
+ update(
+ mlxOffloadModePatch(
+ v === "manual",
+ sliders.map(({ part, key }) => ({ key, decided: decided?.[part] ?? null })),
+ ),
+ );
+ }}
+ >
+ <SelectTrigger
+ animateRadius={false}
+ icon={ChevronDownStandardIcon}
+ iconClassName="size-3.5"
+ className={SELECT_TRIGGER_CLASS}
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">
+ <SelectItem value="auto">Default</SelectItem>
+ <SelectItem value="manual" disabled={!isManual && !decided}>
+ Manual
+ </SelectItem>
+ </SelectContent>
+ </Select>
+ </div>
+ {isManual &&
+ sliders.map(({ part, key, label, hint }) => (
+ <AdvancedGpuSlider
+ key={part}
+ label={label}
+ value={mlxOffloadSliderValue(config[key])}
+ min={MLX_OFFLOAD_SLIDER_AUTO}
+ max={100}
+ step={5}
+ onChange={(v) => update({ [key]: mlxOffloadFromSlider(v) })}
+ format={(v) => (v < 0 ? "Auto" : `${v}%`)}
+ info={hint}
+ />
+ ))}
+ {shownNotes.map((note) => (
+ <p key={note} className="text-ui-11 text-muted-foreground">
+ {note}
+ </p>
+ ))}
+ </>
+ );
+}
+
function AdvancedSettingsToggle({
checked,
onCheckedChange,
@@ -1511,6 +1628,9 @@ function MlxAdvancedSettings({
servedByMlx,
int8PrefillAvailable,
onInt8PrefillChange,
+ offloadable,
+ offloadDecided,
+ offloadNotes,
speculativeFallback,
modelPath,
onEditTemplate,
@@ -1524,6 +1644,9 @@ function MlxAdvancedSettings({
servedByMlx: boolean;
int8PrefillAvailable: boolean;
onInt8PrefillChange: (checked: boolean) => void;
+ offloadable: readonly MlxOffloadablePart[];
+ offloadDecided: Record<MlxOffloadablePart, number | null> | null;
+ offloadNotes: Record<MlxOffloadablePart, string | null>;
speculativeFallback: string;
modelPath: string;
onEditTemplate: () => void;
@@ -1606,6 +1729,15 @@ function MlxAdvancedSettings({
{servedByMlx && (
<ParallelSlotsRow config={config} update={update} hint={MLX_PARALLEL_HINT} />
)}
+ {servedByMlx && (
+ <MlxOffloadSettings
+ config={config}
+ update={update}
+ offloadable={offloadable}
+ decided={offloadDecided}
+ notes={offloadNotes}
+ />
+ )}
<div className="space-y-1">
<ChatTemplateSetting
config={config}
@@ -2443,6 +2575,16 @@ export function ModelConfigPage({
useShallow(selectResidentEstimateSettings),
);
const mlxKvQuantNote = useChatRuntimeStore((s) => s.mlxKvQuantNote);
+ const loadedMlxPleOffloadRequested = useChatRuntimeStore(
+ (s) => s.loadedMlxPleOffloadRequested,
+ );
+ const loadedMlxExpertOffloadRequested = useChatRuntimeStore(
+ (s) => s.loadedMlxExpertOffloadRequested,
+ );
+ const loadedMlxPleOffload = useChatRuntimeStore((s) => s.loadedMlxPleOffload);
+ const loadedMlxExpertOffload = useChatRuntimeStore((s) => s.loadedMlxExpertOffload);
+ const mlxPleOffloadReason = useChatRuntimeStore((s) => s.mlxPleOffloadReason);
+ const mlxExpertOffloadReason = useChatRuntimeStore((s) => s.mlxExpertOffloadReason);
const loadedMlxKvQuantRequested = useChatRuntimeStore(
(s) => s.loadedMlxKvQuantRequested,
);
@@ -2630,16 +2772,23 @@ export function ModelConfigPage({
const [initialMlxInt8Prefill] = useState(
() => configState.mlxInt8Prefill ?? false,
);
+ const [initialMlxOffload] = useState(
+ () => configState.mlxPleOffload != null || configState.mlxExpertOffload != null,
+ );
// Applicability stays live, unlike the snapshot above: MLX can become available after mount,
// and a width that starts applying then has to surface.
const autoOpenForMlxKvQuant = servedByMlx && initialMlxKvQuant != null;
const autoOpenForMlxInt8Prefill = servedByMlx && initialMlxInt8Prefill;
+ const autoOpenForMlxOffload = servedByMlx && initialMlxOffload;
// Opened to change one field: show its section for this visit, leaving the saved switch alone.
const [revealForFocus, setRevealForFocus] = useState(focusField != null);
const showAdvanced =
revealForFocus ||
(advancedPreference ??
- (autoOpenAdvanced || autoOpenForMlxKvQuant || autoOpenForMlxInt8Prefill));
+ (autoOpenAdvanced ||
+ autoOpenForMlxKvQuant ||
+ autoOpenForMlxInt8Prefill ||
+ autoOpenForMlxOffload));
const toggleAdvanced = (open: boolean) => {
setRevealForFocus(false);
saveAdvancedSettingsOpen(open);
@@ -3257,6 +3406,8 @@ export function ModelConfigPage({
? null
: resolveMlxEstimateContext(savedContextPin(config)),
mlxKvQuant: runtimeConfig.mlxKvQuant ?? null,
+ mlxPleOffload: targetIsMlx ? (runtimeConfig.mlxPleOffload ?? null) : null,
+ mlxExpertOffload: targetIsMlx ? (runtimeConfig.mlxExpertOffload ?? null) : null,
nParallel: runtimeConfig.nParallel,
nBatch: runtimeConfig.nBatch,
nUbatch: runtimeConfig.nUbatch,
@@ -3287,6 +3438,24 @@ export function ModelConfigPage({
}
: null;
const memoryEstimate = useMemoryEstimate(memoryEstimateRequest);
+ // The loaded model's reason answers only the request it was given.
+ const mlxOffloadNote = (
+ value: MlxOffload | undefined,
+ loadedRequested: MlxOffload,
+ reason: string | null,
+ ): string | null =>
+ isActiveModel && reason && (value ?? null) === (loadedRequested ?? null)
+ ? reason
+ : null;
+ const mlxOffloadNotes = {
+ ple: mlxOffloadNote(config.mlxPleOffload, loadedMlxPleOffloadRequested, mlxPleOffloadReason),
+ experts: mlxOffloadNote(
+ config.mlxExpertOffload,
+ loadedMlxExpertOffloadRequested,
+ mlxExpertOffloadReason,
+ ),
+ };
+ const mlxOffloadDecidedNow = mlxOffloadDecided(memoryEstimate);
const mlxFittedWindow = targetIsMlx
? servedWindow(memoryEstimate.estimate?.contextFitted)
: null;
@@ -3318,7 +3487,11 @@ export function ModelConfigPage({
residentEstimateSettings,
residentContext,
targetIsMlx && !residentModelLoading
- ? { kvQuant: loadedMlxKvQuantRequested ?? null }
+ ? {
+ kvQuant: loadedMlxKvQuantRequested ?? null,
+ pleOffload: loadedMlxPleOffload,
+ expertOffload: loadedMlxExpertOffload,
+ }
: undefined,
);
const residentEstimate = useMemoryEstimate(residentEstimateRequest, {
@@ -3958,6 +4131,9 @@ export function ModelConfigPage({
? setInt8PrefillConfirmOpen(true)
: update({ mlxInt8Prefill: false })
}
+ offloadable={memoryEstimate.estimate?.mlxOffloadable ?? []}
+ offloadDecided={mlxOffloadDecidedNow}
+ offloadNotes={mlxOffloadNotes}
speculativeFallback={speculativeFallback}
modelPath={target.id}
onEditTemplate={() => setTemplateOpen(true)}
diff --git a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
index 3b166a93919..27920cc88aa 100644
--- a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
+++ b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
@@ -41,6 +41,8 @@ export function useActiveModelConfig(): ActiveModelConfigState {
const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype);
const mlxKvQuant = useChatRuntimeStore((s) => s.mlxKvQuant);
const mlxInt8Prefill = useChatRuntimeStore((s) => s.mlxInt8Prefill);
+ const mlxPleOffload = useChatRuntimeStore((s) => s.mlxPleOffload);
+ const mlxExpertOffload = useChatRuntimeStore((s) => s.mlxExpertOffload);
const speculativeType = useChatRuntimeStore((s) => s.speculativeType);
const specDraftNMax = useChatRuntimeStore((s) => s.specDraftNMax);
const nParallel = useChatRuntimeStore((s) => s.nParallel);
@@ -100,6 +102,8 @@ export function useActiveModelConfig(): ActiveModelConfigState {
// over a field it cannot show.
const effectiveMlxKvQuant = isMlx ? (mlxKvQuant ?? null) : null;
const effectiveMlxInt8Prefill = isMlx && mlxInt8Prefill;
+ const effectiveMlxPleOffload = isMlx ? (mlxPleOffload ?? null) : null;
+ const effectiveMlxExpertOffload = isMlx ? (mlxExpertOffload ?? null) : null;
const config = useMemo<PerModelConfig | null>(() => {
if (!checkpoint || isExternalModelId(checkpoint)) {
@@ -117,6 +121,8 @@ export function useActiveModelConfig(): ActiveModelConfigState {
kvCacheDtype: kvCacheDtype ?? null,
mlxKvQuant: effectiveMlxKvQuant,
mlxInt8Prefill: effectiveMlxInt8Prefill,
+ mlxPleOffload: effectiveMlxPleOffload,
+ mlxExpertOffload: effectiveMlxExpertOffload,
speculativeType: speculativeType ?? "auto",
specDraftNMax: specDraftNMax ?? null,
nParallel: nParallel ?? null,
@@ -174,6 +180,8 @@ export function useActiveModelConfig(): ActiveModelConfigState {
kvCacheDtype,
effectiveMlxKvQuant,
effectiveMlxInt8Prefill,
+ effectiveMlxPleOffload,
+ effectiveMlxExpertOffload,
speculativeType,
specDraftNMax,
nParallel,
diff --git a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts
index 96915398db7..dc69770a909 100644
--- a/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts
+++ b/studio/frontend/src/features/model-picker/hooks/use-memory-estimate.ts
@@ -38,6 +38,8 @@ function estimateKey(request: MemoryEstimateRequest | null): string | null {
request.cacheTypeKv ?? null,
request.maxSeqLength ?? null,
request.mlxKvQuant ?? null,
+ request.mlxPleOffload ?? null,
+ request.mlxExpertOffload ?? null,
request.nParallel ?? null,
request.nBatch ?? null,
request.nUbatch ?? null,
diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
index b26bc55db48..ccf4ceaeb2a 100644
--- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
@@ -78,6 +78,8 @@ export function applyPerModelConfigToRuntime(
customContextLength: config.customContextLength ?? null,
mlxKvQuant: config.mlxKvQuant ?? null,
mlxInt8Prefill: config.mlxInt8Prefill ?? false,
+ mlxPleOffload: config.mlxPleOffload ?? null,
+ mlxExpertOffload: config.mlxExpertOffload ?? null,
kvCacheDtype: config.kvCacheDtype ?? null,
speculativeType:
normalizeSpeculativeType(config.speculativeType) ??
@@ -154,6 +156,8 @@ export function currentRuntimePerModelConfig(
kvCacheDtype: s.kvCacheDtype ?? null,
mlxKvQuant: s.mlxKvQuant ?? null,
mlxInt8Prefill: s.mlxInt8Prefill ?? false,
+ mlxPleOffload: s.mlxPleOffload ?? null,
+ mlxExpertOffload: s.mlxExpertOffload ?? null,
speculativeType: normalizeSpeculativeType(s.speculativeType),
specDraftNMax: s.specDraftNMax ?? null,
specDraftCacheDtype: s.specDraftCacheDtype ?? null,
@@ -211,6 +215,8 @@ export function perModelConfigsEqual(
(a.kvCacheDtype ?? null) === (b.kvCacheDtype ?? null) &&
(a.mlxKvQuant ?? null) === (b.mlxKvQuant ?? null) &&
Boolean(a.mlxInt8Prefill) === Boolean(b.mlxInt8Prefill) &&
+ (a.mlxPleOffload ?? null) === (b.mlxPleOffload ?? null) &&
+ (a.mlxExpertOffload ?? null) === (b.mlxExpertOffload ?? null) &&
speculative(a.speculativeType) === speculative(b.speculativeType) &&
(a.specDraftNMax ?? null) === (b.specDraftNMax ?? null) &&
(a.specDraftCacheDtype ?? null) === (b.specDraftCacheDtype ?? null) &&
diff --git a/studio/frontend/src/features/model-picker/model-config/config-signature.ts b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
index b97d354ab80..d5b62963259 100644
--- a/studio/frontend/src/features/model-picker/model-config/config-signature.ts
+++ b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
@@ -60,6 +60,8 @@ export function loadedConfigSignature(
config.kvCacheDtype ?? "",
config.mlxKvQuant ?? "",
config.mlxInt8Prefill ? "1" : "0",
+ config.mlxPleOffload ?? "",
+ config.mlxExpertOffload ?? "",
config.speculativeType ?? "",
config.specDraftNMax ?? "",
config.specDraftCacheDtype ?? "",
diff --git a/studio/frontend/src/features/model-picker/model-config/memory-fit.ts b/studio/frontend/src/features/model-picker/model-config/memory-fit.ts
index 28620b2c674..6f627cb2f1e 100644
--- a/studio/frontend/src/features/model-picker/model-config/memory-fit.ts
+++ b/studio/frontend/src/features/model-picker/model-config/memory-fit.ts
@@ -28,6 +28,7 @@ import type {
SystemGpuDevice,
} from "../../../hooks/gpu-selection.ts";
import { gpuMemoryTotalsGb, sharesHostMemory } from "../../../hooks/gpu-vram.ts";
+import type { MlxOffload } from "./per-model-config.ts";
/** A memory figure in bytes, to two decimals. @deprecated Prefer `formatBytesGiB` from
* `@/lib/memory/format`, whose name says which unit it takes. This alias exists because a
@@ -560,6 +561,45 @@ export function resolveMemoryAdvisory(
/** What `resolveKvNote` joins its items with, and what `glueNoteItems` splits on. */
export const NOTE_SEPARATOR = " · ";
+/** What choosing Default or Manual disk offload stores: Default leaves each part to Auto, Manual
+ * pins the fraction Auto would pick, or off where Auto keeps the part resident. */
+export function mlxOffloadModePatch(
+ manual: boolean,
+ parts: readonly { key: "mlxPleOffload" | "mlxExpertOffload"; decided: number | null }[],
+): Record<string, MlxOffload> {
+ return Object.fromEntries(
+ parts.map(({ key, decided }) => [key, manual ? decided || "off" : null]),
+ );
+}
+
+/** Disk offload slider position for Auto, one step below 0% (off), as GPU Layers has. */
+export const MLX_OFFLOAD_SLIDER_AUTO = -5;
+
+export function mlxOffloadSliderValue(value: MlxOffload | undefined): number {
+ if (value == null) return MLX_OFFLOAD_SLIDER_AUTO;
+ return value === "off" ? 0 : Math.round(value * 100);
+}
+
+export function mlxOffloadFromSlider(position: number): MlxOffload {
+ if (position < 0) return null;
+ return position === 0 ? "off" : position / 100;
+}
+
+/** The fractions Auto picks for the current settings, or null until an estimate of them settles: a stale or
+ * loading estimate answers earlier ones, and reading its parts as resident would pin them off. */
+export function mlxOffloadDecided(state: {
+ estimate: { mlxPleOffload?: number | null; mlxExpertOffload?: number | null } | null;
+ stale: boolean;
+ loading: boolean;
+}): { ple: number | null; experts: number | null } | null {
+ if (state.stale || state.loading || !state.estimate) return null;
+ return { ple: state.estimate.mlxPleOffload ?? null, experts: state.estimate.mlxExpertOffload ?? null };
+}
+
+export function mlxOffloadPercent(fraction: number): string {
+ return `${Math.round(fraction * 100)}%`;
+}
+
/** A separated caption, breakable only between its items. A caption like "f16 - 262,144 tokens
* - 4 slots" does not fit a narrow panel, and the browser breaks at the last space that
* fits, orphaning "slots". Gluing each item with U+00A0 leaves one break opportunity per
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index b5438115cfc..1e37b14c4c9 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -36,6 +36,9 @@ export interface PerModelConfig {
mlxKvQuant?: MlxKvQuant | null;
/** MLX only: run quantized projections with int8 activations, where the model supports it. */
mlxInt8Prefill?: boolean;
+ /** MLX only: share of the PLE tables / routed experts read from disk, "off", or null for Auto. */
+ mlxPleOffload?: MlxOffload;
+ mlxExpertOffload?: MlxOffload;
speculativeType: string | null;
specDraftNMax: number | null;
/** KV cache dtype for the DRAFT context, sized and quantized independently of kvCacheDtype.
@@ -88,6 +91,8 @@ export const DEFAULT_PER_MODEL_CONFIG: PerModelConfig = {
kvCacheDtype: null,
mlxKvQuant: null,
mlxInt8Prefill: false,
+ mlxPleOffload: null,
+ mlxExpertOffload: null,
speculativeType: null,
specDraftNMax: null,
specDraftCacheDtype: null,
@@ -388,6 +393,15 @@ export function normalizeMlxKvQuant(
}
const VALID_KV_CACHE_DTYPES = new Set<string>(KV_CACHE_DTYPES);
+export type MlxOffload = "off" | number | null;
+
+export function normalizeMlxOffload(value: unknown): MlxOffload {
+ if (value === "off") return "off";
+ return typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1
+ ? value
+ : null;
+}
+
// llama-server's --load-mode enum in --help order. "auto" is the default: the UI shows it, storage keeps null and
// the backend emits no flag. Never sent verbatim; builds like b10360 reject "auto" as a value.
export const LOAD_MODES = [
@@ -427,9 +441,10 @@ const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";
// would normalize the unknown field straight back out of the record.
// v2 added nBatch/nUbatch, v3 llamaExtraArgs, v4 disableVision, v5 the llama-server tuning group
// (loadMode / specDraftCacheDtype / ctxCheckpoints / cacheRam), v6 the reasoning budget pair,
-// v7 mlxKvQuant, v9 mlxInt8Prefill, v10 the per-GPU split, v11 the MLX drafter. v8 is skipped: nightly builds stamped it for the reverted custom
+// v7 mlxKvQuant, v9 mlxInt8Prefill, v10 the per-GPU split, v11 the MLX drafter, v12 the MLX offload pair. v8 is skipped: nightly builds stamped it for the reverted custom
// llama.cpp config (#12725), so a v8 client must not claim to understand an int8 prefill record.
-const STORAGE_SCHEMA_VERSION = 11;
+const STORAGE_SCHEMA_VERSION = 12;
+const PRE_MLX_OFFLOAD_SCHEMA_VERSION = 11;
const PRE_MLX_DRAFTER_SCHEMA_VERSION = 10;
const PRE_TENSOR_SPLIT_SCHEMA_VERSION = 9;
const PRE_MLX_INT8_PREFILL_SCHEMA_VERSION = 7;
@@ -475,6 +490,8 @@ const STORED_CONFIG_FIELDS = new Set([
"kvCacheDtype",
"mlxKvQuant",
"mlxInt8Prefill",
+ "mlxPleOffload",
+ "mlxExpertOffload",
"loadIn4bit",
"speculativeType",
"specDraftNMax",
@@ -1127,6 +1144,8 @@ function normalizeV1(partial: RawConfig): PerModelConfig {
maxSeqLength: normalizeMaxSeqLength(partial.maxSeqLength),
mlxKvQuant: normalizeMlxKvQuant(partial.mlxKvQuant, partial.mlxKvBits),
mlxInt8Prefill: partial.mlxInt8Prefill === true,
+ mlxPleOffload: normalizeMlxOffload(partial.mlxPleOffload),
+ mlxExpertOffload: normalizeMlxOffload(partial.mlxExpertOffload),
kvCacheDtype:
typeof partial.kvCacheDtype === "string" &&
VALID_KV_CACHE_DTYPES.has(partial.kvCacheDtype)
@@ -1213,9 +1232,12 @@ function storedSchemaVersion(normalized: PerModelConfig): number {
const mode = normalized.speculativeType;
// A kept Auto and the MLX-only modes are values an older client folds back to unset.
const mlxMode = mode === "auto" || (MLX_ONLY_SPEC_TYPES as readonly string[]).includes(mode ?? "");
- if (normalized.specDraftModel != null || mlxMode) {
+ if (normalized.mlxPleOffload != null || normalized.mlxExpertOffload != null) {
return STORAGE_SCHEMA_VERSION;
}
+ if (normalized.specDraftModel != null || mlxMode) {
+ return PRE_MLX_OFFLOAD_SCHEMA_VERSION;
+ }
if (normalized.tensorSplit != null) {
return PRE_MLX_DRAFTER_SCHEMA_VERSION;
}
@@ -1405,6 +1427,8 @@ export function isDefaultConfig(config: PerModelConfig): boolean {
(config.kvCacheDtype ?? null) === DEFAULT_PER_MODEL_CONFIG.kvCacheDtype &&
(config.mlxKvQuant ?? null) === DEFAULT_PER_MODEL_CONFIG.mlxKvQuant &&
!config.mlxInt8Prefill &&
+ config.mlxPleOffload == null &&
+ config.mlxExpertOffload == null &&
config.speculativeType === DEFAULT_PER_MODEL_CONFIG.speculativeType &&
config.specDraftNMax == null &&
config.nParallel == null &&
diff --git a/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts b/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts
index 45bcf8a1e01..ed594526b8c 100644
--- a/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts
+++ b/studio/frontend/src/features/model-picker/model-config/resident-memory-request.ts
@@ -66,12 +66,17 @@ export function selectResidentEstimateSettings(state: ResidentState) {
}
/** Copy only source identity from the pending request. `mlx`: the resident is an MLX load,
- * which reads its window from maxSeqLength and its cache width from mlxKvQuant, not nCtx. */
+ * which reads its window from maxSeqLength and its cache width from mlxKvQuant, not nCtx, and
+ * is priced at the offload it applied rather than what Auto would pick now. */
export function resolveResidentEstimateRequest(
source: MemoryEstimateRequest | null,
settings: ReturnType<typeof selectResidentEstimateSettings>,
context: number | null,
- mlx?: { kvQuant: string | null },
+ mlx?: {
+ kvQuant: string | null;
+ pleOffload?: number | null;
+ expertOffload?: number | null;
+ },
): MemoryEstimateRequest | null {
if (
!source ||
@@ -90,7 +95,12 @@ export function resolveResidentEstimateRequest(
...(settings ?? {}),
nCtx: Math.floor(context),
...(mlx
- ? { maxSeqLength: Math.floor(context), mlxKvQuant: mlx.kvQuant }
+ ? {
+ maxSeqLength: Math.floor(context),
+ mlxKvQuant: mlx.kvQuant,
+ mlxPleOffload: mlx.pleOffload ?? "off",
+ mlxExpertOffload: mlx.expertOffload ?? "off",
+ }
: {}),
};
}
diff --git a/studio/frontend/src/features/model-picker/sharing/fields.ts b/studio/frontend/src/features/model-picker/sharing/fields.ts
index eb0ab5de32a..09b1e7b8a87 100644
--- a/studio/frontend/src/features/model-picker/sharing/fields.ts
+++ b/studio/frontend/src/features/model-picker/sharing/fields.ts
@@ -47,6 +47,8 @@ export type SharedConfigKey = Exclude<
| "maxSeqLength"
| "mlxKvQuant"
| "mlxInt8Prefill"
+ | "mlxPleOffload"
+ | "mlxExpertOffload"
| "engine"
| "enginePrecision"
| "engineParallelism"
diff --git a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts b/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
index 9108601737e..5d6cbb50580 100644
--- a/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
+++ b/studio/frontend/tests/hydration-clears-stale-local-record.test.ts
@@ -38,6 +38,15 @@ test("int8 prefill reaches the server row only when on", () => {
assert.equal(fromApiOverride({}, on).mlxInt8Prefill, true);
});
+test("MLX offload is always sent, null clearing to Auto, and hydrates from the row else the local value", () => {
+ const set = { ...DEFAULT_PER_MODEL_CONFIG, mlxPleOffload: 0.5, mlxExpertOffload: "off" as const };
+ assert.deepEqual([toApiOverride(set).mlx_ple_offload, toApiOverride(set).mlx_expert_offload], [0.5, "off"]);
+ const auto = toApiOverride(DEFAULT_PER_MODEL_CONFIG);
+ assert.deepEqual([auto.mlx_ple_offload, auto.mlx_expert_offload], [null, null]);
+ assert.equal(fromApiOverride({ mlx_expert_offload: 0.25 }, set).mlxExpertOffload, 0.25);
+ assert.equal(fromApiOverride({}, set).mlxPleOffload, 0.5);
+});
+
test("a server row holding the superseded cache width hydrates as the single choice", () => {
assert.equal(fromApiOverride({ mlx_kv_bits: 8 }).mlxKvQuant, "8");
assert.equal(fromApiOverride({ mlx_kv_quant: "tq-4", mlx_kv_bits: 8 }).mlxKvQuant, "tq-4");
diff --git a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts b/studio/frontend/tests/memory-estimate-narrow-panel.test.ts
index 7535d704d13..8af1111b40c 100644
--- a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts
+++ b/studio/frontend/tests/memory-estimate-narrow-panel.test.ts
@@ -59,6 +59,9 @@ const props: Props = {
layerCount: 27,
gpuLayers: 12,
moeOffloadUnmodelled: false,
+ mlxPleOffload: null,
+ mlxExpertOffload: null,
+ mlxOffloadable: [],
},
loading: false,
stale: false,
@@ -126,6 +129,12 @@ test("context checkpoints get their own host RAM row instead of inflating the KV
assert.doesNotMatch(render({ expanded: true }), /Context checkpoints/);
});
+test("MLX offload is noted on the Weights line", () => {
+ const html = render({ expanded: true, estimate: { ...props.estimate!, mlxExpertOffload: 0.5 } });
+ assert.match(html, /Weights[\s\S]*?experts\u00a050%\u00a0on\u00a0disk/);
+ assert.doesNotMatch(render({ expanded: true }), /on\u00a0disk/);
+});
+
test("a shared pool shows the total without a duplicate GPU figure", () => {
const html = render({ singleMemoryPool: true, isUnifiedMemory: true });
assert.match(html, />Unified<\/span>/);
diff --git a/studio/frontend/tests/memory-estimate-skew.test.ts b/studio/frontend/tests/memory-estimate-skew.test.ts
index ca57999b92e..c27607c009f 100644
--- a/studio/frontend/tests/memory-estimate-skew.test.ts
+++ b/studio/frontend/tests/memory-estimate-skew.test.ts
@@ -274,6 +274,16 @@ test("a field arriving as a string or a non-finite number does not become a figu
assert.equal(classifyMemoryFit(estimate.totalBytes, 24), "unknown");
});
+test("the MLX offload sends the knobs as stored and reads the nested answer", async () => {
+ answer(() => json({ ...FULL, mlx_offload: { ple: null, experts: 0.5, offloadable: ["experts", "x"] } }));
+ const estimate = await fetchMemoryEstimate({ ...REQUEST, mlxPleOffload: "off", mlxExpertOffload: 0.25 });
+ const body = requests.at(-1)?.body as Record<string, unknown>;
+ assert.deepEqual([body.mlx_ple_offload, body.mlx_expert_offload], ["off", 0.25]);
+ assert.deepEqual([estimate.mlxPleOffload, estimate.mlxExpertOffload, estimate.mlxOffloadable], [null, 0.5, ["experts"]]);
+ answer(() => json({ ...FULL, mlx_offload: null }));
+ assert.deepEqual((await fetchMemoryEstimate(REQUEST)).mlxOffloadable, []);
+});
+
test("the checkpoint share defaults to none and never exceeds the cache", async () => {
answer(() => json(FULL));
assert.equal((await fetchMemoryEstimate(REQUEST)).kvCheckpointBytes, 0);
diff --git a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
index 5e15bc55b3a..96dc28c44c5 100644
--- a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
+++ b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
@@ -189,8 +189,8 @@ const ROWS: Row[] = [
"and reads normally.",
},
{
- name: "version 12 (genuinely future)",
- raw: { version: 12, customContextLength: 32768 },
+ name: "version 13 (genuinely future)",
+ raw: { version: 13, customContextLength: 32768 },
normalizedPin: null,
rawPin: 32768,
isDefault: true,
@@ -335,6 +335,10 @@ test("both pin shapes are stamped version 1, so neither is distinguishable by ve
);
assert.equal(
stage({ version: 12, customContextLength: 32768 }).remembered,
+ true,
+ );
+ assert.equal(
+ stage({ version: 13, customContextLength: 32768 }).remembered,
false,
);
});
@@ -358,6 +362,18 @@ test("int8 prefill defaults off, and only an enabled one stamps v9", () => {
assert.equal(stage({ mlxInt8Prefill: true }).config.mlxInt8Prefill, true);
});
+test("MLX offload keeps off or a fraction in (0, 1], and only a set one stamps v12", () => {
+ const offload = (value: unknown) => normalizePerModelConfig({ mlxExpertOffload: value }).mlxExpertOffload;
+ assert.deepEqual([0.5, 1, "off"].map(offload), [0.5, 1, "off"]);
+ assert.deepEqual([0, 1.5, -0.2, Number.NaN, "auto", "0.5", true].map(offload), Array(7).fill(null));
+ assert.equal(isDefaultConfig({ ...DEFAULT_PER_MODEL_CONFIG, mlxPleOffload: "off" }), false);
+ assert.equal(stampedVersion({ specDraftModel: "org/drafter", speculativeType: "dflash" }), 11);
+ assert.equal(stampedVersion({ mlxPleOffload: 0.25 }), 12);
+ assert.equal(stampedVersion({ mlxExpertOffload: "off" }), 12);
+ const config = stage({ version: 12, mlxPleOffload: 0.75, mlxExpertOffload: "off" }).config;
+ assert.deepEqual([config.mlxPleOffload, config.mlxExpertOffload], [0.75, "off"]);
+});
+
/** Old config page, non-GGUF: the number reads maxSeqLength; customContextLength only sets atDefault. */
function oldClientConfigPage(
diff --git a/studio/frontend/tests/mlx-offload-mode.test.ts b/studio/frontend/tests/mlx-offload-mode.test.ts
new file mode 100644
index 00000000000..ba9e387666e
--- /dev/null
+++ b/studio/frontend/tests/mlx-offload-mode.test.ts
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ MLX_OFFLOAD_SLIDER_AUTO,
+ mlxOffloadDecided,
+ mlxOffloadFromSlider,
+ mlxOffloadModePatch,
+ mlxOffloadSliderValue,
+} from "../src/features/model-picker/model-config/memory-fit.ts";
+
+test("Manual pins what Auto would pick, and off where it keeps a part resident", () => {
+ const parts = [
+ { key: "mlxExpertOffload", decided: 0.6 },
+ { key: "mlxPleOffload", decided: null },
+ ] as const;
+ assert.deepEqual(mlxOffloadModePatch(true, parts), { mlxExpertOffload: 0.6, mlxPleOffload: "off" });
+ assert.deepEqual(mlxOffloadModePatch(true, [{ key: "mlxPleOffload", decided: 0 }]), {
+ mlxPleOffload: "off",
+ });
+});
+
+test("Default leaves every shown part to Auto", () => {
+ assert.deepEqual(
+ mlxOffloadModePatch(false, [
+ { key: "mlxExpertOffload", decided: 0.6 },
+ { key: "mlxPleOffload", decided: 1 },
+ ]),
+ { mlxExpertOffload: null, mlxPleOffload: null },
+ );
+});
+
+test("the slider has an Auto step below off, so a part on Auto is never shown as off", () => {
+ assert.equal(mlxOffloadSliderValue(null), MLX_OFFLOAD_SLIDER_AUTO);
+ assert.equal(mlxOffloadSliderValue("off"), 0);
+ assert.equal(mlxOffloadSliderValue(0.25), 25);
+ assert.deepEqual([MLX_OFFLOAD_SLIDER_AUTO, 0, 60].map(mlxOffloadFromSlider), [null, "off", 0.6]);
+});
+
+test("Auto's pick is read only from an estimate of the current settings", () => {
+ const estimate = { mlxPleOffload: 1, mlxExpertOffload: 0.5 };
+ assert.deepEqual(mlxOffloadDecided({ estimate, stale: false, loading: false }), { ple: 1, experts: 0.5 });
+ // Unknown, not resident: Manual would otherwise pin every part off while the estimate refreshes.
+ assert.equal(mlxOffloadDecided({ estimate, stale: true, loading: false }), null);
+ assert.equal(mlxOffloadDecided({ estimate, stale: false, loading: true }), null);
+ assert.equal(mlxOffloadDecided({ estimate: null, stale: false, loading: false }), null);
+});
diff --git a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
index 26955e9f1dc..6fcdc318fdf 100644
--- a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
+++ b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
@@ -91,8 +91,13 @@ type FieldCase = {
different: unknown;
/** The backend compares offload knobs only under Manual, MoE count only with a layer pin. */
live?: { config: Record<string, unknown>; status: Record<string, unknown> };
+ wrap?: (value: unknown) => unknown;
};
+const offloadRequest = (part: "ple" | "experts") => (requested: unknown) => ({
+ [part]: { applied: null, requested, reason: null },
+});
+
const MANUAL_MODE = {
config: { gpuMemoryMode: "manual" },
status: { gpu_memory_mode: "manual" },
@@ -249,8 +254,29 @@ const FIELDS: FieldCase[] = [
same: true,
different: false,
},
+ { key: "mlxPleOffload", statusKey: "mlx_offload", same: 0.5, different: "off", wrap: offloadRequest("ple") },
+ {
+ key: "mlxExpertOffload",
+ statusKey: "mlx_offload",
+ same: "off",
+ different: 0.25,
+ wrap: offloadRequest("experts"),
+ },
];
+test("an Auto offload adopts a load that asked for auto, whatever fraction it applied", () => {
+ const status = {
+ ...ACCELERATORS["apple-mlx"],
+ mlx_offload: {
+ ple: { applied: null, requested: null, reason: null },
+ experts: { applied: 0.5, requested: "auto" as const, reason: null },
+ disk_bytes: null,
+ },
+ };
+ assert.equal(residentRuntimeMatchesConfig(status, BLANK), true);
+ assert.equal(residentRuntimeMatchesConfig(status, { ...BLANK, mlxExpertOffload: 0.5 }), false);
+});
+
for (const [accelerator, base] of Object.entries(ACCELERATORS)) {
test(`[${accelerator}] a config pinning nothing adopts the resident load`, () => {
assert.equal(residentRuntimeMatchesConfig(base, BLANK), true);
@@ -261,7 +287,7 @@ for (const [accelerator, base] of Object.entries(ACCELERATORS)) {
test(`[${accelerator}] ${field.key} the resident load already runs is adopted`, () => {
assert.equal(
residentRuntimeMatchesConfig(
- { ...base, ...field.live?.status, [field.statusKey]: field.same },
+ { ...base, ...field.live?.status, [field.statusKey]: (field.wrap ?? ((v) => v))(field.same) },
{ ...BLANK, ...field.live?.config, [field.key]: field.same },
),
true,
@@ -274,7 +300,7 @@ for (const [accelerator, base] of Object.entries(ACCELERATORS)) {
{
...base,
...field.live?.status,
- [field.statusKey]: field.different,
+ [field.statusKey]: (field.wrap ?? ((v) => v))(field.different),
},
{ ...BLANK, ...field.live?.config, [field.key]: field.same },
),
diff --git a/studio/frontend/tests/resident-memory-request.test.ts b/studio/frontend/tests/resident-memory-request.test.ts
index 643205a70b0..51dce6aa312 100644
--- a/studio/frontend/tests/resident-memory-request.test.ts
+++ b/studio/frontend/tests/resident-memory-request.test.ts
@@ -147,6 +147,15 @@ test("an MLX resident is priced at its served window and loaded cache width", ()
});
assert.equal(request?.maxSeqLength, 8192);
assert.equal(request?.mlxKvQuant, "4");
+ assert.equal(request?.mlxPleOffload, "off");
+ assert.equal(request?.mlxExpertOffload, "off");
+ const offloaded = resolveResidentEstimateRequest(pending, settings, 8192, {
+ kvQuant: null,
+ pleOffload: 1,
+ expertOffload: 0.5,
+ });
+ assert.equal(offloaded?.mlxPleOffload, 1);
+ assert.equal(offloaded?.mlxExpertOffload, 0.5);
const auto = resolveResidentEstimateRequest(pending, settings, 8192, {
kvQuant: null,
});
diff --git a/studio/frontend/tests/same-model-reload-load-settings.test.ts b/studio/frontend/tests/same-model-reload-load-settings.test.ts
index 39dbacf5bab..246a8b229a5 100644
--- a/studio/frontend/tests/same-model-reload-load-settings.test.ts
+++ b/studio/frontend/tests/same-model-reload-load-settings.test.ts
@@ -105,6 +105,11 @@ test("a same-model MLX reload elsewhere moves the KV width, int8 prefill and ver
mlx_kv_quant_reason: "Quantized KV cache to 8 bits",
mlx_kv_quant_note: "from token 256",
mlx_int8_prefill_requested: true,
+ mlx_offload: {
+ ple: { applied: 0.5, requested: "auto", reason: null },
+ experts: { applied: null, requested: 0.25, reason: "no_quantized_experts" },
+ disk_bytes: 1024,
+ },
}),
MLX,
);
@@ -115,6 +120,10 @@ test("a same-model MLX reload elsewhere moves the KV width, int8 prefill and ver
assert.equal(s.mlxKvQuantNote, "from token 256");
assert.equal(s.mlxInt8Prefill, true);
assert.equal(s.loadedMlxInt8PrefillRequested, true);
+ assert.deepEqual(
+ [s.mlxPleOffload, s.loadedMlxPleOffload, s.mlxExpertOffload, s.loadedMlxExpertOffload, s.mlxExpertOffloadReason],
+ [null, 0.5, 0.25, null, "no_quantized_experts"],
+ );
});
test("a same-model MLX reload elsewhere moves a width that was already seeded", () => {
diff --git a/studio/frontend/tests/server-tuning-settings.test.ts b/studio/frontend/tests/server-tuning-settings.test.ts
index 5363ee9abc6..5c8ab0a7d25 100644
--- a/studio/frontend/tests/server-tuning-settings.test.ts
+++ b/studio/frontend/tests/server-tuning-settings.test.ts
@@ -102,8 +102,9 @@ test("the four take part in the editor's identity", () => {
test("a record only claims the new schema version when it carries one", () => {
// toStoredConfig stamps the oldest version that understands every field present.
const source = readSrc("features/model-picker/model-config/per-model-config.ts");
- assert.match(source, /const STORAGE_SCHEMA_VERSION = 11;\nconst PRE_MLX_DRAFTER_SCHEMA_VERSION = 10;/);
- assert.match(source, /specDraftModel != null \|\| mlxMode\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);
+ assert.match(source, /const STORAGE_SCHEMA_VERSION = 12;\nconst PRE_MLX_OFFLOAD_SCHEMA_VERSION = 11;\nconst PRE_MLX_DRAFTER_SCHEMA_VERSION = 10;/);
+ assert.match(source, /mlxExpertOffload != null\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);
+ assert.match(source, /specDraftModel != null \|\| mlxMode\)\s*\{\s*return PRE_MLX_OFFLOAD_SCHEMA_VERSION;/);
assert.match(source, /tensorSplit != null\)\s*\{\s*return PRE_MLX_DRAFTER_SCHEMA_VERSION;/);
assert.match(source, /const PRE_TENSOR_SPLIT_SCHEMA_VERSION = 9;/);
assert.match(source, /const PRE_MLX_INT8_PREFILL_SCHEMA_VERSION = 7;/);
diff --git a/tests/studio/test_cli_attach_load_knobs.py b/tests/studio/test_cli_attach_load_knobs.py
index b90532416a1..dad22f4e5d3 100644
--- a/tests/studio/test_cli_attach_load_knobs.py
+++ b/tests/studio/test_cli_attach_load_knobs.py
@@ -945,6 +945,31 @@ def test_a_turboquant_resident_reloads_as_turboquant(monkeypatch):
assert server.loads[0]["mlx_kv_quant"] == "tq-4"
+def test_a_resident_keeps_its_mlx_offload_on_reload(monkeypatch):
+ server = FakeServer(
+ [dict(RESIDENT)],
+ {
+ "is_gguf": False,
+ "active_model": RESIDENT["id"],
+ "model_identifier": RESIDENT["id"],
+ "requested_context_length": 4096,
+ "load_in_4bit": True,
+ "mlx_offload": {
+ "ple": {"requested": "off", "applied": None},
+ # applied, which a reload must not send back as the request
+ "experts": {"requested": 0.75, "applied": None},
+ },
+ },
+ ).install(monkeypatch)
+
+ start_cli._resolve_model(BASE, KEY, None, start_cli.LoadOptions(max_seq_length = 32768))
+
+ assert (server.loads[0]["mlx_ple_offload"], server.loads[0]["mlx_expert_offload"]) == (
+ "off",
+ 0.75,
+ )
+
+
def test_a_proven_no_op_skips_the_preload_gate(monkeypatch):
"""A proven no-op skips the gate."""
server = FakeServer([dict(RESIDENT)], _gguf_status()).install(monkeypatch)
diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py
index 666ca7a6d6c..b2c74e70679 100644
--- a/unsloth_cli/commands/start.py
+++ b/unsloth_cli/commands/start.py
@@ -2167,6 +2167,11 @@ def _resident_runtime_payload(status: dict, payload: dict) -> dict:
if value is None:
continue
carried[load_field] = value
+ offload = status.get("mlx_offload") or {}
+ for part, load_field in (("ple", "mlx_ple_offload"), ("experts", "mlx_expert_offload")):
+ requested = (offload.get(part) or {}).get("requested")
+ if load_field not in payload and requested is not None:
+ carried[load_field] = requested
return carried