Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

perf(studio): prefill MLX loads in the planned step while experts are on disk (#14466)

Long Yixingd0bdc7a04da4
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 9d4af942017..c27dda451cd 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -77,6 +77,18 @@ VLM_PREFILL_STEP = 2048
VLM_PROMPT_CACHE_ENTRIES = 6
+def _prefill_layer_step(model, step):
+ if not step:
+ return None
+ try:
+ from unsloth_zoo.mlx.offload import wide_prefill_step
+ wide = wide_prefill_step(model)
+ except Exception:
+ return None
+ own = vlm_prefill_step()
+ return own if wide == own and step % own == 0 else None
+
+
def vlm_prefill_step():
"""mlx-vlm's own step, so an unreused request prefills as mlx-vlm does. It divides every
boundary, so a step no grid can be built on falls back here rather than reaching a request."""
@@ -243,6 +255,7 @@ class _ForwardRecord:
__slots__ = (
"forwards",
"capture_at",
+ "capture_rows",
"snapshot",
"tail_at",
"tail",
@@ -253,6 +266,7 @@ class _ForwardRecord:
def __init__(self):
self.forwards = 0
self.capture_at = 0
+ self.capture_rows = 0
self.snapshot = None
self.tail_at = 0
self.tail = None
@@ -277,6 +291,21 @@ def _prompt_wide_position_ids(args, kwargs):
return bool(chunk_shape) and len(chunk_shape) > 1 and shape[-1] != chunk_shape[1]
+def _forward_capturing(base, model, args, kwargs, cache, rows):
+ from unsloth_zoo.mlx.offload import capture_prefill_rows
+
+ def copy(entry):
+ try:
+ return copy_cache_entries([entry])[0]
+ except Exception as exc:
+ logger.info("MLX VLM prompt cache: cache layout not copied (%s)", exc)
+
+ with capture_prefill_rows(rows, copy) as captured:
+ output = base.__call__(model, *args, **kwargs)
+ snapshot = [captured.get(id(entry)) for entry in cache]
+ return output, snapshot if all(entry is not None for entry in snapshot) else None
+
+
def _recording_class(base):
"""``base`` with a forward that counts itself and copies the cache on cue."""
cls = _RECORDING_CLASSES.get(base)
@@ -301,10 +330,11 @@ def _recording_class(base):
)
# As the next forward begins, not as the boundary one returns: generate_step converts the cache between the
# two (KV quantization).
+ boundary = record is not None and record.forwards == record.capture_at
if (
record is not None
and record.forwards
- and record.forwards in (record.capture_at, record.tail_at)
+ and (boundary and not record.capture_rows or record.forwards == record.tail_at)
and cache is not None
):
try:
@@ -312,11 +342,16 @@ def _recording_class(base):
except Exception as exc:
logger.info("MLX VLM prompt cache: cache layout not copied (%s)", exc)
else:
- if record.forwards == record.capture_at:
+ if boundary:
record.snapshot = copied
else:
record.tail = copied
- output = base.__call__(self, *args, **kwargs)
+ if boundary and record.capture_rows and cache is not None:
+ output, record.snapshot = _forward_capturing(
+ base, self, args, kwargs, cache, record.capture_rows
+ )
+ else:
+ output = base.__call__(self, *args, **kwargs)
if record is not None:
record.forwards += 1
return output
@@ -637,8 +672,10 @@ class VLMPromptCacheSession:
media_block = None,
policy_hosts = (),
step = VLM_PREFILL_STEP,
+ window = None,
):
self.step = step
+ self.window = window or step
self._store = store
self._key = key
self._media_token_ids = tuple(media_token_ids)
@@ -708,7 +745,10 @@ class VLMPromptCacheSession:
self.cache = entries
record.on_resume = self._detach_served
self._keep((self._key, tuple(token_ids[:prefix_len])) if prefix_len else None)
- record.capture_at = (boundary - prefix_len) // self.step
+ rows = boundary - prefix_len
+ record.capture_at, record.capture_rows = (
+ divmod(rows, self.window) if rows >= 0 else (rows, 0)
+ )
if prefix_len <= boundary < len(token_ids) - 1:
record.tail_at = record.capture_at + 1
self.reused_tokens = prefix_len
@@ -1038,9 +1078,10 @@ def mlx_resolve_offload(
) -> tuple:
"""The ``(ple, expert)`` fractions a load of *model_dir* reads from disk (None for off), from
knobs as ``normalize_mlx_offload`` gives them: an ``"auto"`` knob decided by unsloth_zoo's 50/50
- rule on this load's pricing, and off where that cannot be planned; any other knob as given.
- ``vision=True`` prices the mlx-vlm route a speculative text load takes."""
- fixed = tuple(None if knob == "auto" else knob for knob in knobs)
+ rule on this load's pricing, and off where that cannot be planned; any other knob as given;
+ then the planned prefill step for experts on disk, else None. ``vision=True`` prices the
+ mlx-vlm route a speculative text load takes."""
+ fixed = tuple(None if knob == "auto" else knob for knob in knobs) + (None,)
if "auto" not in knobs or not model_dir:
return fixed
from core.inference.mlx_memory import mlx_offload_plan, mlx_offload_planner
@@ -1087,6 +1128,17 @@ def mlx_checkpoint_window(model_dir) -> Optional[int]:
return None if native is None else min(int(native), MAX_REQUESTABLE_CONTEXT)
+def _takes_prefill_step(target) -> bool:
+ fields = getattr(target, "__dataclass_fields__", None)
+ if fields is not None:
+ return "prefill_step_size" in fields
+ try:
+ params = inspect.signature(target).parameters
+ except (TypeError, ValueError):
+ return False
+ return "prefill_step_size" in params or any(p.kind is p.VAR_KEYWORD for p in params.values())
+
+
def _mlx_offload_request(knobs, plan):
"""The offload a load asks for: the ``(ple, experts)`` knobs as sent, the fraction each wants,
the ``FastMLXModel.from_pretrained`` kwargs, and why none is passed when unsloth_zoo cannot
@@ -1098,7 +1150,13 @@ def _mlx_offload_request(knobs, plan):
wanted = {part: normalize_mlx_offload(value) for part, value in requested.items()}
if plan is None:
wanted = {part: None if value == "auto" else value for part, value in wanted.items()}
- offload = {"requested": requested, "wanted": wanted, "reason": None, "kwargs": {}}
+ offload = {
+ "requested": requested,
+ "wanted": wanted,
+ "reason": None,
+ "kwargs": {},
+ "prefill_step": None,
+ }
if not any(wanted.values()):
return offload
if mlx_offload_planner() is None:
@@ -1106,7 +1164,11 @@ def _mlx_offload_request(knobs, plan):
elif "auto" in wanted.values():
def decide(model_dir):
- wanted.update(zip(wanted, plan(model_dir, tuple(wanted.values()))))
+ *fractions, step = plan(model_dir, tuple(wanted.values()))
+ # A repeat call sees fractions, not "auto", and plans no step: keep the first one.
+ if not offload["decided"]:
+ offload["prefill_step"] = step
+ wanted.update(zip(wanted, fractions))
offload["decided"] = True
return dict(wanted)
@@ -2833,6 +2895,7 @@ def mlx_drafter_fit(
load_in_4bit: bool,
costless = False,
offload = None,
+ prefill_chunk = None,
):
"""``(attaches, fitted context or None)`` for *source*'s drafter on an mlx-vlm load of *model_dir*:
only a priced fit reserves it, a pinned context must fit whole, and a ``costless`` one must fit
@@ -2853,6 +2916,7 @@ def mlx_drafter_fit(
load_in_4bit = load_in_4bit,
vision = True,
offload = offload,
+ prefill_chunk = prefill_chunk,
)
outcome, fitted = fit(drafter = (source.path, source.builtin))
if outcome == "unsizable":
@@ -3966,6 +4030,7 @@ class _TextBatchSession:
],
completion_batch_size = width,
prefill_batch_size = width,
+ **backend._prefill_step_kwargs(BatchGenerator),
)
except BaseException:
self._held.close()
@@ -4278,6 +4343,7 @@ class _VisionBatchSession:
completion_batch_size = width,
# Rows quantize their own caches where the single path's decode does.
**backend._kv_runtime_quant_kwargs(),
+ **backend._prefill_step_kwargs(GenerationDefaults, grid = True),
),
**({} if self._speculative is None else {"speculative": self._speculative}),
)
@@ -4452,6 +4518,8 @@ class MLXInferenceBackend:
self._reads_vision = None
self._turboquant_refusal = ""
self._speculative_draft = None
+ self._prefill_step = None
+ self._prefill_layer_step = None
self._multi_image_marker = None
self._config = {}
self._distributed_group = None
@@ -4596,6 +4664,7 @@ class MLXInferenceBackend:
media_block = block,
policy_hosts = (self._model, language_model) if block is not None else (),
step = vlm_prefill_step(),
+ window = self._session_window(),
)
except Exception as exc:
# A layout that cannot be built once cannot be built later.
@@ -4777,6 +4846,31 @@ class MLXInferenceBackend:
return entries
return _quantize_kv_entries(entries, bits)
+ def _session_window(self):
+ # TurboQuant converts the cache between forwards and a KV window rotates it per forward,
+ # so neither can take in-window snapshots.
+ step = getattr(self, "_prefill_step", None)
+ if (
+ step
+ and getattr(self, "_prefill_layer_step", None)
+ and getattr(self, "_kv_cache_window", None) is None
+ and not self._kv_runtime_quant_kwargs()
+ ):
+ return step
+ return vlm_prefill_step()
+
+ def _prefill_step_kwargs(
+ self,
+ target = None,
+ grid = False,
+ ):
+ step = getattr(self, "_prefill_step", None)
+ if step is not None and grid:
+ step = vlm_prefill_step()
+ if step is None or not (target is None or _takes_prefill_step(target)):
+ return {}
+ return {"prefill_step_size": step}
+
def _kv_runtime_quant_kwargs(self):
"""TurboQuant's width for mlx-vlm's converter; start 0, since its default leaves 5000 tokens unquantized."""
bits = self._kv_quant_bits()
@@ -5271,6 +5365,10 @@ class MLXInferenceBackend:
_offload = _mlx_offload_status(offload, self._model)
_offloaded = (_offload["ple"]["applied"], _offload["experts"]["applied"])
_offloaded = {"offload": _offloaded} if any(_offloaded) else {}
+ self._prefill_step = offload["prefill_step"] if _offload["experts"]["applied"] else None
+ self._prefill_layer_step = _prefill_layer_step(self._model, self._prefill_step)
+ if self._prefill_step:
+ _offloaded["prefill_chunk"] = self._prefill_step
_priceable = not (is_distributed or is_lora or dtype is not None)
spec_kind = _drafter_ctx = None
_drafter_fitted = False
@@ -5582,6 +5680,8 @@ class MLXInferenceBackend:
if model_name in self.models:
del self.models[model_name]
self._speculative_draft = None
+ self._prefill_step = None
+ self._prefill_layer_step = None
self._model = None
self._tokenizer = None
self._processor = None
@@ -6167,6 +6267,7 @@ class MLXInferenceBackend:
sampler = sampler,
)
gen_kwargs.update(self._kv_window_generate_kwargs())
+ gen_kwargs.update(self._prefill_step_kwargs())
if prompt_cache is not None:
gen_kwargs["prompt_cache"] = prompt_cache
else:
@@ -6949,9 +7050,10 @@ class MLXInferenceBackend:
if session.media_block is not None:
vlm_kwargs.update(session.media_block.generate_kwargs())
# Reused and unreused turns must prefill on the same grid to match.
- vlm_kwargs["prefill_step_size"] = session.step
+ vlm_kwargs["prefill_step_size"] = session.window
else:
vlm_kwargs.update(self._kv_quant_generate_kwargs())
+ vlm_kwargs.update(self._prefill_step_kwargs())
session_scope = session if session is not None else nullcontext()
processors = vlm_kwargs.get("logits_processors") or ()
# The draft is told these processors, and takes only ones that read just the history (a grammar's
@@ -7374,6 +7476,7 @@ class MLXInferenceBackend:
max_tokens = max(plan.max_tokens for plan in plans),
prefill_batch_size = len(plans),
completion_batch_size = len(plans),
+ **self._prefill_step_kwargs(GenerationDefaults, grid = True),
)
with (
@@ -7579,6 +7682,7 @@ class MLXInferenceBackend:
**self._kv_runtime_quant_kwargs(),
**self._kv_quant_generate_kwargs(),
**self._kv_window_generate_kwargs(),
+ **self._prefill_step_kwargs(),
)
)
) as responses:
diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.py
index 2909688cb2d..ba21d690782 100644
--- a/studio/backend/core/inference/mlx_memory.py
+++ b/studio/backend/core/inference/mlx_memory.py
@@ -1480,7 +1480,7 @@ def mlx_offload_plan(
load_in_4bit: bool = False,
vision: Optional[bool] = None,
) -> Optional[tuple]:
- """The ``(ple, experts)`` fractions unsloth_zoo's 50/50 rule reads from disk for this load, priced
+ """The ``(ple, experts, prefill_step)`` unsloth_zoo's 50/50 rule takes for this load, priced
as Studio prices it; a knob not ``"auto"`` and a pinned ``context`` are kept. When nothing fits,
everything offloadable goes to disk. None when the load cannot be planned."""
planner = mlx_offload_planner()
@@ -1489,9 +1489,12 @@ def mlx_offload_plan(
return None
with _on_route(vision):
found = _offloadable(model_dir, _snapshot_config(model_dir))
+ sized = {None: sizing, sizing.chunk: sizing}
- def context_cost(tokens, _step):
- priced = _priced_at(sizing, tokens)
+ def context_cost(tokens, step):
+ if step not in sized:
+ sized[step] = _size_load(model_dir, kv_bits, step, load_in_4bit, vision)
+ priced = sized[step] and _priced_at(sized[step], tokens)
if priced is None:
raise ValueError(f"cannot price {tokens} tokens")
return priced.total_bytes - sizing.weights
@@ -1504,15 +1507,15 @@ def mlx_offload_plan(
ceiling = max_ctx,
minimum = MLX_FIT_MIN_CONTEXT,
context = context,
- prefill_step = sizing.chunk,
+ prefill_step = None if experts and found["expert_bytes"] else sizing.chunk,
offload = {"ple": ple, "experts": experts},
)
if plan["context"] is None:
return tuple(
(1.0 if found[key] is None else None) if value == "auto" else value
for value, key in ((ple, "ple"), (experts, "experts"))
- )
- return plan["ple"], plan["experts"]
+ ) + (None,)
+ return plan["ple"], plan["experts"], plan.get("prefill_step") if plan["experts"] else None
def mlx_fit_context(model_dir: str, **kwargs) -> Optional[int]:
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 65d5c291f78..b3b33047671 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -16049,7 +16049,9 @@ def _mlx_estimate_offload(
kv_bits = kv_bits,
vision = vision,
)
- return ({"offload": offload} if any(offload) else {}), parts
+ *offload, step = offload
+ route = {"offload": tuple(offload)} if any(offload) else {}
+ return ({**route, "prefill_chunk": step} if step else route), parts
def _mlx_offload_warning(info: dict) -> Optional[str]:
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index c5db7aab990..6b315503047 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -887,6 +887,21 @@ class TestEstimateMemoryRoute:
)
costless = [fit(None, *c) for c in (("fits", 4096), ("fits", 8192), ("unsizable", 4096))]
assert costless == [(True, 4096), (False, 4096), (None, 4096)]
+ asked = {}
+ monkeypatch.setattr(
+ mlx_memory, "mlx_fit_outcome", lambda d, **kw: asked.update(kw) or ("fits", 4096)
+ )
+ source = DrafterSource("dflash", "/drafter", False)
+ mlx_inference.mlx_drafter_fit(
+ "/m",
+ 32768,
+ None,
+ source,
+ load_in_4bit = False,
+ offload = (None, 0.6),
+ prefill_chunk = 8192,
+ )
+ assert (asked["offload"], asked["prefill_chunk"]) == ((None, 0.6), 8192)
def test_mlx_model_not_on_disk_is_not_downloaded(self, monkeypatch):
self._mlx_target(monkeypatch, None)
@@ -1118,21 +1133,30 @@ class TestEstimateMemoryRoute:
monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: object())
monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda retains_history: 1 << 40)
monkeypatch.setattr(
- mlx_memory, "mlx_offload_plan", lambda d, **kw: planned.append(kw) or (kw["ple"], 0.6)
+ mlx_memory,
+ "mlx_offload_plan",
+ lambda d, **kw: planned.append(kw)
+ or (kw["ple"], 0.6 if kw["experts"] == "auto" else kw["experts"], 8192),
)
monkeypatch.setattr(
mlx_memory,
"mlx_memory_breakdown",
- lambda model_dir, *, n_ctx, **kw: seen.append(kw.get("offload"))
+ lambda model_dir, *, n_ctx, **kw: seen.append(
+ (kw.get("offload"), kw.get("prefill_chunk"))
+ )
or mlx_memory.MlxMemoryBreakdown(1, 1, 1, 3, 3, n_ctx),
)
estimate = lambda **knobs: _estimate(model_path = "org/model", max_seq_length = 4096, **knobs)
- price = lambda **knobs: estimate(**knobs) and seen[-1]
+ price = lambda **knobs: estimate(**knobs) and seen[-1][0]
(tmp_path / "config.json").write_text(json.dumps({"quantization": {"bits": 4}}))
assert price(mlx_ple_offload = "off", mlx_expert_offload = 0.5) == (None, 0.5)
+ assert (
+ seen[-1][1] is None and not planned
+ ) # a hand-set fraction prefills in the load's own chunk
off = estimate(mlx_ple_offload = "off", mlx_expert_offload = "off")
- assert off.mlx_offload.offloadable == ["experts"]
+ assert off.mlx_offload.offloadable == ["experts"] and seen[-1] == (None, None)
assert price(mlx_ple_offload = "off") == (None, 0.6) and planned[-1]["context"] == 4096
+ assert seen[-1][1] == 8192
assert planned[-1]["vision"] is None
with monkeypatch.context() as patch:
patch.setattr(ri, "_mlx_estimate_speculates", lambda request, model_dir: True)
@@ -3649,11 +3673,20 @@ def test_offload_is_priced_by_what_unsloth_zoos_offload_takes(tmp_path, monkeypa
plan = lambda **kw: mm.mlx_offload_plan(str(tmp_path), max_ctx = 512, **kw)
asked, decide = {}, planner.plan_offload
monkeypatch.setattr(planner, "plan_offload", lambda **kw: asked.update(kw) or decide(**kw))
- assert plan(budget_bytes = 1 << 40) == (None, None)
+ assert plan(budget_bytes = 1 << 40) == (None, None, None)
priced = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 512).total_bytes
assert asked["resident"] + experts + asked["context_cost"](512, None) == priced
- assert plan(budget_bytes = 1) == (None, 1.0)
- assert plan(budget_bytes = 1 << 40, experts = 0.5) == (None, 0.5)
+ wide = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 512, prefill_chunk = 8192).total_bytes
+ assert wide > priced and asked["resident"] + experts + asked["context_cost"](512, 8192) == wide
+ assert plan(budget_bytes = 1) == (None, 1.0, None)
+ assert (
+ plan(budget_bytes = 1 << 40, experts = 0.5) == (None, 0.5, 2048)
+ and asked["prefill_step"] is None
+ )
+ assert (
+ plan(budget_bytes = 1 << 40, experts = None) == (None, None, None)
+ and asked["prefill_step"] == 2048
+ )
mm.mlx_offload_planner.cache_clear()
with monkeypatch.context() as patch: # an unsloth_zoo that would not hand "auto" to Studio
@@ -3688,11 +3721,12 @@ def test_offload_is_priced_by_what_unsloth_zoos_offload_takes(tmp_path, monkeypa
def test_auto_offload_is_planned_on_the_route_the_load_takes(monkeypatch):
routes = []
- decide = lambda inventory, **kw: {
- "context": 512,
+ decide = lambda inventory, context_cost, **kw: {
+ "context": context_cost(512, 8192) and 512,
"ple": None if inventory["ple"] else 1.0,
"experts": 0.5,
}
+ monkeypatch.setattr(mm, "_priced_at", lambda sizing, tokens: SimpleNamespace(total_bytes = 11))
monkeypatch.setattr(mm, "mlx_offload_planner", lambda: SimpleNamespace(plan_offload = decide))
monkeypatch.setattr(
mm, "_snapshot_config", lambda d: {"model_type": "qwen3_moe", "quantization": {"bits": 4}}
@@ -3718,10 +3752,14 @@ def test_auto_offload_is_planned_on_the_route_the_load_takes(monkeypatch):
mm._clear_sizing_cache()
for vision in (None, True): # a speculative text load goes through mlx-vlm
planned = mm.mlx_offload_plan("/x", budget_bytes = 1, max_ctx = 512, vision = vision)
- assert planned == ((1.0, 0.5) if vision else (None, 0.5))
+ assert planned == ((1.0, 0.5, None) if vision else (None, 0.5, None))
parts = mm.mlx_offloadable_parts("/x", vision)
assert parts == (("ple", "experts") if vision else ("experts",))
- assert routes == [("sized", None), ("inventory", False), ("sized", True), ("inventory", True)]
+ assert routes == [("sized", None), ("inventory", False), ("sized", None)] + [
+ ("sized", True),
+ ("inventory", True),
+ ("sized", True),
+ ]
def test_the_offload_probe_maps_a_text_model_on_the_route_unsloth_zoo_builds_it(monkeypatch):
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index fe8d2f14665..cfa5cbd654c 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -550,6 +550,75 @@ def _run_studio_site(site, monkeypatch):
mlx_inference._TextBatchSession(backend, width = 2).close()
+def test_experts_on_disk_prefill_every_text_and_batch_path_in_the_planned_step(
+ monkeypatch, mlx_inference_patches
+):
+ from dataclasses import dataclass
+
+ from core.inference import mlx_inference
+
+ engine = _batch_engine(
+ monkeypatch
+ ) # before the fake MLX, which unsloth_zoo cannot import under
+ _install_fake_mlx(monkeypatch)
+ backend = _byte_level_mlx_backend(monkeypatch, [b"a", b"b"])
+ stream, asked = sys.modules["mlx_lm"].stream_generate, []
+ sys.modules["mlx_lm"].stream_generate = lambda m, t, **kw: asked.append(kw) or stream(
+ m, t, **kw
+ )
+ chat = lambda: list(
+ backend.generate_chat_response(
+ messages = [{"role": "user", "content": "?"}], max_new_tokens = 2
+ )
+ )
+ chat()
+ assert "prefill_step_size" not in asked[-1]
+ backend._prefill_step = 8192
+ chat()
+ assert asked[-1]["prefill_step_size"] == 8192
+
+ built = []
+ generate = types.ModuleType("mlx_lm.generate")
+ generate.BatchGenerator = lambda *_a, **k: built.append(k) or SimpleNamespace(
+ close = lambda: None
+ )
+ backend._tokenizer.eos_token_ids = [2]
+ with monkeypatch.context() as patch:
+ patch.setitem(sys.modules, "mlx_lm.generate", generate)
+ mlx_inference._TextBatchSession(backend, width = 2).close()
+ generate.BatchGenerator = (
+ lambda model, stop_tokens, completion_batch_size, prefill_batch_size: (
+ built.append({}) or SimpleNamespace(close = lambda: None)
+ )
+ )
+ mlx_inference._TextBatchSession(backend, width = 2).close()
+ assert built[-2]["prefill_step_size"] == 8192 and built[-1] == {}
+
+ @dataclass
+ class Defaults:
+ prefill_batch_size: int = 1
+ completion_batch_size: int = 1
+ prefill_step_size: int = None
+
+ @dataclass
+ class Older:
+ prefill_batch_size: int = 1
+ completion_batch_size: int = 1
+
+ opened = []
+ monkeypatch.setattr(
+ engine,
+ "BatchStream",
+ lambda *a, defaults, **k: opened.append(defaults) or SimpleNamespace(close = lambda: None),
+ raising = False,
+ )
+ backend._processor = object()
+ for kind, step in ((Defaults, 2048), (Older, None)):
+ monkeypatch.setattr(engine, "GenerationDefaults", kind, raising = False)
+ mlx_inference._VisionBatchSession(backend, width = 2).close()
+ assert getattr(opened[-1], "prefill_step_size", None) == step
+
+
@pytest.fixture(params = ["stub", "zoo"])
def zoo_registry(request, monkeypatch, mlx_inference_patches):
"""Register scopes in zoo's registry only, entered through the stub's composer or zoo's own."""
@@ -1038,10 +1107,10 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatch
from core.inference.mlx_inference import MLXInferenceBackend
_install_fake_mlx(monkeypatch)
- calls, fits = [], []
+ calls, fits, chunks, steps = [], [], [], []
_install_fake_fast_mlx(monkeypatch, calls)
loader = sys.modules["unsloth_zoo.mlx.loader"].FastMLXModel
- stock, refusal = loader.from_pretrained, {"on": False}
+ stock, refusal, has_experts = loader.from_pretrained, {"on": False}, {"on": True}
def from_pretrained(
*args,
@@ -1057,7 +1126,7 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatch
model, tokenizer = stock(*args, **kwargs)
if refused:
raise ValueError("Unsloth: offload reads the checkpoint's tensors as stored")
- if (offload or {}).get("experts"): # this model has experts, no PLE table
+ if (offload or {}).get("experts") and has_experts["on"]:
model._unsloth_offload = {"experts": {"on_disk_bytes": 7}}
return model, tokenizer
@@ -1066,7 +1135,9 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatch
monkeypatch.setattr(
mlx_inference,
"_fitted_context",
- lambda *a, **kw: fits.append(kw.get("offload")) or (None, None),
+ lambda *a, **kw: fits.append(kw.get("offload"))
+ or chunks.append(kw.get("prefill_chunk"))
+ or (None, None),
)
planned, history = [], [] # here auto takes 0.6 of the experts
budget = lambda retains_history: history.append(retains_history) or 1 << 40
@@ -1076,7 +1147,7 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatch
mlx_memory,
"mlx_offload_plan",
lambda model_dir, **kw: planned.append((model_dir, kw))
- or (None if kw["ple"] == "auto" else kw["ple"], 0.6),
+ or (None if kw["ple"] == "auto" else kw["ple"], 0.6, 8192),
)
config = SimpleNamespace(identifier = "fake/moe", is_vision = False, is_lora = False)
@@ -1086,11 +1157,13 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatch
calls.clear()
backend = MLXInferenceBackend()
backend.load_model(config, offload = (ple, experts), **{**base, **dict(*more)})
+ steps.append(backend._prefill_step)
record = backend.models["fake/moe"]["mlx_offload"]
return calls[-1][1].get("offload", {}), record
sent, record = load(0.5, 0.75)
assert sent == {"ple": 0.5, "experts": 0.75} and fits[-1] == (None, 0.75)
+ assert steps[-1] is None and chunks[-1] is None
assert record == {
"ple": {
"applied": None,
@@ -1105,6 +1178,11 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatch
assert not planned
sent, record = load(None, None)
assert sent == {"experts": 0.6} and fits[-1] == (None, 0.6) and history == [True]
+ assert steps[-1] == chunks[-1] == 8192
+ has_experts["on"] = False
+ load(None, None)
+ assert steps.pop() is None and chunks.pop() is None and planned.pop()
+ has_experts["on"] = True
pick = lambda *keys: tuple(planned[-1][1][key] for key in keys)
assert planned[-1][0] == "/snapshot" and pick("ple", "experts") == ("auto", "auto")
assert pick("context", "max_ctx", "kv_bits", "load_in_4bit", "vision") == (
@@ -1143,6 +1221,7 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatch
assert record["experts"]["reason"].startswith("Unsloth: offload reads")
sent, record = load(None, None)
assert sent == {} and not (record["experts"]["applied"] or record["experts"]["reason"])
+ assert steps[-1] is None
# A hand-set part beside "auto" goes off with the planner unsloth_zoo never called, saying why.
sent, record = load(1.0, None)
assert (
@@ -1165,23 +1244,33 @@ def test_auto_offload_is_planned_as_the_load_is_priced(monkeypatch):
plans = []
monkeypatch.setattr(mm, "mlx_offload_planner", lambda: object())
- monkeypatch.setattr(mm, "mlx_offload_plan", lambda d, **kw: plans.append(kw) or (1.0, 0.5))
+ monkeypatch.setattr(
+ mm, "mlx_offload_plan", lambda d, **kw: plans.append(kw) or (1.0, 0.5, 8192)
+ )
monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: 1 << 40)
monkeypatch.setattr(mi, "mlx_checkpoint_window", lambda d: 32768)
monkeypatch.setattr(mi, "mlx_kv_quant_is_refused", lambda d, bits: True)
priced = {"pinned": None, "load_in_4bit": False, "retains_history": True, "kv_bits": 2}
auto = lambda at, *knobs: mi.mlx_resolve_offload(at, knobs, **priced)
- assert auto("/snap", "auto", "auto") == (1.0, 0.5) and plans[-1]["vision"] is None
+ assert auto("/snap", "auto", "auto") == (1.0, 0.5, 8192) and plans[-1]["vision"] is None
assert (plans[-1]["budget_bytes"], plans[-1]["kv_bits"]) == (1 << 40, None)
mi.mlx_resolve_offload("/snap", ("auto", None), vision = True, **priced)
assert plans[-1]["vision"] is True
plans.pop()
- assert auto("/snap", 0.5, None) == (0.5, None) and auto(None, "auto", "auto") == (None, None)
+ assert (
+ auto("/snap", 0.5, None) == (0.5, None, None) and auto(None, "auto", "auto") == (None,) * 3
+ )
monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: None)
- assert auto("/snap", "auto", "auto") == (None, None) and len(plans) == 1
+ assert auto("/snap", "auto", "auto") == (None,) * 3 and len(plans) == 1
monkeypatch.setattr(mm, "mlx_offload_planner", lambda: None)
monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: 1 / 0)
- assert auto("/snap", "auto", 0.5) == (None, 0.5) and len(plans) == 1
+ assert auto("/snap", "auto", 0.5) == (None, 0.5, None) and len(plans) == 1
+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: object())
+ plan = lambda d, knobs: (None, 0.6, 8192) if "auto" in knobs else (*knobs, None)
+ request = mi._mlx_offload_request(("auto", "auto"), plan)
+ decide = request["kwargs"]["offload"]
+ assert decide("/snap") == decide("/snap") == {"ple": None, "experts": 0.6}
+ assert request["prefill_step"] == 8192
@pytest.mark.parametrize("decided_on_mlx_vlm", [False, True])
@@ -1207,7 +1296,7 @@ def test_a_load_falling_back_to_mlx_lm_plans_auto_offload_again(monkeypatch, dec
def resolve(model_dir, knobs, **kw):
plans.append((knobs, kw["vision"]))
# A plan for the mlx-vlm route differs from the one mlx-lm needs, so a reused plan shows.
- return (None, 0.25) if kw["vision"] else (None, 1.0)
+ return (None, 0.25, 4096) if kw["vision"] else (None, 1.0, 8192)
loader = types.ModuleType("unsloth_zoo.mlx.loader")
loader.FastMLXModel = _Loader
@@ -1985,6 +2074,10 @@ def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch):
list(backend._generate_vlm(*((text_messages, None) + args[2:]), tools = tools))
assert calls["stream"][-1][1]["kv_bits"] == 3.5
assert calls["stream"][-1][1]["kv_quant_scheme"] == "turboquant"
+ assert "prefill_step_size" not in calls["stream"][-1][1]
+ backend._prefill_step = 8192
+ list(backend._generate_vlm(*((text_messages, None) + args[2:]), tools = tools))
+ assert calls["stream"][-1][1]["prefill_step_size"] == 8192
state["generic"] = ValueError("generic rendering failed")
state["model"] = f"User: {args[0][0]['content']}"
@@ -4105,6 +4198,11 @@ def test_mlx_generate_audio_input_deltas_and_reject(monkeypatch):
assert list(backend.generate_audio_input_response(**args)) == ["H", "e", "l"]
assert calls["kwargs"]["audio"] == [audio] and calls["kwargs"]["temperature"] == 0.0
assert calls["audios"] == 1 and backend.last_generation_stats is not None
+ assert "prefill_step_size" not in calls["kwargs"]
+ backend._prefill_step = 8192
+ list(backend.generate_audio_input_response(**args))
+ assert calls["kwargs"]["prefill_step_size"] == 8192
+ backend._prefill_step = None
assert calls["messages"][-1]["content"][1:] == [{"type": "text", "text": "what is said?"}]
args["messages"] = [{"role": "user", "content": " "}]
@@ -7609,13 +7707,31 @@ def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypat
def stream_batch(model, processor, batch, defaults):
added.extend(request.prompt for request in batch)
+ asked.append(defaults)
raise LookupError
+ asked = []
monkeypatch.setattr(engine, "stream_batch", stream_batch, raising = False)
monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)
backend._reads_vision, backend._generation_lock = False, threading.Lock()
with pytest.raises(LookupError):
next(backend._generate_vlm_batch([{}]))
+ from dataclasses import make_dataclass
+
+ fields = (
+ "max_tokens",
+ "prefill_batch_size",
+ "completion_batch_size",
+ ("prefill_step_size", int, None),
+ )
+ monkeypatch.setattr(
+ engine, "GenerationDefaults", make_dataclass("Defaults", fields), raising = False
+ )
+ backend._prefill_step = 8192
+ with pytest.raises(LookupError):
+ next(backend._generate_vlm_batch([{}]))
+ assert asked[-1].prefill_step_size == 2048 and added.pop() == "hi"
+ backend._prefill_step = None
for reads_vision in (False, True):
backend._reads_vision = reads_vision
session.admit({}, reads_vision)
@@ -9199,6 +9315,24 @@ def test_mlx_vlm_prompt_cache_session_prefills_on_mlx_vlm_default_step(monkeypat
session = backend._vlm_prompt_cache_session("base")
assert mlx_inference.vlm_prefill_step() == 1234 and session.step == 1234
assert mlx_inference.shape_stable_prefix(1300, step = session.step) == 1234
+ backend._prefill_step, backend._kv_runtime_quant_kwargs = 8192, lambda: {}
+ grid = lambda: (lambda session: (session.step, session.window))(
+ backend._vlm_prompt_cache_session("base")
+ )
+ assert grid() == (1234, 1234) and backend._vlm_batch_row_cache("base", ())._step == 1234
+ zoo = types.SimpleNamespace(wide_prefill_step = lambda model: 1234 if model == "wide" else None)
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.offload", zoo)
+ probe = [
+ mlx_inference._prefill_layer_step(*args)
+ for args in (("wide", 4936), ("wide", 8192), ("whole", 4936))
+ ]
+ assert probe == [1234, None, None]
+ backend._prefill_layer_step = 1234
+ assert grid() == (1234, 8192) and backend._vlm_batch_row_cache("base", ())._step == 1234
+ backend._kv_runtime_quant_kwargs = lambda: {"kv_bits": 4}
+ assert grid() == (1234, 1234)
+ backend._kv_runtime_quant_kwargs, backend._kv_cache_window = lambda: {}, 4096
+ assert grid() == (1234, 1234)
_BUDGET_WINDOW = 2048
@@ -9482,6 +9616,17 @@ def test_mlx_vlm_a_video_turn_never_resumes_a_prompt_cache_snapshot(monkeypatch)
((_, stream_kwargs, _),) = streams
assert "prompt_cache" not in stream_kwargs
assert "prompt_cache_state" not in stream_kwargs
+ backend._model.language_model = type("LanguageModel", (), {})()
+ backend._prefill_step, backend._prefill_layer_step = 8192, 2048
+ assert list(
+ backend._generate_vlm(
+ *(([{"role": "user", "content": "hi"}],) + args[1:]), _adapter_state = False
+ )
+ ) == ["ok"]
+ assert (
+ streams[-1][1]["prefill_step_size"] == 8192
+ and streams[-1][1]["prompt_cache_state"].step == 2048
+ )
def test_mlx_vlm_a_video_turn_releases_retained_snapshots(monkeypatch):
@@ -10509,6 +10654,7 @@ def test_turboquant_sends_its_width_on_both_vlm_cache_branches(monkeypatch):
class _Session(contextlib.AbstractContextManager):
cache, media_block, step, produced_tokens, produced_seconds = [object()], None, 8, 0, 0.0
+ window = step
def __exit__(self, *_exc):
return False
diff --git a/studio/backend/tests/test_mlx_vlm_prompt_cache.py b/studio/backend/tests/test_mlx_vlm_prompt_cache.py
index 6fad8651aeb..5b17401a5da 100644
--- a/studio/backend/tests/test_mlx_vlm_prompt_cache.py
+++ b/studio/backend/tests/test_mlx_vlm_prompt_cache.py
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import contextlib
import logging
import sys
import types
@@ -39,6 +40,8 @@ def _plain_logger(monkeypatch):
class FakeArray:
+ __repr__ = lambda self: repr(self.rows)
+
def __init__(self, rows):
self.rows = list(rows)
@@ -446,6 +449,54 @@ def test_session_captures_the_boundary_and_resumes_leaving_a_copy_behind(fake_mx
assert session.reused_tokens == 768 and stored and len(store) == 5
+class LayeredLanguageModel(FakeLanguageModel):
+ capture, skipped = None, ()
+
+ def __call__(
+ self,
+ inputs,
+ cache = None,
+ **kwargs,
+ ):
+ for at in range(0, len(inputs), STEP):
+ output = super().__call__(inputs[at : at + STEP], cache, **kwargs)
+ if self.capture and min(at + STEP, len(inputs)) == self.capture[0]:
+ self.capture[2].update(
+ (id(e), self.capture[1](e))
+ for i, e in enumerate(cache)
+ if i not in self.skipped
+ )
+ return output
+
+
+def test_a_window_of_grid_steps_snapshots_the_grid_inside_it(fake_mx, monkeypatch):
+ model, zoo, ids = (
+ LayeredLanguageModel(),
+ types.ModuleType("unsloth_zoo.mlx.offload"),
+ list(range(1900)),
+ )
+
+ def capture(rows, copy):
+ model.capture = (rows, copy, {})
+ yield model.capture[2]
+ model.capture = None
+
+ zoo.capture_prefill_rows = contextlib.contextmanager(capture)
+ monkeypatch.setitem(sys.modules, zoo.__name__, zoo)
+ stores = [VLMPromptSnapshotStore(max_bytes = 10**9) for _ in range(2)]
+ for tokens in (ids, ids + [1] * 300):
+ _generate(stores[0], FakeLanguageModel(), tokens)
+ _generate(stores[1], model, tokens, window = 4 * STEP, kwargs = {"step": 4 * STEP})
+ held = [
+ {key: [entry.state for entry in entries] for key, (entries, _) in store._entries.items()}
+ for store in stores
+ ]
+ assert [len(key[1]) for key in held[1]] == [1792, 2048, 2199] and repr(held[0]) == repr(held[1])
+ model.skipped, store = (1,), VLMPromptSnapshotStore(max_bytes = 10**9)
+ _generate(store, model, ids, window = 4 * STEP, kwargs = {"step": 4 * STEP})
+ assert [len(key[1]) for key in store._entries] == [1899]
+
+
def test_session_replays_a_prompt_from_where_its_prefill_ended(fake_mx):
store = VLMPromptSnapshotStore(max_bytes = 10**9)
language_model = FakeLanguageModel()