Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
perf(studio): prefill MLX loads in the planned step while experts are on disk (#14466)
Long Yixingd0bdc7a04da4
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 9d4af942017..c27dda451cd 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -77,6 +77,18 @@ VLM_PREFILL_STEP = 2048VLM_PROMPT_CACHE_ENTRIES = 6+def _prefill_layer_step(model, step):+ if not step:+ return None+ try:+ from unsloth_zoo.mlx.offload import wide_prefill_step+ wide = wide_prefill_step(model)+ except Exception:+ return None+ own = vlm_prefill_step()+ return own if wide == own and step % own == 0 else None++def vlm_prefill_step():"""mlx-vlm's own step, so an unreused request prefills as mlx-vlm does. It divides everyboundary, so a step no grid can be built on falls back here rather than reaching a request."""@@ -243,6 +255,7 @@ class _ForwardRecord:__slots__ = ("forwards","capture_at",+ "capture_rows","snapshot","tail_at","tail",@@ -253,6 +266,7 @@ class _ForwardRecord:def __init__(self):self.forwards = 0self.capture_at = 0+ self.capture_rows = 0self.snapshot = Noneself.tail_at = 0self.tail = None@@ -277,6 +291,21 @@ def _prompt_wide_position_ids(args, kwargs):return bool(chunk_shape) and len(chunk_shape) > 1 and shape[-1] != chunk_shape[1]+def _forward_capturing(base, model, args, kwargs, cache, rows):+ from unsloth_zoo.mlx.offload import capture_prefill_rows++ def copy(entry):+ try:+ return copy_cache_entries([entry])[0]+ except Exception as exc:+ logger.info("MLX VLM prompt cache: cache layout not copied (%s)", exc)++ with capture_prefill_rows(rows, copy) as captured:+ output = base.__call__(model, *args, **kwargs)+ snapshot = [captured.get(id(entry)) for entry in cache]+ return output, snapshot if all(entry is not None for entry in snapshot) else None++def _recording_class(base):"""``base`` with a forward that counts itself and copies the cache on cue."""cls = _RECORDING_CLASSES.get(base)@@ -301,10 +330,11 @@ def _recording_class(base):)# As the next forward begins, not as the boundary one returns: generate_step converts the cache between the# two (KV quantization).+ boundary = record is not None and record.forwards == record.capture_atif (record is not Noneand record.forwards- and record.forwards in (record.capture_at, record.tail_at)+ and (boundary and not record.capture_rows or record.forwards == record.tail_at)and cache is not None):try:@@ -312,11 +342,16 @@ def _recording_class(base):except Exception as exc:logger.info("MLX VLM prompt cache: cache layout not copied (%s)", exc)else:- if record.forwards == record.capture_at:+ if boundary:record.snapshot = copiedelse:record.tail = copied- output = base.__call__(self, *args, **kwargs)+ if boundary and record.capture_rows and cache is not None:+ output, record.snapshot = _forward_capturing(+ base, self, args, kwargs, cache, record.capture_rows+ )+ else:+ output = base.__call__(self, *args, **kwargs)if record is not None:record.forwards += 1return output@@ -637,8 +672,10 @@ class VLMPromptCacheSession:media_block = None,policy_hosts = (),step = VLM_PREFILL_STEP,+ window = None,):self.step = step+ self.window = window or stepself._store = storeself._key = keyself._media_token_ids = tuple(media_token_ids)@@ -708,7 +745,10 @@ class VLMPromptCacheSession:self.cache = entriesrecord.on_resume = self._detach_servedself._keep((self._key, tuple(token_ids[:prefix_len])) if prefix_len else None)- record.capture_at = (boundary - prefix_len) // self.step+ rows = boundary - prefix_len+ record.capture_at, record.capture_rows = (+ divmod(rows, self.window) if rows >= 0 else (rows, 0)+ )if prefix_len <= boundary < len(token_ids) - 1:record.tail_at = record.capture_at + 1self.reused_tokens = prefix_len@@ -1038,9 +1078,10 @@ def mlx_resolve_offload() -> tuple:"""The ``(ple, expert)`` fractions a load of *model_dir* reads from disk (None for off), fromknobs as ``normalize_mlx_offload`` gives them: an ``"auto"`` knob decided by unsloth_zoo's 50/50- rule on this load's pricing, and off where that cannot be planned; any other knob as given.- ``vision=True`` prices the mlx-vlm route a speculative text load takes."""- fixed = tuple(None if knob == "auto" else knob for knob in knobs)+ rule on this load's pricing, and off where that cannot be planned; any other knob as given;+ then the planned prefill step for experts on disk, else None. ``vision=True`` prices the+ mlx-vlm route a speculative text load takes."""+ fixed = tuple(None if knob == "auto" else knob for knob in knobs) + (None,)if "auto" not in knobs or not model_dir:return fixedfrom core.inference.mlx_memory import mlx_offload_plan, mlx_offload_planner@@ -1087,6 +1128,17 @@ def mlx_checkpoint_window(model_dir) -> Optional[int]:return None if native is None else min(int(native), MAX_REQUESTABLE_CONTEXT)+def _takes_prefill_step(target) -> bool:+ fields = getattr(target, "__dataclass_fields__", None)+ if fields is not None:+ return "prefill_step_size" in fields+ try:+ params = inspect.signature(target).parameters+ except (TypeError, ValueError):+ return False+ return "prefill_step_size" in params or any(p.kind is p.VAR_KEYWORD for p in params.values())++def _mlx_offload_request(knobs, plan):"""The offload a load asks for: the ``(ple, experts)`` knobs as sent, the fraction each wants,the ``FastMLXModel.from_pretrained`` kwargs, and why none is passed when unsloth_zoo cannot@@ -1098,7 +1150,13 @@ def _mlx_offload_request(knobs, plan):wanted = {part: normalize_mlx_offload(value) for part, value in requested.items()}if plan is None:wanted = {part: None if value == "auto" else value for part, value in wanted.items()}- offload = {"requested": requested, "wanted": wanted, "reason": None, "kwargs": {}}+ offload = {+ "requested": requested,+ "wanted": wanted,+ "reason": None,+ "kwargs": {},+ "prefill_step": None,+ }if not any(wanted.values()):return offloadif mlx_offload_planner() is None:@@ -1106,7 +1164,11 @@ def _mlx_offload_request(knobs, plan):elif "auto" in wanted.values():def decide(model_dir):- wanted.update(zip(wanted, plan(model_dir, tuple(wanted.values()))))+ *fractions, step = plan(model_dir, tuple(wanted.values()))+ # A repeat call sees fractions, not "auto", and plans no step: keep the first one.+ if not offload["decided"]:+ offload["prefill_step"] = step+ wanted.update(zip(wanted, fractions))offload["decided"] = Truereturn dict(wanted)@@ -2833,6 +2895,7 @@ def mlx_drafter_fit(load_in_4bit: bool,costless = False,offload = None,+ prefill_chunk = None,):"""``(attaches, fitted context or None)`` for *source*'s drafter on an mlx-vlm load of *model_dir*:only a priced fit reserves it, a pinned context must fit whole, and a ``costless`` one must fit@@ -2853,6 +2916,7 @@ def mlx_drafter_fit(load_in_4bit = load_in_4bit,vision = True,offload = offload,+ prefill_chunk = prefill_chunk,)outcome, fitted = fit(drafter = (source.path, source.builtin))if outcome == "unsizable":@@ -3966,6 +4030,7 @@ class _TextBatchSession:],completion_batch_size = width,prefill_batch_size = width,+ **backend._prefill_step_kwargs(BatchGenerator),)except BaseException:self._held.close()@@ -4278,6 +4343,7 @@ class _VisionBatchSession:completion_batch_size = width,# Rows quantize their own caches where the single path's decode does.**backend._kv_runtime_quant_kwargs(),+ **backend._prefill_step_kwargs(GenerationDefaults, grid = True),),**({} if self._speculative is None else {"speculative": self._speculative}),)@@ -4452,6 +4518,8 @@ class MLXInferenceBackend:self._reads_vision = Noneself._turboquant_refusal = ""self._speculative_draft = None+ self._prefill_step = None+ self._prefill_layer_step = Noneself._multi_image_marker = Noneself._config = {}self._distributed_group = None@@ -4596,6 +4664,7 @@ class MLXInferenceBackend:media_block = block,policy_hosts = (self._model, language_model) if block is not None else (),step = vlm_prefill_step(),+ window = self._session_window(),)except Exception as exc:# A layout that cannot be built once cannot be built later.@@ -4777,6 +4846,31 @@ class MLXInferenceBackend:return entriesreturn _quantize_kv_entries(entries, bits)+ def _session_window(self):+ # TurboQuant converts the cache between forwards and a KV window rotates it per forward,+ # so neither can take in-window snapshots.+ step = getattr(self, "_prefill_step", None)+ if (+ step+ and getattr(self, "_prefill_layer_step", None)+ and getattr(self, "_kv_cache_window", None) is None+ and not self._kv_runtime_quant_kwargs()+ ):+ return step+ return vlm_prefill_step()++ def _prefill_step_kwargs(+ self,+ target = None,+ grid = False,+ ):+ step = getattr(self, "_prefill_step", None)+ if step is not None and grid:+ step = vlm_prefill_step()+ if step is None or not (target is None or _takes_prefill_step(target)):+ return {}+ return {"prefill_step_size": step}+def _kv_runtime_quant_kwargs(self):"""TurboQuant's width for mlx-vlm's converter; start 0, since its default leaves 5000 tokens unquantized."""bits = self._kv_quant_bits()@@ -5271,6 +5365,10 @@ class MLXInferenceBackend:_offload = _mlx_offload_status(offload, self._model)_offloaded = (_offload["ple"]["applied"], _offload["experts"]["applied"])_offloaded = {"offload": _offloaded} if any(_offloaded) else {}+ self._prefill_step = offload["prefill_step"] if _offload["experts"]["applied"] else None+ self._prefill_layer_step = _prefill_layer_step(self._model, self._prefill_step)+ if self._prefill_step:+ _offloaded["prefill_chunk"] = self._prefill_step_priceable = not (is_distributed or is_lora or dtype is not None)spec_kind = _drafter_ctx = None_drafter_fitted = False@@ -5582,6 +5680,8 @@ class MLXInferenceBackend:if model_name in self.models:del self.models[model_name]self._speculative_draft = None+ self._prefill_step = None+ self._prefill_layer_step = Noneself._model = Noneself._tokenizer = Noneself._processor = None@@ -6167,6 +6267,7 @@ class MLXInferenceBackend:sampler = sampler,)gen_kwargs.update(self._kv_window_generate_kwargs())+ gen_kwargs.update(self._prefill_step_kwargs())if prompt_cache is not None:gen_kwargs["prompt_cache"] = prompt_cacheelse:@@ -6949,9 +7050,10 @@ class MLXInferenceBackend:if session.media_block is not None:vlm_kwargs.update(session.media_block.generate_kwargs())# Reused and unreused turns must prefill on the same grid to match.- vlm_kwargs["prefill_step_size"] = session.step+ vlm_kwargs["prefill_step_size"] = session.windowelse:vlm_kwargs.update(self._kv_quant_generate_kwargs())+ vlm_kwargs.update(self._prefill_step_kwargs())session_scope = session if session is not None else nullcontext()processors = vlm_kwargs.get("logits_processors") or ()# The draft is told these processors, and takes only ones that read just the history (a grammar's@@ -7374,6 +7476,7 @@ class MLXInferenceBackend:max_tokens = max(plan.max_tokens for plan in plans),prefill_batch_size = len(plans),completion_batch_size = len(plans),+ **self._prefill_step_kwargs(GenerationDefaults, grid = True),)with (@@ -7579,6 +7682,7 @@ class MLXInferenceBackend:**self._kv_runtime_quant_kwargs(),**self._kv_quant_generate_kwargs(),**self._kv_window_generate_kwargs(),+ **self._prefill_step_kwargs(),))) as responses:diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.pyindex 2909688cb2d..ba21d690782 100644--- a/studio/backend/core/inference/mlx_memory.py+++ b/studio/backend/core/inference/mlx_memory.py@@ -1480,7 +1480,7 @@ def mlx_offload_plan(load_in_4bit: bool = False,vision: Optional[bool] = None,) -> Optional[tuple]:- """The ``(ple, experts)`` fractions unsloth_zoo's 50/50 rule reads from disk for this load, priced+ """The ``(ple, experts, prefill_step)`` unsloth_zoo's 50/50 rule takes for this load, pricedas Studio prices it; a knob not ``"auto"`` and a pinned ``context`` are kept. When nothing fits,everything offloadable goes to disk. None when the load cannot be planned."""planner = mlx_offload_planner()@@ -1489,9 +1489,12 @@ def mlx_offload_plan(return Nonewith _on_route(vision):found = _offloadable(model_dir, _snapshot_config(model_dir))+ sized = {None: sizing, sizing.chunk: sizing}- def context_cost(tokens, _step):- priced = _priced_at(sizing, tokens)+ def context_cost(tokens, step):+ if step not in sized:+ sized[step] = _size_load(model_dir, kv_bits, step, load_in_4bit, vision)+ priced = sized[step] and _priced_at(sized[step], tokens)if priced is None:raise ValueError(f"cannot price {tokens} tokens")return priced.total_bytes - sizing.weights@@ -1504,15 +1507,15 @@ def mlx_offload_plan(ceiling = max_ctx,minimum = MLX_FIT_MIN_CONTEXT,context = context,- prefill_step = sizing.chunk,+ prefill_step = None if experts and found["expert_bytes"] else sizing.chunk,offload = {"ple": ple, "experts": experts},)if plan["context"] is None:return tuple((1.0 if found[key] is None else None) if value == "auto" else valuefor value, key in ((ple, "ple"), (experts, "experts"))- )- return plan["ple"], plan["experts"]+ ) + (None,)+ return plan["ple"], plan["experts"], plan.get("prefill_step") if plan["experts"] else Nonedef mlx_fit_context(model_dir: str, **kwargs) -> Optional[int]:diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 65d5c291f78..b3b33047671 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -16049,7 +16049,9 @@ def _mlx_estimate_offload(kv_bits = kv_bits,vision = vision,)- return ({"offload": offload} if any(offload) else {}), parts+ *offload, step = offload+ route = {"offload": tuple(offload)} if any(offload) else {}+ return ({**route, "prefill_chunk": step} if step else route), partsdef _mlx_offload_warning(info: dict) -> Optional[str]:diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex c5db7aab990..6b315503047 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -887,6 +887,21 @@ class TestEstimateMemoryRoute:)costless = [fit(None, *c) for c in (("fits", 4096), ("fits", 8192), ("unsizable", 4096))]assert costless == [(True, 4096), (False, 4096), (None, 4096)]+ asked = {}+ monkeypatch.setattr(+ mlx_memory, "mlx_fit_outcome", lambda d, **kw: asked.update(kw) or ("fits", 4096)+ )+ source = DrafterSource("dflash", "/drafter", False)+ mlx_inference.mlx_drafter_fit(+ "/m",+ 32768,+ None,+ source,+ load_in_4bit = False,+ offload = (None, 0.6),+ prefill_chunk = 8192,+ )+ assert (asked["offload"], asked["prefill_chunk"]) == ((None, 0.6), 8192)def test_mlx_model_not_on_disk_is_not_downloaded(self, monkeypatch):self._mlx_target(monkeypatch, None)@@ -1118,21 +1133,30 @@ class TestEstimateMemoryRoute:monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: object())monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda retains_history: 1 << 40)monkeypatch.setattr(- mlx_memory, "mlx_offload_plan", lambda d, **kw: planned.append(kw) or (kw["ple"], 0.6)+ mlx_memory,+ "mlx_offload_plan",+ lambda d, **kw: planned.append(kw)+ or (kw["ple"], 0.6 if kw["experts"] == "auto" else kw["experts"], 8192),)monkeypatch.setattr(mlx_memory,"mlx_memory_breakdown",- lambda model_dir, *, n_ctx, **kw: seen.append(kw.get("offload"))+ lambda model_dir, *, n_ctx, **kw: seen.append(+ (kw.get("offload"), kw.get("prefill_chunk"))+ )or mlx_memory.MlxMemoryBreakdown(1, 1, 1, 3, 3, n_ctx),)estimate = lambda **knobs: _estimate(model_path = "org/model", max_seq_length = 4096, **knobs)- price = lambda **knobs: estimate(**knobs) and seen[-1]+ price = lambda **knobs: estimate(**knobs) and seen[-1][0](tmp_path / "config.json").write_text(json.dumps({"quantization": {"bits": 4}}))assert price(mlx_ple_offload = "off", mlx_expert_offload = 0.5) == (None, 0.5)+ assert (+ seen[-1][1] is None and not planned+ ) # a hand-set fraction prefills in the load's own chunkoff = estimate(mlx_ple_offload = "off", mlx_expert_offload = "off")- assert off.mlx_offload.offloadable == ["experts"]+ assert off.mlx_offload.offloadable == ["experts"] and seen[-1] == (None, None)assert price(mlx_ple_offload = "off") == (None, 0.6) and planned[-1]["context"] == 4096+ assert seen[-1][1] == 8192assert planned[-1]["vision"] is Nonewith monkeypatch.context() as patch:patch.setattr(ri, "_mlx_estimate_speculates", lambda request, model_dir: True)@@ -3649,11 +3673,20 @@ def test_offload_is_priced_by_what_unsloth_zoos_offload_takes(tmp_path, monkeypaplan = lambda **kw: mm.mlx_offload_plan(str(tmp_path), max_ctx = 512, **kw)asked, decide = {}, planner.plan_offloadmonkeypatch.setattr(planner, "plan_offload", lambda **kw: asked.update(kw) or decide(**kw))- assert plan(budget_bytes = 1 << 40) == (None, None)+ assert plan(budget_bytes = 1 << 40) == (None, None, None)priced = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 512).total_bytesassert asked["resident"] + experts + asked["context_cost"](512, None) == priced- assert plan(budget_bytes = 1) == (None, 1.0)- assert plan(budget_bytes = 1 << 40, experts = 0.5) == (None, 0.5)+ wide = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 512, prefill_chunk = 8192).total_bytes+ assert wide > priced and asked["resident"] + experts + asked["context_cost"](512, 8192) == wide+ assert plan(budget_bytes = 1) == (None, 1.0, None)+ assert (+ plan(budget_bytes = 1 << 40, experts = 0.5) == (None, 0.5, 2048)+ and asked["prefill_step"] is None+ )+ assert (+ plan(budget_bytes = 1 << 40, experts = None) == (None, None, None)+ and asked["prefill_step"] == 2048+ )mm.mlx_offload_planner.cache_clear()with monkeypatch.context() as patch: # an unsloth_zoo that would not hand "auto" to Studio@@ -3688,11 +3721,12 @@ def test_offload_is_priced_by_what_unsloth_zoos_offload_takes(tmp_path, monkeypadef test_auto_offload_is_planned_on_the_route_the_load_takes(monkeypatch):routes = []- decide = lambda inventory, **kw: {- "context": 512,+ decide = lambda inventory, context_cost, **kw: {+ "context": context_cost(512, 8192) and 512,"ple": None if inventory["ple"] else 1.0,"experts": 0.5,}+ monkeypatch.setattr(mm, "_priced_at", lambda sizing, tokens: SimpleNamespace(total_bytes = 11))monkeypatch.setattr(mm, "mlx_offload_planner", lambda: SimpleNamespace(plan_offload = decide))monkeypatch.setattr(mm, "_snapshot_config", lambda d: {"model_type": "qwen3_moe", "quantization": {"bits": 4}}@@ -3718,10 +3752,14 @@ def test_auto_offload_is_planned_on_the_route_the_load_takes(monkeypatch):mm._clear_sizing_cache()for vision in (None, True): # a speculative text load goes through mlx-vlmplanned = mm.mlx_offload_plan("/x", budget_bytes = 1, max_ctx = 512, vision = vision)- assert planned == ((1.0, 0.5) if vision else (None, 0.5))+ assert planned == ((1.0, 0.5, None) if vision else (None, 0.5, None))parts = mm.mlx_offloadable_parts("/x", vision)assert parts == (("ple", "experts") if vision else ("experts",))- assert routes == [("sized", None), ("inventory", False), ("sized", True), ("inventory", True)]+ assert routes == [("sized", None), ("inventory", False), ("sized", None)] + [+ ("sized", True),+ ("inventory", True),+ ("sized", True),+ ]def test_the_offload_probe_maps_a_text_model_on_the_route_unsloth_zoo_builds_it(monkeypatch):diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex fe8d2f14665..cfa5cbd654c 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -550,6 +550,75 @@ def _run_studio_site(site, monkeypatch):mlx_inference._TextBatchSession(backend, width = 2).close()+def test_experts_on_disk_prefill_every_text_and_batch_path_in_the_planned_step(+ monkeypatch, mlx_inference_patches+):+ from dataclasses import dataclass++ from core.inference import mlx_inference++ engine = _batch_engine(+ monkeypatch+ ) # before the fake MLX, which unsloth_zoo cannot import under+ _install_fake_mlx(monkeypatch)+ backend = _byte_level_mlx_backend(monkeypatch, [b"a", b"b"])+ stream, asked = sys.modules["mlx_lm"].stream_generate, []+ sys.modules["mlx_lm"].stream_generate = lambda m, t, **kw: asked.append(kw) or stream(+ m, t, **kw+ )+ chat = lambda: list(+ backend.generate_chat_response(+ messages = [{"role": "user", "content": "?"}], max_new_tokens = 2+ )+ )+ chat()+ assert "prefill_step_size" not in asked[-1]+ backend._prefill_step = 8192+ chat()+ assert asked[-1]["prefill_step_size"] == 8192++ built = []+ generate = types.ModuleType("mlx_lm.generate")+ generate.BatchGenerator = lambda *_a, **k: built.append(k) or SimpleNamespace(+ close = lambda: None+ )+ backend._tokenizer.eos_token_ids = [2]+ with monkeypatch.context() as patch:+ patch.setitem(sys.modules, "mlx_lm.generate", generate)+ mlx_inference._TextBatchSession(backend, width = 2).close()+ generate.BatchGenerator = (+ lambda model, stop_tokens, completion_batch_size, prefill_batch_size: (+ built.append({}) or SimpleNamespace(close = lambda: None)+ )+ )+ mlx_inference._TextBatchSession(backend, width = 2).close()+ assert built[-2]["prefill_step_size"] == 8192 and built[-1] == {}++ @dataclass+ class Defaults:+ prefill_batch_size: int = 1+ completion_batch_size: int = 1+ prefill_step_size: int = None++ @dataclass+ class Older:+ prefill_batch_size: int = 1+ completion_batch_size: int = 1++ opened = []+ monkeypatch.setattr(+ engine,+ "BatchStream",+ lambda *a, defaults, **k: opened.append(defaults) or SimpleNamespace(close = lambda: None),+ raising = False,+ )+ backend._processor = object()+ for kind, step in ((Defaults, 2048), (Older, None)):+ monkeypatch.setattr(engine, "GenerationDefaults", kind, raising = False)+ mlx_inference._VisionBatchSession(backend, width = 2).close()+ assert getattr(opened[-1], "prefill_step_size", None) == step++@pytest.fixture(params = ["stub", "zoo"])def zoo_registry(request, monkeypatch, mlx_inference_patches):"""Register scopes in zoo's registry only, entered through the stub's composer or zoo's own."""@@ -1038,10 +1107,10 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatchfrom core.inference.mlx_inference import MLXInferenceBackend_install_fake_mlx(monkeypatch)- calls, fits = [], []+ calls, fits, chunks, steps = [], [], [], []_install_fake_fast_mlx(monkeypatch, calls)loader = sys.modules["unsloth_zoo.mlx.loader"].FastMLXModel- stock, refusal = loader.from_pretrained, {"on": False}+ stock, refusal, has_experts = loader.from_pretrained, {"on": False}, {"on": True}def from_pretrained(*args,@@ -1057,7 +1126,7 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatchmodel, tokenizer = stock(*args, **kwargs)if refused:raise ValueError("Unsloth: offload reads the checkpoint's tensors as stored")- if (offload or {}).get("experts"): # this model has experts, no PLE table+ if (offload or {}).get("experts") and has_experts["on"]:model._unsloth_offload = {"experts": {"on_disk_bytes": 7}}return model, tokenizer@@ -1066,7 +1135,9 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatchmonkeypatch.setattr(mlx_inference,"_fitted_context",- lambda *a, **kw: fits.append(kw.get("offload")) or (None, None),+ lambda *a, **kw: fits.append(kw.get("offload"))+ or chunks.append(kw.get("prefill_chunk"))+ or (None, None),)planned, history = [], [] # here auto takes 0.6 of the expertsbudget = lambda retains_history: history.append(retains_history) or 1 << 40@@ -1076,7 +1147,7 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatchmlx_memory,"mlx_offload_plan",lambda model_dir, **kw: planned.append((model_dir, kw))- or (None if kw["ple"] == "auto" else kw["ple"], 0.6),+ or (None if kw["ple"] == "auto" else kw["ple"], 0.6, 8192),)config = SimpleNamespace(identifier = "fake/moe", is_vision = False, is_lora = False)@@ -1086,11 +1157,13 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatchcalls.clear()backend = MLXInferenceBackend()backend.load_model(config, offload = (ple, experts), **{**base, **dict(*more)})+ steps.append(backend._prefill_step)record = backend.models["fake/moe"]["mlx_offload"]return calls[-1][1].get("offload", {}), recordsent, record = load(0.5, 0.75)assert sent == {"ple": 0.5, "experts": 0.75} and fits[-1] == (None, 0.75)+ assert steps[-1] is None and chunks[-1] is Noneassert record == {"ple": {"applied": None,@@ -1105,6 +1178,11 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatchassert not plannedsent, record = load(None, None)assert sent == {"experts": 0.6} and fits[-1] == (None, 0.6) and history == [True]+ assert steps[-1] == chunks[-1] == 8192+ has_experts["on"] = False+ load(None, None)+ assert steps.pop() is None and chunks.pop() is None and planned.pop()+ has_experts["on"] = Truepick = lambda *keys: tuple(planned[-1][1][key] for key in keys)assert planned[-1][0] == "/snapshot" and pick("ple", "experts") == ("auto", "auto")assert pick("context", "max_ctx", "kv_bits", "load_in_4bit", "vision") == (@@ -1143,6 +1221,7 @@ def test_mlx_load_offloads_what_unsloth_zoo_applies_and_says_why_not(monkeypatchassert record["experts"]["reason"].startswith("Unsloth: offload reads")sent, record = load(None, None)assert sent == {} and not (record["experts"]["applied"] or record["experts"]["reason"])+ assert steps[-1] is None# A hand-set part beside "auto" goes off with the planner unsloth_zoo never called, saying why.sent, record = load(1.0, None)assert (@@ -1165,23 +1244,33 @@ def test_auto_offload_is_planned_as_the_load_is_priced(monkeypatch):plans = []monkeypatch.setattr(mm, "mlx_offload_planner", lambda: object())- monkeypatch.setattr(mm, "mlx_offload_plan", lambda d, **kw: plans.append(kw) or (1.0, 0.5))+ monkeypatch.setattr(+ mm, "mlx_offload_plan", lambda d, **kw: plans.append(kw) or (1.0, 0.5, 8192)+ )monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: 1 << 40)monkeypatch.setattr(mi, "mlx_checkpoint_window", lambda d: 32768)monkeypatch.setattr(mi, "mlx_kv_quant_is_refused", lambda d, bits: True)priced = {"pinned": None, "load_in_4bit": False, "retains_history": True, "kv_bits": 2}auto = lambda at, *knobs: mi.mlx_resolve_offload(at, knobs, **priced)- assert auto("/snap", "auto", "auto") == (1.0, 0.5) and plans[-1]["vision"] is None+ assert auto("/snap", "auto", "auto") == (1.0, 0.5, 8192) and plans[-1]["vision"] is Noneassert (plans[-1]["budget_bytes"], plans[-1]["kv_bits"]) == (1 << 40, None)mi.mlx_resolve_offload("/snap", ("auto", None), vision = True, **priced)assert plans[-1]["vision"] is Trueplans.pop()- assert auto("/snap", 0.5, None) == (0.5, None) and auto(None, "auto", "auto") == (None, None)+ assert (+ auto("/snap", 0.5, None) == (0.5, None, None) and auto(None, "auto", "auto") == (None,) * 3+ )monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: None)- assert auto("/snap", "auto", "auto") == (None, None) and len(plans) == 1+ assert auto("/snap", "auto", "auto") == (None,) * 3 and len(plans) == 1monkeypatch.setattr(mm, "mlx_offload_planner", lambda: None)monkeypatch.setattr(mi, "mlx_memory_budget", lambda retains_history: 1 / 0)- assert auto("/snap", "auto", 0.5) == (None, 0.5) and len(plans) == 1+ assert auto("/snap", "auto", 0.5) == (None, 0.5, None) and len(plans) == 1+ monkeypatch.setattr(mm, "mlx_offload_planner", lambda: object())+ plan = lambda d, knobs: (None, 0.6, 8192) if "auto" in knobs else (*knobs, None)+ request = mi._mlx_offload_request(("auto", "auto"), plan)+ decide = request["kwargs"]["offload"]+ assert decide("/snap") == decide("/snap") == {"ple": None, "experts": 0.6}+ assert request["prefill_step"] == 8192@pytest.mark.parametrize("decided_on_mlx_vlm", [False, True])@@ -1207,7 +1296,7 @@ def test_a_load_falling_back_to_mlx_lm_plans_auto_offload_again(monkeypatch, decdef resolve(model_dir, knobs, **kw):plans.append((knobs, kw["vision"]))# A plan for the mlx-vlm route differs from the one mlx-lm needs, so a reused plan shows.- return (None, 0.25) if kw["vision"] else (None, 1.0)+ return (None, 0.25, 4096) if kw["vision"] else (None, 1.0, 8192)loader = types.ModuleType("unsloth_zoo.mlx.loader")loader.FastMLXModel = _Loader@@ -1985,6 +2074,10 @@ def test_mlx_vlm_generation_selects_renderer_by_capability(monkeypatch):list(backend._generate_vlm(*((text_messages, None) + args[2:]), tools = tools))assert calls["stream"][-1][1]["kv_bits"] == 3.5assert calls["stream"][-1][1]["kv_quant_scheme"] == "turboquant"+ assert "prefill_step_size" not in calls["stream"][-1][1]+ backend._prefill_step = 8192+ list(backend._generate_vlm(*((text_messages, None) + args[2:]), tools = tools))+ assert calls["stream"][-1][1]["prefill_step_size"] == 8192state["generic"] = ValueError("generic rendering failed")state["model"] = f"User: {args[0][0]['content']}"@@ -4105,6 +4198,11 @@ def test_mlx_generate_audio_input_deltas_and_reject(monkeypatch):assert list(backend.generate_audio_input_response(**args)) == ["H", "e", "l"]assert calls["kwargs"]["audio"] == [audio] and calls["kwargs"]["temperature"] == 0.0assert calls["audios"] == 1 and backend.last_generation_stats is not None+ assert "prefill_step_size" not in calls["kwargs"]+ backend._prefill_step = 8192+ list(backend.generate_audio_input_response(**args))+ assert calls["kwargs"]["prefill_step_size"] == 8192+ backend._prefill_step = Noneassert calls["messages"][-1]["content"][1:] == [{"type": "text", "text": "what is said?"}]args["messages"] = [{"role": "user", "content": " "}]@@ -7609,13 +7707,31 @@ def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypatdef stream_batch(model, processor, batch, defaults):added.extend(request.prompt for request in batch)+ asked.append(defaults)raise LookupError+ asked = []monkeypatch.setattr(engine, "stream_batch", stream_batch, raising = False)monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)backend._reads_vision, backend._generation_lock = False, threading.Lock()with pytest.raises(LookupError):next(backend._generate_vlm_batch([{}]))+ from dataclasses import make_dataclass++ fields = (+ "max_tokens",+ "prefill_batch_size",+ "completion_batch_size",+ ("prefill_step_size", int, None),+ )+ monkeypatch.setattr(+ engine, "GenerationDefaults", make_dataclass("Defaults", fields), raising = False+ )+ backend._prefill_step = 8192+ with pytest.raises(LookupError):+ next(backend._generate_vlm_batch([{}]))+ assert asked[-1].prefill_step_size == 2048 and added.pop() == "hi"+ backend._prefill_step = Nonefor reads_vision in (False, True):backend._reads_vision = reads_visionsession.admit({}, reads_vision)@@ -9199,6 +9315,24 @@ def test_mlx_vlm_prompt_cache_session_prefills_on_mlx_vlm_default_step(monkeypatsession = backend._vlm_prompt_cache_session("base")assert mlx_inference.vlm_prefill_step() == 1234 and session.step == 1234assert mlx_inference.shape_stable_prefix(1300, step = session.step) == 1234+ backend._prefill_step, backend._kv_runtime_quant_kwargs = 8192, lambda: {}+ grid = lambda: (lambda session: (session.step, session.window))(+ backend._vlm_prompt_cache_session("base")+ )+ assert grid() == (1234, 1234) and backend._vlm_batch_row_cache("base", ())._step == 1234+ zoo = types.SimpleNamespace(wide_prefill_step = lambda model: 1234 if model == "wide" else None)+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.offload", zoo)+ probe = [+ mlx_inference._prefill_layer_step(*args)+ for args in (("wide", 4936), ("wide", 8192), ("whole", 4936))+ ]+ assert probe == [1234, None, None]+ backend._prefill_layer_step = 1234+ assert grid() == (1234, 8192) and backend._vlm_batch_row_cache("base", ())._step == 1234+ backend._kv_runtime_quant_kwargs = lambda: {"kv_bits": 4}+ assert grid() == (1234, 1234)+ backend._kv_runtime_quant_kwargs, backend._kv_cache_window = lambda: {}, 4096+ assert grid() == (1234, 1234)_BUDGET_WINDOW = 2048@@ -9482,6 +9616,17 @@ def test_mlx_vlm_a_video_turn_never_resumes_a_prompt_cache_snapshot(monkeypatch)((_, stream_kwargs, _),) = streamsassert "prompt_cache" not in stream_kwargsassert "prompt_cache_state" not in stream_kwargs+ backend._model.language_model = type("LanguageModel", (), {})()+ backend._prefill_step, backend._prefill_layer_step = 8192, 2048+ assert list(+ backend._generate_vlm(+ *(([{"role": "user", "content": "hi"}],) + args[1:]), _adapter_state = False+ )+ ) == ["ok"]+ assert (+ streams[-1][1]["prefill_step_size"] == 8192+ and streams[-1][1]["prompt_cache_state"].step == 2048+ )def test_mlx_vlm_a_video_turn_releases_retained_snapshots(monkeypatch):@@ -10509,6 +10654,7 @@ def test_turboquant_sends_its_width_on_both_vlm_cache_branches(monkeypatch):class _Session(contextlib.AbstractContextManager):cache, media_block, step, produced_tokens, produced_seconds = [object()], None, 8, 0, 0.0+ window = stepdef __exit__(self, *_exc):return Falsediff --git a/studio/backend/tests/test_mlx_vlm_prompt_cache.py b/studio/backend/tests/test_mlx_vlm_prompt_cache.pyindex 6fad8651aeb..5b17401a5da 100644--- a/studio/backend/tests/test_mlx_vlm_prompt_cache.py+++ b/studio/backend/tests/test_mlx_vlm_prompt_cache.py@@ -1,6 +1,7 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+import contextlibimport loggingimport sysimport types@@ -39,6 +40,8 @@ def _plain_logger(monkeypatch):class FakeArray:+ __repr__ = lambda self: repr(self.rows)+def __init__(self, rows):self.rows = list(rows)@@ -446,6 +449,54 @@ def test_session_captures_the_boundary_and_resumes_leaving_a_copy_behind(fake_mxassert session.reused_tokens == 768 and stored and len(store) == 5+class LayeredLanguageModel(FakeLanguageModel):+ capture, skipped = None, ()++ def __call__(+ self,+ inputs,+ cache = None,+ **kwargs,+ ):+ for at in range(0, len(inputs), STEP):+ output = super().__call__(inputs[at : at + STEP], cache, **kwargs)+ if self.capture and min(at + STEP, len(inputs)) == self.capture[0]:+ self.capture[2].update(+ (id(e), self.capture[1](e))+ for i, e in enumerate(cache)+ if i not in self.skipped+ )+ return output+++def test_a_window_of_grid_steps_snapshots_the_grid_inside_it(fake_mx, monkeypatch):+ model, zoo, ids = (+ LayeredLanguageModel(),+ types.ModuleType("unsloth_zoo.mlx.offload"),+ list(range(1900)),+ )++ def capture(rows, copy):+ model.capture = (rows, copy, {})+ yield model.capture[2]+ model.capture = None++ zoo.capture_prefill_rows = contextlib.contextmanager(capture)+ monkeypatch.setitem(sys.modules, zoo.__name__, zoo)+ stores = [VLMPromptSnapshotStore(max_bytes = 10**9) for _ in range(2)]+ for tokens in (ids, ids + [1] * 300):+ _generate(stores[0], FakeLanguageModel(), tokens)+ _generate(stores[1], model, tokens, window = 4 * STEP, kwargs = {"step": 4 * STEP})+ held = [+ {key: [entry.state for entry in entries] for key, (entries, _) in store._entries.items()}+ for store in stores+ ]+ assert [len(key[1]) for key in held[1]] == [1792, 2048, 2199] and repr(held[0]) == repr(held[1])+ model.skipped, store = (1,), VLMPromptSnapshotStore(max_bytes = 10**9)+ _generate(store, model, ids, window = 4 * STEP, kwargs = {"step": 4 * STEP})+ assert [len(key[1]) for key in store._entries] == [1899]++def test_session_replays_a_prompt_from_where_its_prefill_ended(fake_mx):store = VLMPromptSnapshotStore(max_bytes = 10**9)language_model = FakeLanguageModel()