Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
perf(studio): enter Zoo's exact verify, NAX and packed gated-delta scopes on Unsloth Studio's own MLX paths (#13404)
Long Yixing937d54eebd40
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex b63a6746f8e..5abc6e5557f 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -889,7 +889,7 @@ def _mlx_inference_patch(name):@contextmanager-def _mlx_optional_fusion(name, model):+def _mlx_optional_fusion(name, model, *args):"""Hold an optional Zoo fusion scope, or yield the model unfused. Guard ENTRY only:an open scope must unwind through the ExitStack as an unguarded `with` would, or aninterrupted generation stops restoring the module tree. A dead Metal queue is not guarded."""@@ -898,7 +898,7 @@ def _mlx_optional_fusion(name, model):active = modelif patch is not None:try:- entered = scope.enter_context(patch(model))+ entered = scope.enter_context(patch(model, *args))except Exception as error:if is_metal_queue_dead(error):raise@@ -933,6 +933,14 @@ def _mlx_fused_residual_norm_handoff(model):return _mlx_optional_fusion("fused_residual_norm_handoff", model)+def _mlx_fused_packed_gated_delta(model):+ return _mlx_optional_fusion("fused_packed_gated_delta", model)+++def _mlx_exact_speculative_verify(model):+ return _mlx_optional_fusion("exact_speculative_verify", model)++def _int8_prefill_zoo():"""unsloth_zoo's MLX inference module when it can run int8 prefill, else None."""try:@@ -3895,7 +3903,7 @@ class _VisionBatchSession:self._held.enter_context(_temporary_mlx_adapter_state(backend._model, adapter_state))- self._held.enter_context(backend._int8_prefill_scope(zoo_generation = True))+ self._held.enter_context(backend._int8_prefill_scope())self._speculative = getattr(backend, "_speculative_draft", None)self.stream = BatchStream(backend._model,@@ -4375,14 +4383,16 @@ class MLXInferenceBackend:max_new_tokens = max(1, min(int(max_new_tokens), int(budget) - len(prompt_tokens)))return max_new_tokens- def _int8_prefill_scope(self, zoo_generation = False):- """Outermost scope of a generation, which every Zoo scope nested in it inherits. A path- running Zoo's own generation scopes is pinned off when inactive, or they read the env."""+ def _int8_prefill_scope(self):+ """Outermost scope of a generation, which every Zoo scope nested in it inherits. Pinned+ off when inactive, so Zoo's full-precision NAX routes still run and no scope reads the Int8+ Prefill env."""scope = (getattr(self, "_int8_prefill", None) or {}).get("scope")if scope is not None:return scope(self._model, True)- zoo = _int8_prefill_zoo() if zoo_generation else None- return zoo.nax_quantized_linear(self._model, False) if zoo is not None else nullcontext()+ if _int8_prefill_zoo() is None:+ return nullcontext()+ return _mlx_optional_fusion("nax_quantized_linear", self._model, False)def _kv_quant_bits(self):return (getattr(self, "_kv_quant", None) or {}).get("kv_bits")@@ -5584,6 +5594,7 @@ class MLXInferenceBackend:_mlx_fused_moe_router(self._model),_mlx_fused_moe_routed_experts(self._model),_mlx_fused_residual_norm_handoff(self._model),+ _mlx_fused_packed_gated_delta(self._model),):(gen_prompt,@@ -6371,11 +6382,13 @@ class MLXInferenceBackend:generation_scope.enter_context(_mlx_fused_moe_router(self._model))generation_scope.enter_context(_mlx_fused_moe_routed_experts(self._model))generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))+ generation_scope.enter_context(_mlx_fused_packed_gated_delta(self._model))final_response = Noneclip_path = Noneif draft is not None:from unsloth_zoo.mlx.generate import SamplingParams+ generation_scope.enter_context(_mlx_exact_speculative_verify(self._model))# Under the lock: the load's one draft serves one generation at a time.draft.prepare(vlm_kwargs["input_ids"][0].tolist(),@@ -6716,7 +6729,7 @@ class MLXInferenceBackend:with (self._generation_lock,_temporary_mlx_adapter_state(self._model, _adapter_state),- self._int8_prefill_scope(zoo_generation = True),+ self._int8_prefill_scope(),):if any(plan.images for plan in plans):self._release_vlm_snapshots()@@ -6906,6 +6919,7 @@ class MLXInferenceBackend:generation_scope.enter_context(_mlx_fused_moe_router(self._model))generation_scope.enter_context(_mlx_fused_moe_routed_experts(self._model))generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))+ generation_scope.enter_context(_mlx_fused_packed_gated_delta(self._model))final_response = Nonetry:with closing(diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 6f13121726f..1f9407d9e80 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -31,6 +31,7 @@ FUSIONS = ("moe_router","moe_routed_experts","residual_norm_handoff",+ "packed_gated_delta",)@@ -3687,7 +3688,7 @@ def test_int8_prefill_reloads_on_change_and_reports_why_it_is_off(mlx_inference_@pytest.mark.parametrize("requested,available", [(True, True), (True, False), (False, True)])-def test_int8_prefill_is_the_outermost_scope_of_generation_only_when_active(+def test_int8_prefill_is_the_outermost_scope_of_generation_pinned_off_when_inactive(monkeypatch, mlx_inference_patches, requested, available):from core.inference import mlx_inference@@ -3727,8 +3728,19 @@ def test_int8_prefill_is_the_outermost_scope_of_generation_only_when_active(args = ([{"role": "user", "content": [{"type": "image"}]}], [object()], 0.7, 0.9, 40, 0.01, 4)assert list(backend._generate_vlm(*args, 1.0, None)) == ["ok"]- expected = [("int8", True)] + [("fusion", None)] * len(FUSIONS) + [("stream", None)]- assert events == (expected + [("int8_exit", True)] if requested and available else expected[1:])+ active = requested and available+ assert events == [("int8", active)] + [("fusion", None)] * len(FUSIONS) + [+ ("stream", None),+ ("int8_exit", active),+ ]++ def broken(model, int8_prefill = None):+ raise ImportError("skewed zoo")++ mlx_inference_patches.nax_quantized_linear = broken+ if not active: # an inactive load never asked for the scope, so its failure costs no request+ with backend._int8_prefill_scope():+ passdef test_a_batch_session_holds_int8_prefill_for_its_whole_life(monkeypatch, mlx_inference_patches):@@ -5452,6 +5464,7 @@ def _drive_vlm_generation(max_new_tokens = 1,session = None,video = None,+ on_stream = None,**request,):import sys@@ -5466,6 +5479,8 @@ def _drive_vlm_generation(def _stream(*_a, **kwargs):seen.clear()seen.update(kwargs)+ if on_stream is not None:+ on_stream()yield SimpleNamespace(text = "ok", prompt_tokens = 3, generation_tokens = 1)mlx_vlm.stream_generate = _stream@@ -5508,17 +5523,39 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):utils = types.ModuleType("mlx_vlm.utils")utils.prepare_inputs = lambda *_a, **_k: {"input_ids": [_Ids([7, 8, 9])], "extra": 1}monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)- prepared = []+ prepared, verify_scope, scopes = [], [], []+ # A Zoo without the scope leaves the reply drafted and unfused.+ with monkeypatch.context() as patch:+ patch.setattr(mlx_inference, "_mlx_inference_patch", lambda name: verify_scope.append(name))+ with mlx_inference._mlx_exact_speculative_verify(utils) as active:+ assert active is utils and verify_scope.pop() == "exact_speculative_verify"++ @contextmanager+ def exact_verify(model):+ verify_scope.append("open")+ try:+ yield model+ finally:+ verify_scope.append("closed")++ monkeypatch.setattr(mlx_inference, "_mlx_exact_speculative_verify", exact_verify)draft = SimpleNamespace(draft_kind = "mtp", draft_n = 6, draft_n_accepted = 4)- draft.prepare = lambda ids, sampling, given: prepared.append(- (ids, sampling.temperature, sampling.top_k)- ) or setattr(draft, "given", given)+ draft.prepare = (+ lambda ids, sampling, given: prepared.append((ids, sampling.temperature, sampling.top_k))+ or setattr(draft, "given", given)+ or scopes.append(list(verify_scope))+ )backend = mlx_inference.MLXInferenceBackend()backend._speculative_draft = draft# top_k -1 is the API's "disabled", which the drafter's sampling settings spell 0.seen = _drive_vlm_generation(- backend, monkeypatch, temperature = 0.5, top_k = -1, repetition_penalty = 1.1+ backend,+ monkeypatch,+ temperature = 0.5,+ top_k = -1,+ repetition_penalty = 1.1,+ on_stream = lambda: scopes.append(list(verify_scope)),)assert seen["draft_model"] is draft and seen["extra"] == 1 and prepared == [([7, 8, 9], 0.5, 0)]# The draft is told every processor the reply runs: none reaches mlx-vlm as a shortcut.@@ -5536,7 +5573,8 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):seen = _drive_vlm_generation(backend, monkeypatch, **request)assert "max_tokens" in seen and "draft_model" not in seenassert "draft_n" not in backend.last_generation_stats["timings"]- assert len(prepared) == 1+ # Only the drafted reply entered the verify scope: open from preparing the draft through decoding.+ assert len(prepared) == 1 and scopes == [["open"]] * 2 and verify_scope == ["open", "closed"]opened, backend._is_vlm, backend._vlm_batch_unavailable_reason = [], True, lambda r: Nonebackend._vlm_resident_unavailable_reason = lambda r, gaps = iter(["no stream"]): next(gaps, None)assert backend.batch_unavailable_reason([{}, {}]) == "no stream"