Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

perf(studio): enter Zoo's exact verify, NAX and packed gated-delta scopes on Unsloth Studio's own MLX paths (#13404)

Long Yixing937d54eebd40
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index b63a6746f8e..5abc6e5557f 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -889,7 +889,7 @@ def _mlx_inference_patch(name):
@contextmanager
-def _mlx_optional_fusion(name, model):
+def _mlx_optional_fusion(name, model, *args):
"""Hold an optional Zoo fusion scope, or yield the model unfused. Guard ENTRY only:
an open scope must unwind through the ExitStack as an unguarded `with` would, or an
interrupted generation stops restoring the module tree. A dead Metal queue is not guarded."""
@@ -898,7 +898,7 @@ def _mlx_optional_fusion(name, model):
active = model
if patch is not None:
try:
- entered = scope.enter_context(patch(model))
+ entered = scope.enter_context(patch(model, *args))
except Exception as error:
if is_metal_queue_dead(error):
raise
@@ -933,6 +933,14 @@ def _mlx_fused_residual_norm_handoff(model):
return _mlx_optional_fusion("fused_residual_norm_handoff", model)
+def _mlx_fused_packed_gated_delta(model):
+ return _mlx_optional_fusion("fused_packed_gated_delta", model)
+
+
+def _mlx_exact_speculative_verify(model):
+ return _mlx_optional_fusion("exact_speculative_verify", model)
+
+
def _int8_prefill_zoo():
"""unsloth_zoo's MLX inference module when it can run int8 prefill, else None."""
try:
@@ -3895,7 +3903,7 @@ class _VisionBatchSession:
self._held.enter_context(
_temporary_mlx_adapter_state(backend._model, adapter_state)
)
- self._held.enter_context(backend._int8_prefill_scope(zoo_generation = True))
+ self._held.enter_context(backend._int8_prefill_scope())
self._speculative = getattr(backend, "_speculative_draft", None)
self.stream = BatchStream(
backend._model,
@@ -4375,14 +4383,16 @@ class MLXInferenceBackend:
max_new_tokens = max(1, min(int(max_new_tokens), int(budget) - len(prompt_tokens)))
return max_new_tokens
- def _int8_prefill_scope(self, zoo_generation = False):
- """Outermost scope of a generation, which every Zoo scope nested in it inherits. A path
- running Zoo's own generation scopes is pinned off when inactive, or they read the env."""
+ def _int8_prefill_scope(self):
+ """Outermost scope of a generation, which every Zoo scope nested in it inherits. Pinned
+ off when inactive, so Zoo's full-precision NAX routes still run and no scope reads the Int8
+ Prefill env."""
scope = (getattr(self, "_int8_prefill", None) or {}).get("scope")
if scope is not None:
return scope(self._model, True)
- zoo = _int8_prefill_zoo() if zoo_generation else None
- return zoo.nax_quantized_linear(self._model, False) if zoo is not None else nullcontext()
+ if _int8_prefill_zoo() is None:
+ return nullcontext()
+ return _mlx_optional_fusion("nax_quantized_linear", self._model, False)
def _kv_quant_bits(self):
return (getattr(self, "_kv_quant", None) or {}).get("kv_bits")
@@ -5584,6 +5594,7 @@ class MLXInferenceBackend:
_mlx_fused_moe_router(self._model),
_mlx_fused_moe_routed_experts(self._model),
_mlx_fused_residual_norm_handoff(self._model),
+ _mlx_fused_packed_gated_delta(self._model),
):
(
gen_prompt,
@@ -6371,11 +6382,13 @@ class MLXInferenceBackend:
generation_scope.enter_context(_mlx_fused_moe_router(self._model))
generation_scope.enter_context(_mlx_fused_moe_routed_experts(self._model))
generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))
+ generation_scope.enter_context(_mlx_fused_packed_gated_delta(self._model))
final_response = None
clip_path = None
if draft is not None:
from unsloth_zoo.mlx.generate import SamplingParams
+ generation_scope.enter_context(_mlx_exact_speculative_verify(self._model))
# Under the lock: the load's one draft serves one generation at a time.
draft.prepare(
vlm_kwargs["input_ids"][0].tolist(),
@@ -6716,7 +6729,7 @@ class MLXInferenceBackend:
with (
self._generation_lock,
_temporary_mlx_adapter_state(self._model, _adapter_state),
- self._int8_prefill_scope(zoo_generation = True),
+ self._int8_prefill_scope(),
):
if any(plan.images for plan in plans):
self._release_vlm_snapshots()
@@ -6906,6 +6919,7 @@ class MLXInferenceBackend:
generation_scope.enter_context(_mlx_fused_moe_router(self._model))
generation_scope.enter_context(_mlx_fused_moe_routed_experts(self._model))
generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))
+ generation_scope.enter_context(_mlx_fused_packed_gated_delta(self._model))
final_response = None
try:
with closing(
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 6f13121726f..1f9407d9e80 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -31,6 +31,7 @@ FUSIONS = (
"moe_router",
"moe_routed_experts",
"residual_norm_handoff",
+ "packed_gated_delta",
)
@@ -3687,7 +3688,7 @@ def test_int8_prefill_reloads_on_change_and_reports_why_it_is_off(mlx_inference_
@pytest.mark.parametrize("requested,available", [(True, True), (True, False), (False, True)])
-def test_int8_prefill_is_the_outermost_scope_of_generation_only_when_active(
+def test_int8_prefill_is_the_outermost_scope_of_generation_pinned_off_when_inactive(
monkeypatch, mlx_inference_patches, requested, available
):
from core.inference import mlx_inference
@@ -3727,8 +3728,19 @@ def test_int8_prefill_is_the_outermost_scope_of_generation_only_when_active(
args = ([{"role": "user", "content": [{"type": "image"}]}], [object()], 0.7, 0.9, 40, 0.01, 4)
assert list(backend._generate_vlm(*args, 1.0, None)) == ["ok"]
- expected = [("int8", True)] + [("fusion", None)] * len(FUSIONS) + [("stream", None)]
- assert events == (expected + [("int8_exit", True)] if requested and available else expected[1:])
+ active = requested and available
+ assert events == [("int8", active)] + [("fusion", None)] * len(FUSIONS) + [
+ ("stream", None),
+ ("int8_exit", active),
+ ]
+
+ def broken(model, int8_prefill = None):
+ raise ImportError("skewed zoo")
+
+ mlx_inference_patches.nax_quantized_linear = broken
+ if not active: # an inactive load never asked for the scope, so its failure costs no request
+ with backend._int8_prefill_scope():
+ pass
def test_a_batch_session_holds_int8_prefill_for_its_whole_life(monkeypatch, mlx_inference_patches):
@@ -5452,6 +5464,7 @@ def _drive_vlm_generation(
max_new_tokens = 1,
session = None,
video = None,
+ on_stream = None,
**request,
):
import sys
@@ -5466,6 +5479,8 @@ def _drive_vlm_generation(
def _stream(*_a, **kwargs):
seen.clear()
seen.update(kwargs)
+ if on_stream is not None:
+ on_stream()
yield SimpleNamespace(text = "ok", prompt_tokens = 3, generation_tokens = 1)
mlx_vlm.stream_generate = _stream
@@ -5508,17 +5523,39 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):
utils = types.ModuleType("mlx_vlm.utils")
utils.prepare_inputs = lambda *_a, **_k: {"input_ids": [_Ids([7, 8, 9])], "extra": 1}
monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)
- prepared = []
+ prepared, verify_scope, scopes = [], [], []
+ # A Zoo without the scope leaves the reply drafted and unfused.
+ with monkeypatch.context() as patch:
+ patch.setattr(mlx_inference, "_mlx_inference_patch", lambda name: verify_scope.append(name))
+ with mlx_inference._mlx_exact_speculative_verify(utils) as active:
+ assert active is utils and verify_scope.pop() == "exact_speculative_verify"
+
+ @contextmanager
+ def exact_verify(model):
+ verify_scope.append("open")
+ try:
+ yield model
+ finally:
+ verify_scope.append("closed")
+
+ monkeypatch.setattr(mlx_inference, "_mlx_exact_speculative_verify", exact_verify)
draft = SimpleNamespace(draft_kind = "mtp", draft_n = 6, draft_n_accepted = 4)
- draft.prepare = lambda ids, sampling, given: prepared.append(
- (ids, sampling.temperature, sampling.top_k)
- ) or setattr(draft, "given", given)
+ draft.prepare = (
+ lambda ids, sampling, given: prepared.append((ids, sampling.temperature, sampling.top_k))
+ or setattr(draft, "given", given)
+ or scopes.append(list(verify_scope))
+ )
backend = mlx_inference.MLXInferenceBackend()
backend._speculative_draft = draft
# top_k -1 is the API's "disabled", which the drafter's sampling settings spell 0.
seen = _drive_vlm_generation(
- backend, monkeypatch, temperature = 0.5, top_k = -1, repetition_penalty = 1.1
+ backend,
+ monkeypatch,
+ temperature = 0.5,
+ top_k = -1,
+ repetition_penalty = 1.1,
+ on_stream = lambda: scopes.append(list(verify_scope)),
)
assert seen["draft_model"] is draft and seen["extra"] == 1 and prepared == [([7, 8, 9], 0.5, 0)]
# The draft is told every processor the reply runs: none reaches mlx-vlm as a shortcut.
@@ -5536,7 +5573,8 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):
seen = _drive_vlm_generation(backend, monkeypatch, **request)
assert "max_tokens" in seen and "draft_model" not in seen
assert "draft_n" not in backend.last_generation_stats["timings"]
- assert len(prepared) == 1
+ # Only the drafted reply entered the verify scope: open from preparing the draft through decoding.
+ assert len(prepared) == 1 and scopes == [["open"]] * 2 and verify_scope == ["open", "closed"]
opened, backend._is_vlm, backend._vlm_batch_unavailable_reason = [], True, lambda r: None
backend._vlm_resident_unavailable_reason = lambda r, gaps = iter(["no stream"]): next(gaps, None)
assert backend.batch_unavailable_reason([{}, {}]) == "no stream"