Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

perf(studio): enter Zoo's inference scope registry on Unsloth Studio's MLX paths (#13975)

Long Yixinga50be79cede4
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 08e0a04e3e7..7a0bc330809 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -913,36 +913,68 @@ def _mlx_optional_fusion(name, model, *args):
yield active
-def _mlx_fused_moe_gate_up(model):
- return _mlx_optional_fusion("fused_moe_gate_up", model)
-
-
-def _mlx_fused_decode_conv_silu(model):
- return _mlx_optional_fusion("fused_decode_conv_silu", model)
-
-
-def _mlx_fused_residual_norm(model):
- return _mlx_optional_fusion("fused_residual_norm", model)
-
-
-def _mlx_fused_moe_router(model):
- return _mlx_optional_fusion("fused_moe_router", model)
-
-
-def _mlx_fused_moe_routed_experts(model):
- return _mlx_optional_fusion("fused_moe_routed_experts", model)
-
-
-def _mlx_fused_residual_norm_handoff(model):
- return _mlx_optional_fusion("fused_residual_norm_handoff", model)
+def _mlx_scope_refused(name, error):
+ if is_metal_queue_dead(error):
+ raise error
+ _mlx_fusion_unavailable(name, error)
+
+
+# What Studio entered by hand before Zoo's registry; a Zoo without it keeps exactly these.
+_MLX_LEGACY_SCOPES = (
+ "fused_moe_gate_up",
+ "fused_decode_conv_silu",
+ "fused_residual_norm",
+ "fused_moe_router",
+ "fused_moe_routed_experts",
+ "fused_residual_norm_handoff",
+ "fused_packed_gated_delta",
+)
-def _mlx_fused_packed_gated_delta(model):
- return _mlx_optional_fusion("fused_packed_gated_delta", model)
+@contextmanager
+def _mlx_inference_scopes(model, call, int8_prefill):
+ """Hold the Zoo scopes Zoo picks for `call` (its call traits), so a new Zoo scope needs no Studio change.
+ A Zoo without the registry keeps the scopes Studio listed before it (none for the text batch)."""
+ compose = _mlx_inference_patch("inference_scopes")
+ with ExitStack() as scope:
+ if compose is None and "batch" not in call:
+ for name in _MLX_LEGACY_SCOPES:
+ scope.enter_context(_mlx_optional_fusion(name, model))
+ if "speculative" in call:
+ scope.enter_context(_mlx_optional_fusion("exact_speculative_verify", model))
+ elif compose is not None:
+ try:
+ scope.enter_context(
+ compose(
+ model,
+ call,
+ int8_prefill = int8_prefill,
+ on_error = _mlx_scope_refused,
+ )
+ )
+ except Exception as error:
+ if is_metal_queue_dead(error):
+ raise
+ _mlx_fusion_unavailable("inference_scopes", error)
+ yield model
-def _mlx_exact_speculative_verify(model):
- return _mlx_optional_fusion("exact_speculative_verify", model)
+def _mlx_warm_inference(model, int8_prefill):
+ """Zoo's warm-up generation, so the first request skips the scopes' kernel builds and first-use
+ checks. A Zoo without it, or a failed warm-up, leaves them to the first request."""
+ warm = _mlx_inference_patch("warm_inference")
+ if warm is None:
+ return
+ try:
+ warm(model, int8_prefill = int8_prefill)
+ except Exception as error:
+ if is_metal_queue_dead(error):
+ raise
+ logger.warning(
+ "MLX warm-up failed, so the first request runs it: %s: %s",
+ type(error).__name__,
+ error,
+ )
def _int8_prefill_zoo():
@@ -3675,6 +3707,7 @@ class _TextBatchSession:
_temporary_mlx_adapter_state(backend._model, adapter_state)
)
self._held.enter_context(backend._int8_prefill_scope())
+ self._held.enter_context(backend._generation_scopes("batch"))
self.generator = BatchGenerator(
backend._model,
stop_tokens = [
@@ -4452,6 +4485,12 @@ class MLXInferenceBackend:
return nullcontext()
return _mlx_optional_fusion("nax_quantized_linear", self._model, False)
+ def _generation_scopes(self, *call):
+ """Zoo's scopes for a generation of this load, under its Int8 Prefill decision. Nested in
+ _int8_prefill_scope, which stays outermost."""
+ active = (getattr(self, "_int8_prefill", None) or {}).get("active", False)
+ return _mlx_inference_scopes(self._model, call, active)
+
def _kv_quant_bits(self):
return (getattr(self, "_kv_quant", None) or {}).get("kv_bits")
@@ -5119,7 +5158,11 @@ class MLXInferenceBackend:
# The worker owns fixed base weights until unload; adapter requests stay scoped.
if not is_lora:
- self._model_fusion.enter_context(_mlx_fused_moe_gate_up(model))
+ self._model_fusion.enter_context(_mlx_optional_fusion("fused_moe_gate_up", model))
+ # Every rank of a distributed load must run each forward together.
+ if not is_distributed:
+ with mlx_rng_preserved(), self._int8_prefill_scope():
+ _mlx_warm_inference(model, self._int8_prefill["active"])
logger.info("Model %s loaded successfully", model_name)
return True
@@ -5756,13 +5799,7 @@ class MLXInferenceBackend:
self._generation_lock,
_temporary_mlx_adapter_state(self._model, _adapter_state),
self._int8_prefill_scope(),
- _mlx_fused_moe_gate_up(self._model),
- _mlx_fused_decode_conv_silu(self._model),
- _mlx_fused_residual_norm(self._model),
- _mlx_fused_moe_router(self._model),
- _mlx_fused_moe_routed_experts(self._model),
- _mlx_fused_residual_norm_handoff(self._model),
- _mlx_fused_packed_gated_delta(self._model),
+ self._generation_scopes(),
):
(
gen_prompt,
@@ -6586,19 +6623,14 @@ class MLXInferenceBackend:
# The vision pass gets the headroom, under the lock so nothing refills it.
# A video turn resumes no snapshot, so retained ones are pure occupancy.
self._release_vlm_snapshots()
- generation_scope.enter_context(_mlx_fused_moe_gate_up(self._model))
- generation_scope.enter_context(_mlx_fused_decode_conv_silu(self._model))
- generation_scope.enter_context(_mlx_fused_residual_norm(self._model))
- generation_scope.enter_context(_mlx_fused_moe_router(self._model))
- generation_scope.enter_context(_mlx_fused_moe_routed_experts(self._model))
- generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))
- generation_scope.enter_context(_mlx_fused_packed_gated_delta(self._model))
+ generation_scope.enter_context(
+ self._generation_scopes("vision", *(() if draft is None else ("speculative",)))
+ )
final_response = None
clip_path = None
if draft is not None:
from unsloth_zoo.mlx.generate import SamplingParams
- generation_scope.enter_context(_mlx_exact_speculative_verify(self._model))
# Under the lock: the load's one draft serves one generation at a time.
draft.prepare(
vlm_kwargs["input_ids"][0].tolist(),
@@ -7154,13 +7186,7 @@ class MLXInferenceBackend:
):
# As on the image path: the tower gets the headroom, under the lock.
self._release_vlm_snapshots()
- generation_scope.enter_context(_mlx_fused_moe_gate_up(self._model))
- generation_scope.enter_context(_mlx_fused_decode_conv_silu(self._model))
- generation_scope.enter_context(_mlx_fused_residual_norm(self._model))
- generation_scope.enter_context(_mlx_fused_moe_router(self._model))
- generation_scope.enter_context(_mlx_fused_moe_routed_experts(self._model))
- generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))
- generation_scope.enter_context(_mlx_fused_packed_gated_delta(self._model))
+ generation_scope.enter_context(self._generation_scopes("vision", "audio"))
final_response = None
try:
with closing(
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index c305b0be4d9..164a4772495 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -2,10 +2,12 @@
import ast
import asyncio
+import importlib
import builtins
import contextlib
import copy
import json
+import os
import subprocess
import sys
import threading
@@ -33,12 +35,56 @@ FUSIONS = (
"residual_norm_handoff",
"packed_gated_delta",
)
-# Fused scopes zoo's own generate.py enters that Studio does not wrap itself, so they get no
-# _mlx_fused_* helper and stay out of the FUSIONS parametrisations: unsloth_zoo #1757 added
-# fused_verify_attention around speculative verify. The stub still has to answer them.
+# Stub scopes kept out of the FUSIONS parametrisations: unsloth_zoo #1757 added fused_verify_attention
+# around speculative verify. The stub still has to answer them.
ZOO_ONLY_FUSIONS = ("verify_attention",)
+def _zoo_registry():
+ """The stub's INFERENCE_SCOPES in zoo's order: (scope name, call traits it serves[, opt-in switch])."""
+ return [
+ *((f"fused_{name}", ()) for name in FUSIONS[:5]),
+ ("nax_quantized_linear", ()),
+ ("dense_prefill_linear", ()),
+ *((f"fused_{name}", ()) for name in FUSIONS[5:]),
+ ("exact_speculative_verify", ("speculative",)),
+ ("fused_verify_attention", ("speculative",)),
+ ]
+
+
+def _stub_inference_scopes(module):
+ """zoo's inference_scopes over the stub: each registered scope is looked up by name when entered,
+ so a test that replaces a stub scope sees it entered through the registry."""
+
+ @contextmanager
+ def inference_scopes(
+ model = None,
+ call = None,
+ *,
+ int8_prefill = None,
+ on_error = None,
+ _modules = None,
+ ):
+ # The call-shape replay below passes placeholders; anything but a tuple serves every scope.
+ traits = set(call) if isinstance(call, tuple) else None
+ with contextlib.ExitStack() as stack:
+ for name, serves, *opt_in in module.INFERENCE_SCOPES:
+ if traits is not None and serves and traits.isdisjoint(serves):
+ continue
+ if opt_in and opt_in[0] and os.environ.get(opt_in[0]) != "1":
+ continue
+ args = (int8_prefill,) if name == "nax_quantized_linear" else ()
+ try:
+ stack.enter_context(getattr(module, name)(model, *args))
+ except Exception as error:
+ if on_error is None:
+ raise
+ on_error(name, error)
+ yield model
+
+ return inference_scopes
+
+
@pytest.fixture(autouse = True)
def mlx_inference_patches(monkeypatch, native_vlm_generation_context):
from core.inference import mlx_inference
@@ -49,6 +95,8 @@ def mlx_inference_patches(monkeypatch, native_vlm_generation_context):
setattr(module, f"fused_{name}", _neutral_scope)
for name, helper in PLAIN_HELPERS.items():
setattr(module, name, helper)
+ module.INFERENCE_SCOPES = _zoo_registry()
+ module.inference_scopes = _stub_inference_scopes(module)
module.__getattr__ = _neutral_zoo_helper
monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.inference", module)
return module
@@ -87,10 +135,14 @@ def _neutral_zoo_helper(name):
"""Every other public helper zoo's generate.py enters around a model is neutral too.
unsloth_zoo #1541 added nax_quantized_linear to that import list, and a stub holding only the
- fused_* names failed every real-model test on macOS with an ImportError. fused_* stay explicit,
- so a test that deletes one still sees it missing.
+ fused_* names failed every real-model test on macOS with an ImportError. fused_*,
+ inference_scopes and warm_inference stay explicit, so a test that deletes one still sees it missing.
"""
- if name.startswith("_") or name.startswith("fused_"):
+ if (
+ name.startswith("_")
+ or name.startswith("fused_")
+ or name in ("inference_scopes", "warm_inference")
+ ):
raise AttributeError(name)
return _neutral_scope
@@ -125,7 +177,7 @@ def test_mlx_fusion_import_never_fails_the_request(monkeypatch, error, feature):
from core.inference import mlx_inference
module_name = "unsloth_zoo.mlx.inference"
- helper = getattr(mlx_inference, f"_mlx_fused_{feature}")
+ helper = lambda model: mlx_inference._mlx_optional_fusion(f"fused_{feature}", model)
original_import = mlx_inference.importlib.import_module
@@ -157,7 +209,7 @@ def test_mlx_fusion_that_cannot_be_entered_keeps_native(
monkeypatch.setattr(mlx_inference_patches, f"fused_{feature}", refuse)
monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())
model = object()
- with getattr(mlx_inference, f"_mlx_fused_{feature}")(model) as active:
+ with mlx_inference._mlx_optional_fusion(f"fused_{feature}", model) as active:
assert active is model
@@ -185,7 +237,7 @@ def test_mlx_dead_gpu_queue_is_not_reported_as_a_working_fallback(
monkeypatch.setattr(mlx_inference_patches, f"fused_{feature}", fault)
monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())
with pytest.raises(RuntimeError, match = "METAL"):
- with getattr(mlx_inference, f"_mlx_fused_{feature}")(object()):
+ with mlx_inference._mlx_optional_fusion(f"fused_{feature}", object()):
pass # pragma: no cover
assert mlx_inference._MLX_FUSION_UNAVAILABLE == set()
@@ -204,7 +256,7 @@ def test_mlx_packing_that_runs_out_of_memory_still_keeps_native(
monkeypatch.setattr(mlx_inference_patches, f"fused_{feature}", refuse)
monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())
model = object()
- with getattr(mlx_inference, f"_mlx_fused_{feature}")(model) as active:
+ with mlx_inference._mlx_optional_fusion(f"fused_{feature}", model) as active:
assert active is model
@@ -250,6 +302,78 @@ def test_mlx_fusion_failure_at_load_still_loads_the_model(monkeypatch, mlx_moe):
assert backend.unload_model("fake/text")
[email protected]("outcome", ["warmed", "int8", "failed", "dead_queue", "distributed"])
+def test_a_load_warms_zoo_scopes_under_its_int8_decision(monkeypatch, mlx_moe, outcome):
+ from core.inference import mlx_inference
+
+ backend = _install_fake_text_stack(monkeypatch, {"p": [1, 2], "generated": [7, 8]}, [])
+ _install_fake_fast_mlx(monkeypatch, [])
+ sys.modules["mlx.core"].clear_cache = lambda: None
+ warnings = _capture_rng_warnings(monkeypatch, mlx_inference)
+ events = []
+
+ @contextmanager
+ def rng():
+ events.append("rng")
+ try:
+ yield
+ finally:
+ events.append("/rng")
+
+ @contextmanager
+ def gate_up(model, **_):
+ events.append("gate_up")
+ yield model
+
+ @contextmanager
+ def int8(
+ model,
+ int8_prefill = None,
+ **_,
+ ):
+ events.append(("int8", int8_prefill))
+ try:
+ yield model
+ finally:
+ events.append("/int8")
+
+ def warm(model, int8_prefill):
+ events.append(("warm", model is backend._model, int8_prefill))
+ if outcome == "failed":
+ raise RuntimeError("no headroom")
+ if outcome == "dead_queue":
+ raise RuntimeError("[METAL] Command buffer execution failed: Caused GPU Timeout Error.")
+
+ mlx_moe.fused_moe_gate_up, mlx_moe.nax_quantized_linear, mlx_moe.warm_inference = (
+ gate_up,
+ int8,
+ warm,
+ )
+ monkeypatch.setattr(mlx_inference, "mlx_rng_preserved", rng)
+ if outcome == "int8":
+ status = {"requested": True, "active": True, "reason": "", "scope": int8}
+ monkeypatch.setattr(mlx_inference, "_int8_prefill_status", lambda *_: status)
+ config = SimpleNamespace(identifier = "fake/text", is_vision = False, is_lora = False)
+ group = SimpleNamespace(size = lambda: 2, rank = lambda: 0)
+ kwargs = (
+ {"parallel_mode": "tensor", "distributed_group": group} if outcome == "distributed" else {}
+ )
+ with (
+ pytest.raises(RuntimeError, match = "METAL")
+ if outcome == "dead_queue"
+ else contextlib.nullcontext()
+ ):
+ assert backend.load_model(config, **kwargs) is True
+ active = outcome == "int8"
+ warmed = (
+ []
+ if outcome == "distributed"
+ else ["rng", ("int8", active), ("warm", True, active), "/int8", "/rng"]
+ )
+ assert events == ["gate_up", *warmed]
+ assert any("MLX warm-up failed" in warning for warning in warnings) == (outcome == "failed")
+
+
def test_mlx_fusion_that_refuses_everywhere_still_generates(monkeypatch, mlx_moe):
from core.inference import mlx_inference
@@ -378,6 +502,198 @@ def test_mlx_audio_input_generation_survives_every_fusion_refusing(
assert Counter(refused) == Counter(dict.fromkeys(FUSIONS, 1))
+class _Ids(list):
+ def tolist(self):
+ return list(self)
+
+
+def _run_studio_site(site, monkeypatch):
+ """Drive one of Studio's own MLX generation paths once, on fakes."""
+ from core.inference import mlx_inference
+
+ backend = mlx_inference.MLXInferenceBackend()
+ if site == "text":
+ backend = _install_fake_text_stack(monkeypatch, {"p": [1, 2], "generated": [7]}, [])
+ tokenizer = backend._tokenizer
+ _install_fake_fast_mlx(monkeypatch, [])
+ sys.modules["mlx.core"].clear_cache = lambda: None
+ config = SimpleNamespace(identifier = "fake/text", is_vision = False, is_lora = False)
+ assert backend.load_model(config)
+ backend._tokenizer = tokenizer
+ assert list(backend.generate_chat_response(messages = [{"role": "user", "content": "p"}]))
+ elif site.startswith("vision"):
+ if site == "vision-drafted":
+ pytest.importorskip("mlx_lm")
+ utils = types.ModuleType("mlx_vlm.utils")
+ utils.prepare_inputs = lambda *_a, **_k: {"input_ids": [_Ids([7])]}
+ monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)
+ backend._speculative_draft = SimpleNamespace(
+ draft_kind = "mtp", draft_n = 1, draft_n_accepted = 1, prepare = lambda *_a: None
+ )
+ assert "draft_model" in _drive_vlm_generation(backend, monkeypatch) or site == "vision"
+ elif site == "audio":
+ fake_vlm = types.ModuleType("mlx_vlm")
+ fake_vlm.stream_generate = lambda *_a, **_k: iter(
+ [SimpleNamespace(text = "H", prompt_tokens = 3, generation_tokens = 1)]
+ )
+ monkeypatch.setitem(sys.modules, "mlx_vlm", fake_vlm)
+ monkeypatch.setattr(mlx_inference, "_render_registered_vlm_prompt", lambda *_a, **_k: "P")
+ backend._model, backend._processor = _audio_model(), _audio_processor()
+ backend.active_model_name, backend.models = "m", {"m": {"audio_type": "audio_vlm"}}
+ response = backend.generate_audio_input_response(
+ messages = [{"role": "user", "content": "?"}],
+ system_prompt = "",
+ audio_array = [0.0, 0.1],
+ max_new_tokens = 1,
+ )
+ assert list(response) == ["H"]
+ else:
+ generate = types.ModuleType("mlx_lm.generate")
+ generate.BatchGenerator = lambda *_a, **_k: SimpleNamespace(close = lambda: None)
+ backend._model, backend._tokenizer = SimpleNamespace(), SimpleNamespace(eos_token_ids = [2])
+ with (
+ monkeypatch.context() as patch
+ ): # only while it is built: mlx_lm itself imports from it
+ patch.setitem(sys.modules, "mlx_lm.generate", generate)
+ mlx_inference._TextBatchSession(backend, width = 2).close()
+
+
[email protected](params = ["stub", "zoo"])
+def zoo_registry(request, monkeypatch, mlx_inference_patches):
+ """Register scopes in zoo's registry only, entered through the stub's composer or zoo's own."""
+ if request.param == "stub":
+ register = lambda name, serves = (), opt_in = None: (
+ mlx_inference_patches.INFERENCE_SCOPES.append((name, serves, opt_in))
+ )
+ return register
+ monkeypatch.delitem(sys.modules, "unsloth_zoo.mlx.inference")
+ try:
+ zoo = importlib.import_module("unsloth_zoo.mlx.inference")
+ except Exception as error:
+ pytest.skip(f"unsloth_zoo.mlx.inference is not importable here: {error}")
+ finally:
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.inference", mlx_inference_patches)
+ if not hasattr(zoo, "INFERENCE_SCOPES"):
+ pytest.skip("this unsloth_zoo has no inference scope registry")
+ monkeypatch.setattr(zoo, "INFERENCE_SCOPES", [])
+ monkeypatch.setattr(mlx_inference_patches, "inference_scopes", zoo.inference_scopes)
+
+ def register(
+ name,
+ serves = (),
+ opt_in = None,
+ ):
+ scope = getattr(mlx_inference_patches, name)
+ enter = lambda model, *args, _modules: scope(model, *args)
+ enter.__name__ = name
+ zoo.INFERENCE_SCOPES.append(zoo.InferenceScope(enter, frozenset(serves), opt_in))
+
+ return register
+
+
[email protected]("site", ["text", "vision", "vision-drafted", "audio", "text-batch"])
+def test_a_scope_zoo_registers_reaches_each_studio_site_with_no_studio_change(
+ monkeypatch, mlx_inference_patches, zoo_registry, site
+):
+ from core.inference import mlx_inference
+
+ entered = []
+
+ def scope(name):
+ @contextmanager
+ def enter(model, *_args):
+ entered.append(name)
+ yield model
+
+ return enter
+
+ traits = ("speculative", "vision", "audio", "batch")
+ for name, serves, opt_in in (
+ ("every", (), None),
+ *((trait, (trait,), None) for trait in traits),
+ ("switched_off", (), "UNSLOTH_MLX_STUDIO_TEST_SCOPE"),
+ ):
+ setattr(mlx_inference_patches, name, scope(name))
+ zoo_registry(name, serves, opt_in)
+ monkeypatch.delenv("UNSLOTH_MLX_STUDIO_TEST_SCOPE", raising = False)
+ monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())
+ _run_studio_site(site, monkeypatch)
+ assert mlx_inference._MLX_FUSION_UNAVAILABLE == set() # the registry and every scope entered
+ call = {
+ "text": (),
+ "vision": ("vision",),
+ "vision-drafted": ("speculative", "vision"),
+ "audio": ("vision", "audio"),
+ "text-batch": ("batch",),
+ }[site]
+ assert entered == ["every", *call]
+
+
+def test_an_older_zoo_without_the_registry_keeps_the_scopes_studio_listed(
+ monkeypatch, mlx_inference_patches
+):
+ from core.inference import mlx_inference
+
+ pytest.importorskip("mlx_lm") # the drafted vision site imports zoo's generate
+ entered = []
+ for name in (*(f"fused_{name}" for name in FUSIONS), "exact_speculative_verify"):
+ monkeypatch.setattr(
+ mlx_inference_patches,
+ name,
+ lambda model, name = name: entered.append(name) or contextlib.nullcontext(model),
+ )
+ monkeypatch.delattr(mlx_inference_patches, "inference_scopes")
+ monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())
+ backend = _install_fake_text_stack(monkeypatch, {"p": [1, 2], "generated": [7, 8]}, [])
+ _install_fake_fast_mlx(monkeypatch, [])
+ sys.modules["mlx.core"].clear_cache = lambda: None
+ assert backend.load_model(
+ SimpleNamespace(identifier = "fake/text", is_vision = False, is_lora = False)
+ )
+ legacy = [f"fused_{name}" for name in FUSIONS]
+ # The load-time gate/up pack, then each site as Studio entered it before the registry.
+ want = {
+ "load": ["fused_moe_gate_up"],
+ "text": ["fused_moe_gate_up", *legacy],
+ "vision": legacy,
+ "vision-drafted": [*legacy, "exact_speculative_verify"],
+ "audio": legacy,
+ "text-batch": [],
+ }
+ assert entered == want["load"]
+ for site in ("text", "vision", "vision-drafted", "audio", "text-batch"):
+ entered.clear()
+ _run_studio_site(site, monkeypatch)
+ assert entered == want[site], site
+ assert mlx_inference._MLX_FUSION_UNAVAILABLE == set()
+
+
[email protected]("dead", [False, True], ids = ["refused", "dead-queue"])
[email protected]("failing", ["inference_scopes", "fused_moe_router"])
+def test_a_registry_or_scope_that_fails_to_enter_keeps_the_request_native(
+ monkeypatch, mlx_inference_patches, failing, dead
+):
+ """The whole registry is guarded like each scope in it: logged once, unless the Metal queue died."""
+ from core.inference import mlx_inference
+
+ message = "[METAL] Command buffer execution failed: Caused GPU Timeout Error" if dead else "no"
+
+ @contextmanager
+ def broken(*_a, **_k):
+ raise RuntimeError(message)
+ yield # pragma: no cover
+
+ monkeypatch.setattr(mlx_inference_patches, failing, broken)
+ monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())
+ if dead:
+ with pytest.raises(RuntimeError, match = "METAL"):
+ _run_studio_site("audio", monkeypatch)
+ assert mlx_inference._MLX_FUSION_UNAVAILABLE == set()
+ else:
+ _run_studio_site("audio", monkeypatch)
+ assert [key[0] for key in mlx_inference._MLX_FUSION_UNAVAILABLE] == [failing]
+
+
def _zoo_inference_imports():
"""Names zoo's own modules import from unsloth_zoo.mlx.inference, read from source without importing zoo."""
import importlib.util
@@ -510,8 +826,7 @@ def test_mlx_missing_inference_export_keeps_native(monkeypatch, mlx_inference_pa
name = f"fused_{feature}"
monkeypatch.delattr(mlx_inference_patches, name)
model = object()
- helper = getattr(mlx_inference, f"_mlx_{name}")
- with helper(model) as active:
+ with mlx_inference._mlx_optional_fusion(name, model) as active:
assert active is model
@@ -3896,13 +4211,16 @@ def test_int8_prefill_reloads_on_change_and_reports_why_it_is_off(mlx_inference_
assert status(True, object())["active"] and status(False, object())["scope"] is None
[email protected]("registry", [True, False], ids = ["zoo-registry", "older-zoo"])
@pytest.mark.parametrize("requested,available", [(True, True), (True, False), (False, True)])
def test_int8_prefill_is_the_outermost_scope_of_generation_pinned_off_when_inactive(
- monkeypatch, mlx_inference_patches, requested, available
+ monkeypatch, mlx_inference_patches, requested, available, registry
):
from core.inference import mlx_inference
events = _int8_zoo(mlx_inference_patches, available = available)
+ if not registry:
+ monkeypatch.delattr(mlx_inference_patches, "inference_scopes")
@contextmanager
def fusion(_model):
@@ -3938,10 +4256,13 @@ def test_int8_prefill_is_the_outermost_scope_of_generation_pinned_off_when_inact
assert list(backend._generate_vlm(*args, 1.0, None)) == ["ok"]
active = requested and available
- assert events == [("int8", active)] + [("fusion", None)] * len(FUSIONS) + [
- ("stream", None),
- ("int8_exit", active),
- ]
+ # Zoo's registry enters the NAX scope again in its own place; the outer decision holds there too.
+ fusions = [("fusion", None)] * len(FUSIONS)
+ if registry:
+ fusions[5:5] = [("int8", active)]
+ assert events == [("int8", active), *fusions, ("stream", None)] + [("int8_exit", active)] * (
+ 1 + registry
+ )
def broken(model, int8_prefill = None):
raise ImportError("skewed zoo")
@@ -5733,11 +6054,6 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):
utils.prepare_inputs = lambda *_a, **_k: {"input_ids": [_Ids([7, 8, 9])], "extra": 1}
monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)
prepared, verify_scope, scopes = [], [], []
- # A Zoo without the scope leaves the reply drafted and unfused.
- with monkeypatch.context() as patch:
- patch.setattr(mlx_inference, "_mlx_inference_patch", lambda name: verify_scope.append(name))
- with mlx_inference._mlx_exact_speculative_verify(utils) as active:
- assert active is utils and verify_scope.pop() == "exact_speculative_verify"
@contextmanager
def exact_verify(model):
@@ -5747,7 +6063,9 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):
finally:
verify_scope.append("closed")
- monkeypatch.setattr(mlx_inference, "_mlx_exact_speculative_verify", exact_verify)
+ monkeypatch.setattr(
+ sys.modules["unsloth_zoo.mlx.inference"], "exact_speculative_verify", exact_verify
+ )
draft = SimpleNamespace(draft_kind = "mtp", draft_n = 6, draft_n_accepted = 4)
draft.prepare = (
lambda ids, sampling, given: prepared.append((ids, sampling.temperature, sampling.top_k))