Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
perf(studio): enter Zoo's inference scope registry on Unsloth Studio's MLX paths (#13975)
Long Yixinga50be79cede4
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 08e0a04e3e7..7a0bc330809 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -913,36 +913,68 @@ def _mlx_optional_fusion(name, model, *args):yield active-def _mlx_fused_moe_gate_up(model):- return _mlx_optional_fusion("fused_moe_gate_up", model)---def _mlx_fused_decode_conv_silu(model):- return _mlx_optional_fusion("fused_decode_conv_silu", model)---def _mlx_fused_residual_norm(model):- return _mlx_optional_fusion("fused_residual_norm", model)---def _mlx_fused_moe_router(model):- return _mlx_optional_fusion("fused_moe_router", model)---def _mlx_fused_moe_routed_experts(model):- return _mlx_optional_fusion("fused_moe_routed_experts", model)---def _mlx_fused_residual_norm_handoff(model):- return _mlx_optional_fusion("fused_residual_norm_handoff", model)+def _mlx_scope_refused(name, error):+ if is_metal_queue_dead(error):+ raise error+ _mlx_fusion_unavailable(name, error)+++# What Studio entered by hand before Zoo's registry; a Zoo without it keeps exactly these.+_MLX_LEGACY_SCOPES = (+ "fused_moe_gate_up",+ "fused_decode_conv_silu",+ "fused_residual_norm",+ "fused_moe_router",+ "fused_moe_routed_experts",+ "fused_residual_norm_handoff",+ "fused_packed_gated_delta",+)-def _mlx_fused_packed_gated_delta(model):- return _mlx_optional_fusion("fused_packed_gated_delta", model)+@contextmanager+def _mlx_inference_scopes(model, call, int8_prefill):+ """Hold the Zoo scopes Zoo picks for `call` (its call traits), so a new Zoo scope needs no Studio change.+ A Zoo without the registry keeps the scopes Studio listed before it (none for the text batch)."""+ compose = _mlx_inference_patch("inference_scopes")+ with ExitStack() as scope:+ if compose is None and "batch" not in call:+ for name in _MLX_LEGACY_SCOPES:+ scope.enter_context(_mlx_optional_fusion(name, model))+ if "speculative" in call:+ scope.enter_context(_mlx_optional_fusion("exact_speculative_verify", model))+ elif compose is not None:+ try:+ scope.enter_context(+ compose(+ model,+ call,+ int8_prefill = int8_prefill,+ on_error = _mlx_scope_refused,+ )+ )+ except Exception as error:+ if is_metal_queue_dead(error):+ raise+ _mlx_fusion_unavailable("inference_scopes", error)+ yield model-def _mlx_exact_speculative_verify(model):- return _mlx_optional_fusion("exact_speculative_verify", model)+def _mlx_warm_inference(model, int8_prefill):+ """Zoo's warm-up generation, so the first request skips the scopes' kernel builds and first-use+ checks. A Zoo without it, or a failed warm-up, leaves them to the first request."""+ warm = _mlx_inference_patch("warm_inference")+ if warm is None:+ return+ try:+ warm(model, int8_prefill = int8_prefill)+ except Exception as error:+ if is_metal_queue_dead(error):+ raise+ logger.warning(+ "MLX warm-up failed, so the first request runs it: %s: %s",+ type(error).__name__,+ error,+ )def _int8_prefill_zoo():@@ -3675,6 +3707,7 @@ class _TextBatchSession:_temporary_mlx_adapter_state(backend._model, adapter_state))self._held.enter_context(backend._int8_prefill_scope())+ self._held.enter_context(backend._generation_scopes("batch"))self.generator = BatchGenerator(backend._model,stop_tokens = [@@ -4452,6 +4485,12 @@ class MLXInferenceBackend:return nullcontext()return _mlx_optional_fusion("nax_quantized_linear", self._model, False)+ def _generation_scopes(self, *call):+ """Zoo's scopes for a generation of this load, under its Int8 Prefill decision. Nested in+ _int8_prefill_scope, which stays outermost."""+ active = (getattr(self, "_int8_prefill", None) or {}).get("active", False)+ return _mlx_inference_scopes(self._model, call, active)+def _kv_quant_bits(self):return (getattr(self, "_kv_quant", None) or {}).get("kv_bits")@@ -5119,7 +5158,11 @@ class MLXInferenceBackend:# The worker owns fixed base weights until unload; adapter requests stay scoped.if not is_lora:- self._model_fusion.enter_context(_mlx_fused_moe_gate_up(model))+ self._model_fusion.enter_context(_mlx_optional_fusion("fused_moe_gate_up", model))+ # Every rank of a distributed load must run each forward together.+ if not is_distributed:+ with mlx_rng_preserved(), self._int8_prefill_scope():+ _mlx_warm_inference(model, self._int8_prefill["active"])logger.info("Model %s loaded successfully", model_name)return True@@ -5756,13 +5799,7 @@ class MLXInferenceBackend:self._generation_lock,_temporary_mlx_adapter_state(self._model, _adapter_state),self._int8_prefill_scope(),- _mlx_fused_moe_gate_up(self._model),- _mlx_fused_decode_conv_silu(self._model),- _mlx_fused_residual_norm(self._model),- _mlx_fused_moe_router(self._model),- _mlx_fused_moe_routed_experts(self._model),- _mlx_fused_residual_norm_handoff(self._model),- _mlx_fused_packed_gated_delta(self._model),+ self._generation_scopes(),):(gen_prompt,@@ -6586,19 +6623,14 @@ class MLXInferenceBackend:# The vision pass gets the headroom, under the lock so nothing refills it.# A video turn resumes no snapshot, so retained ones are pure occupancy.self._release_vlm_snapshots()- generation_scope.enter_context(_mlx_fused_moe_gate_up(self._model))- generation_scope.enter_context(_mlx_fused_decode_conv_silu(self._model))- generation_scope.enter_context(_mlx_fused_residual_norm(self._model))- generation_scope.enter_context(_mlx_fused_moe_router(self._model))- generation_scope.enter_context(_mlx_fused_moe_routed_experts(self._model))- generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))- generation_scope.enter_context(_mlx_fused_packed_gated_delta(self._model))+ generation_scope.enter_context(+ self._generation_scopes("vision", *(() if draft is None else ("speculative",)))+ )final_response = Noneclip_path = Noneif draft is not None:from unsloth_zoo.mlx.generate import SamplingParams- generation_scope.enter_context(_mlx_exact_speculative_verify(self._model))# Under the lock: the load's one draft serves one generation at a time.draft.prepare(vlm_kwargs["input_ids"][0].tolist(),@@ -7154,13 +7186,7 @@ class MLXInferenceBackend:):# As on the image path: the tower gets the headroom, under the lock.self._release_vlm_snapshots()- generation_scope.enter_context(_mlx_fused_moe_gate_up(self._model))- generation_scope.enter_context(_mlx_fused_decode_conv_silu(self._model))- generation_scope.enter_context(_mlx_fused_residual_norm(self._model))- generation_scope.enter_context(_mlx_fused_moe_router(self._model))- generation_scope.enter_context(_mlx_fused_moe_routed_experts(self._model))- generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))- generation_scope.enter_context(_mlx_fused_packed_gated_delta(self._model))+ generation_scope.enter_context(self._generation_scopes("vision", "audio"))final_response = Nonetry:with closing(diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex c305b0be4d9..164a4772495 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -2,10 +2,12 @@import astimport asyncio+import importlibimport builtinsimport contextlibimport copyimport json+import osimport subprocessimport sysimport threading@@ -33,12 +35,56 @@ FUSIONS = ("residual_norm_handoff","packed_gated_delta",)-# Fused scopes zoo's own generate.py enters that Studio does not wrap itself, so they get no-# _mlx_fused_* helper and stay out of the FUSIONS parametrisations: unsloth_zoo #1757 added-# fused_verify_attention around speculative verify. The stub still has to answer them.+# Stub scopes kept out of the FUSIONS parametrisations: unsloth_zoo #1757 added fused_verify_attention+# around speculative verify. The stub still has to answer them.ZOO_ONLY_FUSIONS = ("verify_attention",)+def _zoo_registry():+ """The stub's INFERENCE_SCOPES in zoo's order: (scope name, call traits it serves[, opt-in switch])."""+ return [+ *((f"fused_{name}", ()) for name in FUSIONS[:5]),+ ("nax_quantized_linear", ()),+ ("dense_prefill_linear", ()),+ *((f"fused_{name}", ()) for name in FUSIONS[5:]),+ ("exact_speculative_verify", ("speculative",)),+ ("fused_verify_attention", ("speculative",)),+ ]+++def _stub_inference_scopes(module):+ """zoo's inference_scopes over the stub: each registered scope is looked up by name when entered,+ so a test that replaces a stub scope sees it entered through the registry."""++ @contextmanager+ def inference_scopes(+ model = None,+ call = None,+ *,+ int8_prefill = None,+ on_error = None,+ _modules = None,+ ):+ # The call-shape replay below passes placeholders; anything but a tuple serves every scope.+ traits = set(call) if isinstance(call, tuple) else None+ with contextlib.ExitStack() as stack:+ for name, serves, *opt_in in module.INFERENCE_SCOPES:+ if traits is not None and serves and traits.isdisjoint(serves):+ continue+ if opt_in and opt_in[0] and os.environ.get(opt_in[0]) != "1":+ continue+ args = (int8_prefill,) if name == "nax_quantized_linear" else ()+ try:+ stack.enter_context(getattr(module, name)(model, *args))+ except Exception as error:+ if on_error is None:+ raise+ on_error(name, error)+ yield model++ return inference_scopes++@pytest.fixture(autouse = True)def mlx_inference_patches(monkeypatch, native_vlm_generation_context):from core.inference import mlx_inference@@ -49,6 +95,8 @@ def mlx_inference_patches(monkeypatch, native_vlm_generation_context):setattr(module, f"fused_{name}", _neutral_scope)for name, helper in PLAIN_HELPERS.items():setattr(module, name, helper)+ module.INFERENCE_SCOPES = _zoo_registry()+ module.inference_scopes = _stub_inference_scopes(module)module.__getattr__ = _neutral_zoo_helpermonkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.inference", module)return module@@ -87,10 +135,14 @@ def _neutral_zoo_helper(name):"""Every other public helper zoo's generate.py enters around a model is neutral too.unsloth_zoo #1541 added nax_quantized_linear to that import list, and a stub holding only the- fused_* names failed every real-model test on macOS with an ImportError. fused_* stay explicit,- so a test that deletes one still sees it missing.+ fused_* names failed every real-model test on macOS with an ImportError. fused_*,+ inference_scopes and warm_inference stay explicit, so a test that deletes one still sees it missing."""- if name.startswith("_") or name.startswith("fused_"):+ if (+ name.startswith("_")+ or name.startswith("fused_")+ or name in ("inference_scopes", "warm_inference")+ ):raise AttributeError(name)return _neutral_scope@@ -125,7 +177,7 @@ def test_mlx_fusion_import_never_fails_the_request(monkeypatch, error, feature):from core.inference import mlx_inferencemodule_name = "unsloth_zoo.mlx.inference"- helper = getattr(mlx_inference, f"_mlx_fused_{feature}")+ helper = lambda model: mlx_inference._mlx_optional_fusion(f"fused_{feature}", model)original_import = mlx_inference.importlib.import_module@@ -157,7 +209,7 @@ def test_mlx_fusion_that_cannot_be_entered_keeps_native(monkeypatch.setattr(mlx_inference_patches, f"fused_{feature}", refuse)monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())model = object()- with getattr(mlx_inference, f"_mlx_fused_{feature}")(model) as active:+ with mlx_inference._mlx_optional_fusion(f"fused_{feature}", model) as active:assert active is model@@ -185,7 +237,7 @@ def test_mlx_dead_gpu_queue_is_not_reported_as_a_working_fallback(monkeypatch.setattr(mlx_inference_patches, f"fused_{feature}", fault)monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())with pytest.raises(RuntimeError, match = "METAL"):- with getattr(mlx_inference, f"_mlx_fused_{feature}")(object()):+ with mlx_inference._mlx_optional_fusion(f"fused_{feature}", object()):pass # pragma: no coverassert mlx_inference._MLX_FUSION_UNAVAILABLE == set()@@ -204,7 +256,7 @@ def test_mlx_packing_that_runs_out_of_memory_still_keeps_native(monkeypatch.setattr(mlx_inference_patches, f"fused_{feature}", refuse)monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())model = object()- with getattr(mlx_inference, f"_mlx_fused_{feature}")(model) as active:+ with mlx_inference._mlx_optional_fusion(f"fused_{feature}", model) as active:assert active is model@@ -250,6 +302,78 @@ def test_mlx_fusion_failure_at_load_still_loads_the_model(monkeypatch, mlx_moe):assert backend.unload_model("fake/text")[email protected]("outcome", ["warmed", "int8", "failed", "dead_queue", "distributed"])+def test_a_load_warms_zoo_scopes_under_its_int8_decision(monkeypatch, mlx_moe, outcome):+ from core.inference import mlx_inference++ backend = _install_fake_text_stack(monkeypatch, {"p": [1, 2], "generated": [7, 8]}, [])+ _install_fake_fast_mlx(monkeypatch, [])+ sys.modules["mlx.core"].clear_cache = lambda: None+ warnings = _capture_rng_warnings(monkeypatch, mlx_inference)+ events = []++ @contextmanager+ def rng():+ events.append("rng")+ try:+ yield+ finally:+ events.append("/rng")++ @contextmanager+ def gate_up(model, **_):+ events.append("gate_up")+ yield model++ @contextmanager+ def int8(+ model,+ int8_prefill = None,+ **_,+ ):+ events.append(("int8", int8_prefill))+ try:+ yield model+ finally:+ events.append("/int8")++ def warm(model, int8_prefill):+ events.append(("warm", model is backend._model, int8_prefill))+ if outcome == "failed":+ raise RuntimeError("no headroom")+ if outcome == "dead_queue":+ raise RuntimeError("[METAL] Command buffer execution failed: Caused GPU Timeout Error.")++ mlx_moe.fused_moe_gate_up, mlx_moe.nax_quantized_linear, mlx_moe.warm_inference = (+ gate_up,+ int8,+ warm,+ )+ monkeypatch.setattr(mlx_inference, "mlx_rng_preserved", rng)+ if outcome == "int8":+ status = {"requested": True, "active": True, "reason": "", "scope": int8}+ monkeypatch.setattr(mlx_inference, "_int8_prefill_status", lambda *_: status)+ config = SimpleNamespace(identifier = "fake/text", is_vision = False, is_lora = False)+ group = SimpleNamespace(size = lambda: 2, rank = lambda: 0)+ kwargs = (+ {"parallel_mode": "tensor", "distributed_group": group} if outcome == "distributed" else {}+ )+ with (+ pytest.raises(RuntimeError, match = "METAL")+ if outcome == "dead_queue"+ else contextlib.nullcontext()+ ):+ assert backend.load_model(config, **kwargs) is True+ active = outcome == "int8"+ warmed = (+ []+ if outcome == "distributed"+ else ["rng", ("int8", active), ("warm", True, active), "/int8", "/rng"]+ )+ assert events == ["gate_up", *warmed]+ assert any("MLX warm-up failed" in warning for warning in warnings) == (outcome == "failed")++def test_mlx_fusion_that_refuses_everywhere_still_generates(monkeypatch, mlx_moe):from core.inference import mlx_inference@@ -378,6 +502,198 @@ def test_mlx_audio_input_generation_survives_every_fusion_refusing(assert Counter(refused) == Counter(dict.fromkeys(FUSIONS, 1))+class _Ids(list):+ def tolist(self):+ return list(self)+++def _run_studio_site(site, monkeypatch):+ """Drive one of Studio's own MLX generation paths once, on fakes."""+ from core.inference import mlx_inference++ backend = mlx_inference.MLXInferenceBackend()+ if site == "text":+ backend = _install_fake_text_stack(monkeypatch, {"p": [1, 2], "generated": [7]}, [])+ tokenizer = backend._tokenizer+ _install_fake_fast_mlx(monkeypatch, [])+ sys.modules["mlx.core"].clear_cache = lambda: None+ config = SimpleNamespace(identifier = "fake/text", is_vision = False, is_lora = False)+ assert backend.load_model(config)+ backend._tokenizer = tokenizer+ assert list(backend.generate_chat_response(messages = [{"role": "user", "content": "p"}]))+ elif site.startswith("vision"):+ if site == "vision-drafted":+ pytest.importorskip("mlx_lm")+ utils = types.ModuleType("mlx_vlm.utils")+ utils.prepare_inputs = lambda *_a, **_k: {"input_ids": [_Ids([7])]}+ monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)+ backend._speculative_draft = SimpleNamespace(+ draft_kind = "mtp", draft_n = 1, draft_n_accepted = 1, prepare = lambda *_a: None+ )+ assert "draft_model" in _drive_vlm_generation(backend, monkeypatch) or site == "vision"+ elif site == "audio":+ fake_vlm = types.ModuleType("mlx_vlm")+ fake_vlm.stream_generate = lambda *_a, **_k: iter(+ [SimpleNamespace(text = "H", prompt_tokens = 3, generation_tokens = 1)]+ )+ monkeypatch.setitem(sys.modules, "mlx_vlm", fake_vlm)+ monkeypatch.setattr(mlx_inference, "_render_registered_vlm_prompt", lambda *_a, **_k: "P")+ backend._model, backend._processor = _audio_model(), _audio_processor()+ backend.active_model_name, backend.models = "m", {"m": {"audio_type": "audio_vlm"}}+ response = backend.generate_audio_input_response(+ messages = [{"role": "user", "content": "?"}],+ system_prompt = "",+ audio_array = [0.0, 0.1],+ max_new_tokens = 1,+ )+ assert list(response) == ["H"]+ else:+ generate = types.ModuleType("mlx_lm.generate")+ generate.BatchGenerator = lambda *_a, **_k: SimpleNamespace(close = lambda: None)+ backend._model, backend._tokenizer = SimpleNamespace(), SimpleNamespace(eos_token_ids = [2])+ with (+ monkeypatch.context() as patch+ ): # only while it is built: mlx_lm itself imports from it+ patch.setitem(sys.modules, "mlx_lm.generate", generate)+ mlx_inference._TextBatchSession(backend, width = 2).close()++[email protected](params = ["stub", "zoo"])+def zoo_registry(request, monkeypatch, mlx_inference_patches):+ """Register scopes in zoo's registry only, entered through the stub's composer or zoo's own."""+ if request.param == "stub":+ register = lambda name, serves = (), opt_in = None: (+ mlx_inference_patches.INFERENCE_SCOPES.append((name, serves, opt_in))+ )+ return register+ monkeypatch.delitem(sys.modules, "unsloth_zoo.mlx.inference")+ try:+ zoo = importlib.import_module("unsloth_zoo.mlx.inference")+ except Exception as error:+ pytest.skip(f"unsloth_zoo.mlx.inference is not importable here: {error}")+ finally:+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.inference", mlx_inference_patches)+ if not hasattr(zoo, "INFERENCE_SCOPES"):+ pytest.skip("this unsloth_zoo has no inference scope registry")+ monkeypatch.setattr(zoo, "INFERENCE_SCOPES", [])+ monkeypatch.setattr(mlx_inference_patches, "inference_scopes", zoo.inference_scopes)++ def register(+ name,+ serves = (),+ opt_in = None,+ ):+ scope = getattr(mlx_inference_patches, name)+ enter = lambda model, *args, _modules: scope(model, *args)+ enter.__name__ = name+ zoo.INFERENCE_SCOPES.append(zoo.InferenceScope(enter, frozenset(serves), opt_in))++ return register++[email protected]("site", ["text", "vision", "vision-drafted", "audio", "text-batch"])+def test_a_scope_zoo_registers_reaches_each_studio_site_with_no_studio_change(+ monkeypatch, mlx_inference_patches, zoo_registry, site+):+ from core.inference import mlx_inference++ entered = []++ def scope(name):+ @contextmanager+ def enter(model, *_args):+ entered.append(name)+ yield model++ return enter++ traits = ("speculative", "vision", "audio", "batch")+ for name, serves, opt_in in (+ ("every", (), None),+ *((trait, (trait,), None) for trait in traits),+ ("switched_off", (), "UNSLOTH_MLX_STUDIO_TEST_SCOPE"),+ ):+ setattr(mlx_inference_patches, name, scope(name))+ zoo_registry(name, serves, opt_in)+ monkeypatch.delenv("UNSLOTH_MLX_STUDIO_TEST_SCOPE", raising = False)+ monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())+ _run_studio_site(site, monkeypatch)+ assert mlx_inference._MLX_FUSION_UNAVAILABLE == set() # the registry and every scope entered+ call = {+ "text": (),+ "vision": ("vision",),+ "vision-drafted": ("speculative", "vision"),+ "audio": ("vision", "audio"),+ "text-batch": ("batch",),+ }[site]+ assert entered == ["every", *call]+++def test_an_older_zoo_without_the_registry_keeps_the_scopes_studio_listed(+ monkeypatch, mlx_inference_patches+):+ from core.inference import mlx_inference++ pytest.importorskip("mlx_lm") # the drafted vision site imports zoo's generate+ entered = []+ for name in (*(f"fused_{name}" for name in FUSIONS), "exact_speculative_verify"):+ monkeypatch.setattr(+ mlx_inference_patches,+ name,+ lambda model, name = name: entered.append(name) or contextlib.nullcontext(model),+ )+ monkeypatch.delattr(mlx_inference_patches, "inference_scopes")+ monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())+ backend = _install_fake_text_stack(monkeypatch, {"p": [1, 2], "generated": [7, 8]}, [])+ _install_fake_fast_mlx(monkeypatch, [])+ sys.modules["mlx.core"].clear_cache = lambda: None+ assert backend.load_model(+ SimpleNamespace(identifier = "fake/text", is_vision = False, is_lora = False)+ )+ legacy = [f"fused_{name}" for name in FUSIONS]+ # The load-time gate/up pack, then each site as Studio entered it before the registry.+ want = {+ "load": ["fused_moe_gate_up"],+ "text": ["fused_moe_gate_up", *legacy],+ "vision": legacy,+ "vision-drafted": [*legacy, "exact_speculative_verify"],+ "audio": legacy,+ "text-batch": [],+ }+ assert entered == want["load"]+ for site in ("text", "vision", "vision-drafted", "audio", "text-batch"):+ entered.clear()+ _run_studio_site(site, monkeypatch)+ assert entered == want[site], site+ assert mlx_inference._MLX_FUSION_UNAVAILABLE == set()++[email protected]("dead", [False, True], ids = ["refused", "dead-queue"])[email protected]("failing", ["inference_scopes", "fused_moe_router"])+def test_a_registry_or_scope_that_fails_to_enter_keeps_the_request_native(+ monkeypatch, mlx_inference_patches, failing, dead+):+ """The whole registry is guarded like each scope in it: logged once, unless the Metal queue died."""+ from core.inference import mlx_inference++ message = "[METAL] Command buffer execution failed: Caused GPU Timeout Error" if dead else "no"++ @contextmanager+ def broken(*_a, **_k):+ raise RuntimeError(message)+ yield # pragma: no cover++ monkeypatch.setattr(mlx_inference_patches, failing, broken)+ monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())+ if dead:+ with pytest.raises(RuntimeError, match = "METAL"):+ _run_studio_site("audio", monkeypatch)+ assert mlx_inference._MLX_FUSION_UNAVAILABLE == set()+ else:+ _run_studio_site("audio", monkeypatch)+ assert [key[0] for key in mlx_inference._MLX_FUSION_UNAVAILABLE] == [failing]++def _zoo_inference_imports():"""Names zoo's own modules import from unsloth_zoo.mlx.inference, read from source without importing zoo."""import importlib.util@@ -510,8 +826,7 @@ def test_mlx_missing_inference_export_keeps_native(monkeypatch, mlx_inference_paname = f"fused_{feature}"monkeypatch.delattr(mlx_inference_patches, name)model = object()- helper = getattr(mlx_inference, f"_mlx_{name}")- with helper(model) as active:+ with mlx_inference._mlx_optional_fusion(name, model) as active:assert active is model@@ -3896,13 +4211,16 @@ def test_int8_prefill_reloads_on_change_and_reports_why_it_is_off(mlx_inference_assert status(True, object())["active"] and status(False, object())["scope"] is None[email protected]("registry", [True, False], ids = ["zoo-registry", "older-zoo"])@pytest.mark.parametrize("requested,available", [(True, True), (True, False), (False, True)])def test_int8_prefill_is_the_outermost_scope_of_generation_pinned_off_when_inactive(- monkeypatch, mlx_inference_patches, requested, available+ monkeypatch, mlx_inference_patches, requested, available, registry):from core.inference import mlx_inferenceevents = _int8_zoo(mlx_inference_patches, available = available)+ if not registry:+ monkeypatch.delattr(mlx_inference_patches, "inference_scopes")@contextmanagerdef fusion(_model):@@ -3938,10 +4256,13 @@ def test_int8_prefill_is_the_outermost_scope_of_generation_pinned_off_when_inactassert list(backend._generate_vlm(*args, 1.0, None)) == ["ok"]active = requested and available- assert events == [("int8", active)] + [("fusion", None)] * len(FUSIONS) + [- ("stream", None),- ("int8_exit", active),- ]+ # Zoo's registry enters the NAX scope again in its own place; the outer decision holds there too.+ fusions = [("fusion", None)] * len(FUSIONS)+ if registry:+ fusions[5:5] = [("int8", active)]+ assert events == [("int8", active), *fusions, ("stream", None)] + [("int8_exit", active)] * (+ 1 + registry+ )def broken(model, int8_prefill = None):raise ImportError("skewed zoo")@@ -5733,11 +6054,6 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):utils.prepare_inputs = lambda *_a, **_k: {"input_ids": [_Ids([7, 8, 9])], "extra": 1}monkeypatch.setitem(sys.modules, "mlx_vlm.utils", utils)prepared, verify_scope, scopes = [], [], []- # A Zoo without the scope leaves the reply drafted and unfused.- with monkeypatch.context() as patch:- patch.setattr(mlx_inference, "_mlx_inference_patch", lambda name: verify_scope.append(name))- with mlx_inference._mlx_exact_speculative_verify(utils) as active:- assert active is utils and verify_scope.pop() == "exact_speculative_verify"@contextmanagerdef exact_verify(model):@@ -5747,7 +6063,9 @@ def test_a_speculative_load_drafts_eligible_single_replies(monkeypatch):finally:verify_scope.append("closed")- monkeypatch.setattr(mlx_inference, "_mlx_exact_speculative_verify", exact_verify)+ monkeypatch.setattr(+ sys.modules["unsloth_zoo.mlx.inference"], "exact_speculative_verify", exact_verify+ )draft = SimpleNamespace(draft_kind = "mtp", draft_n = 6, draft_n_accepted = 4)draft.prepare = (lambda ids, sampling, given: prepared.append((ids, sampling.temperature, sampling.top_k))