Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
perf(studio): enter the MLX routed-experts and norm-handoff fusions per request (#12422)
Long Yixing569a7e55aa57
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 08909de2780..a78b10ee40d 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -922,6 +922,14 @@ def _mlx_fused_moe_router(model):return _mlx_optional_fusion("fused_moe_router", model)+def _mlx_fused_moe_routed_experts(model):+ return _mlx_optional_fusion("fused_moe_routed_experts", model)+++def _mlx_fused_residual_norm_handoff(model):+ return _mlx_optional_fusion("fused_residual_norm_handoff", model)++def _vlm_generation_context():import mlx.core as mxfrom mlx_vlm.generate import generation_stream@@ -5285,6 +5293,8 @@ class MLXInferenceBackend:_mlx_fused_decode_conv_silu(self._model),_mlx_fused_residual_norm(self._model),_mlx_fused_moe_router(self._model),+ _mlx_fused_moe_routed_experts(self._model),+ _mlx_fused_residual_norm_handoff(self._model),):(gen_prompt,@@ -6037,6 +6047,8 @@ class MLXInferenceBackend:generation_scope.enter_context(_mlx_fused_decode_conv_silu(self._model))generation_scope.enter_context(_mlx_fused_residual_norm(self._model))generation_scope.enter_context(_mlx_fused_moe_router(self._model))+ generation_scope.enter_context(_mlx_fused_moe_routed_experts(self._model))+ generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))final_response = Noneclip_path = Nonetry:@@ -6538,6 +6550,8 @@ class MLXInferenceBackend:generation_scope.enter_context(_mlx_fused_decode_conv_silu(self._model))generation_scope.enter_context(_mlx_fused_residual_norm(self._model))generation_scope.enter_context(_mlx_fused_moe_router(self._model))+ generation_scope.enter_context(_mlx_fused_moe_routed_experts(self._model))+ generation_scope.enter_context(_mlx_fused_residual_norm_handoff(self._model))final_response = Nonetry:with closing(diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex f0bc752ae93..0687d1438fb 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -24,16 +24,24 @@ def native_vlm_generation_context():return mlx_inference._vlm_generation_context+FUSIONS = (+ "moe_gate_up",+ "decode_conv_silu",+ "residual_norm",+ "moe_router",+ "moe_routed_experts",+ "residual_norm_handoff",+)++@pytest.fixture(autouse = True)def mlx_inference_patches(monkeypatch, native_vlm_generation_context):from core.inference import mlx_inferencemonkeypatch.setattr(mlx_inference, "_vlm_generation_context", contextlib.nullcontext)module = types.ModuleType("unsloth_zoo.mlx.inference")- module.fused_moe_gate_up = contextlib.nullcontext- module.fused_decode_conv_silu = contextlib.nullcontext- module.fused_residual_norm = contextlib.nullcontext- module.fused_moe_router = contextlib.nullcontext+ for name in FUSIONS:+ setattr(module, f"fused_{name}", contextlib.nullcontext)monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.inference", module)return module@@ -48,9 +56,7 @@ def mlx_decode(mlx_inference_patches):return mlx_inference_patches- "feature", ["moe_gate_up", "decode_conv_silu", "residual_norm", "moe_router"]-)[email protected]("feature", FUSIONS)@pytest.mark.parametrize("error",[@@ -86,9 +92,7 @@ def test_mlx_fusion_import_never_fails_the_request(monkeypatch, error, feature):assert active is model- "feature", ["moe_gate_up", "decode_conv_silu", "residual_norm", "moe_router"]-)[email protected]("feature", FUSIONS)def test_mlx_fusion_that_cannot_be_entered_keeps_native(monkeypatch, mlx_inference_patches, feature):@@ -108,9 +112,7 @@ def test_mlx_fusion_that_cannot_be_entered_keeps_native(assert active is model- "feature", ["moe_gate_up", "decode_conv_silu", "residual_norm", "moe_router"]-)[email protected]("feature", FUSIONS)@pytest.mark.parametrize("message",[@@ -139,9 +141,7 @@ def test_mlx_dead_gpu_queue_is_not_reported_as_a_working_fallback(assert mlx_inference._MLX_FUSION_UNAVAILABLE == set()- "feature", ["moe_gate_up", "decode_conv_silu", "residual_norm", "moe_router"]-)[email protected]("feature", FUSIONS)def test_mlx_packing_that_runs_out_of_memory_still_keeps_native(monkeypatch, mlx_inference_patches, feature):@@ -219,7 +219,7 @@ def test_mlx_fusion_that_refuses_everywhere_still_generates(monkeypatch, mlx_moereturn refuse- for name in ("moe_gate_up", "decode_conv_silu", "residual_norm", "moe_router"):+ for name in FUSIONS:monkeypatch.setattr(mlx_moe, f"fused_{name}", _refusing(name))monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())config = SimpleNamespace(identifier = "fake/text", is_vision = False, is_lora = False)@@ -229,9 +229,7 @@ def test_mlx_fusion_that_refuses_everywhere_still_generates(monkeypatch, mlx_moestream = backend.generate_chat_response(messages = [{"role": "user", "content": "p"}])assert next(stream) == "7"stream.close()- assert Counter(refused) == Counter(- moe_gate_up = 2, decode_conv_silu = 1, residual_norm = 1, moe_router = 1- )+ assert Counter(refused) == Counter({**dict.fromkeys(FUSIONS, 1), "moe_gate_up": 2})def _all_fusions_refuse(monkeypatch, patches):@@ -248,7 +246,7 @@ def _all_fusions_refuse(monkeypatch, patches):return refuse- for name in ("moe_gate_up", "decode_conv_silu", "residual_norm", "moe_router"):+ for name in FUSIONS:monkeypatch.setattr(patches, f"fused_{name}", _refusing(name))monkeypatch.setattr(mlx_inference, "_MLX_FUSION_UNAVAILABLE", set())return refused@@ -292,9 +290,7 @@ def test_mlx_vlm_generation_survives_every_fusion_refusing(monkeypatch, mlx_infeNone,)assert list(backend._generate_vlm(*args)) == ["ok"]- assert Counter(refused) == Counter(- moe_gate_up = 1, decode_conv_silu = 1, residual_norm = 1, moe_router = 1- )+ assert Counter(refused) == Counter(dict.fromkeys(FUSIONS, 1))def test_mlx_audio_input_generation_survives_every_fusion_refusing(@@ -330,14 +326,10 @@ def test_mlx_audio_input_generation_survives_every_fusion_refusing(max_new_tokens = 64,)) == ["H", "e", "l"]- assert Counter(refused) == Counter(- moe_gate_up = 1, decode_conv_silu = 1, residual_norm = 1, moe_router = 1- )+ assert Counter(refused) == Counter(dict.fromkeys(FUSIONS, 1))- "feature", ["moe_gate_up", "decode_conv_silu", "residual_norm", "moe_router"]-)[email protected]("feature", FUSIONS)def test_mlx_missing_inference_export_keeps_native(monkeypatch, mlx_inference_patches, feature):from core.inference import mlx_inference@@ -1038,9 +1030,7 @@ def test_vlm_iterator_restores_each_callers_stream_and_closes_on_generation_streassert closed == [generation]- "feature", ["moe_gate_up", "decode_conv_silu", "residual_norm", "moe_router"]-)[email protected]("feature", FUSIONS)def test_mlx_vlm_reemits_think_prefill_inside_adapter_context(monkeypatch, mlx_moe, mlx_decode, feature):@@ -1300,9 +1290,7 @@ def test_mlx_vlm_model_config_prefers_config_with_model_type():)- "feature", ["moe_gate_up", "decode_conv_silu", "residual_norm", "moe_router"]-)[email protected]("feature", FUSIONS)def test_mlx_generate_text_forwards_kwargs_into_template_helper(monkeypatch, mlx_moe, mlx_decode, feature):@@ -2895,9 +2883,7 @@ def test_mlx_audio_input_normalizes_split_native_reasoning_channels(monkeypatch)) == ["<think>"]- "feature", ["moe_gate_up", "decode_conv_silu", "residual_norm", "moe_router"]-)[email protected]("feature", FUSIONS)def test_mlx_audio_input_honors_adapter_selection(monkeypatch, mlx_moe, mlx_decode, feature):"""Base-vs-LoRA compare sends audio_base64 and use_adapter in one body.