Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 MoE + fast_inference: refuse unsupported checkpoints before vLLM loads, with a hint that works (#13277)
Daniel Handc9b342a2aa2
diff --git a/tests/test_fp8_moe_fast_inference_gate.py b/tests/test_fp8_moe_fast_inference_gate.pynew file mode 100644index 00000000000..72bfb5060d9--- /dev/null+++ b/tests/test_fp8_moe_fast_inference_gate.py@@ -0,0 +1,114 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""fast_inference on FP8 sparse MoE: block FP8 is shared with vLLM by unsloth_zoo, every other FP8+MoE format is refused before vLLM allocates the engine."""++import ast+import re+import builtins+import sys+import types+from pathlib import Path+from types import SimpleNamespace++import pytest++VISION_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "vision.py"+TREE = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))+++def _functions(*names):+ nodes = [n for n in TREE.body if isinstance(n, ast.FunctionDef) and n.name in names]+ assert len(nodes) == len(names)+ namespace = {"re": re}+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(VISION_PATH), "exec"), namespace)+ return namespace+++def _refusal():+ return _functions("_is_sparse_moe_config", "_fp8_moe_fast_inference_refusal")[+ "_fp8_moe_fast_inference_refusal"+ ]+++BLOCK = dict(quant_method = "fp8", weight_block_size = [128, 128], activation_scheme = "dynamic")+++def _moe(**quant):+ return SimpleNamespace(+ num_experts = 128, moe_intermediate_size = 768, quantization_config = {**BLOCK, **quant}+ )+++def zoo_scheme(monkeypatch):+ seen = []++ def fp8_moe_checkpoint_scheme(config):+ seen.append(config)+ quant = config.quantization_config+ if quant.get("weight_block_size") != [128, 128]:+ return False, "weight_block_size is not [128, 128]"+ return True, None++ empty_model = types.ModuleType("unsloth_zoo.empty_model")+ empty_model.fp8_moe_checkpoint_scheme = fp8_moe_checkpoint_scheme+ zoo = types.ModuleType("unsloth_zoo")+ zoo.empty_model = empty_model+ monkeypatch.setitem(sys.modules, "unsloth_zoo", zoo)+ monkeypatch.setitem(sys.modules, "unsloth_zoo.empty_model", empty_model)+ return seen+++def old_zoo(monkeypatch):+ real_import = builtins.__import__++ def fake_import(name, *args, **kwargs):+ if name == "unsloth_zoo.empty_model":+ raise ImportError(name)+ return real_import(name, *args, **kwargs)++ monkeypatch.setattr(builtins, "__import__", fake_import)+++def test_block_fp8_moe_passes_with_a_sharing_zoo(zoo_scheme):+ assert _refusal()(_moe()) is None+ assert len(zoo_scheme) == 1+++def test_other_fp8_moe_schemes_are_refused_with_the_zoo_reason(zoo_scheme):+ assert "weight_block_size" in _refusal()(_moe(weight_block_size = None))+++def test_old_zoo_refuses_every_fp8_moe(old_zoo):+ refusal = _refusal()+ assert "upgrade unsloth_zoo" in refusal(_moe())+ assert "upgrade unsloth_zoo" in refusal(+ SimpleNamespace(num_experts = None, n_routed_experts = 64, quantization_config = BLOCK)+ )+ assert refusal(SimpleNamespace(num_experts = None, quantization_config = BLOCK)) is None+ assert refusal(SimpleNamespace(num_experts = 8)) is None+ assert (+ refusal(+ SimpleNamespace(num_experts = 8, quantization_config = {"quant_method": "bitsandbytes"})+ )+ is None+ )+ assert (+ refusal(SimpleNamespace(num_experts = 8, quantization_config = {"quant_method": "mxfp4"}))+ is None+ )+ # FP8 attention with unquantized experts: vLLM keeps them 16-bit and they are shared as on main.+ assert (+ refusal(_moe(weight_block_size = None, modules_to_not_convert = ["model.layers.0.mlp.experts"]))+ is None+ )+ assert "upgrade" in refusal(_moe(modules_to_not_convert = ["model.layers.0.mlp.shared_experts"]))+++def test_the_refusal_runs_before_load_vllm():+ source = VISION_PATH.read_text(encoding = "utf-8")+ gate = source.index("fp8_moe_refusal = _fp8_moe_fast_inference_refusal(model_config)")+ assert gate < source.index("llm = load_vllm(**load_vllm_kwargs)")+ assert "raise NotImplementedError" in source[gate : gate + 800]diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 5ff6b30ba4e..0465f33b91e 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -1170,6 +1170,48 @@ def _is_sparse_moe_config(config):)+def _fp8_moe_fast_inference_refusal(config):+ """Refusal reason for FP8 sparse MoE + fast_inference, else None (old zoo shares no FP8 experts)."""+ try:+ from unsloth_zoo.empty_model import fp8_moe_checkpoint_scheme+ except Exception:+ fp8_moe_checkpoint_scheme = None+ if fp8_moe_checkpoint_scheme is not None:+ scheme = fp8_moe_checkpoint_scheme(config)+ if scheme is None or scheme[0]:+ return None+ return scheme[1]+ text_config = getattr(config, "text_config", None) or config+ if not (+ _is_sparse_moe_config(config)+ or getattr(text_config, "num_local_experts", None)+ or getattr(text_config, "n_routed_experts", None)+ ):+ return None+ quant = getattr(config, "quantization_config", None) or getattr(+ text_config, "quantization_config", None+ )+ if not quant:+ return None+ if not isinstance(quant, dict):+ quant = quant.to_dict() if hasattr(quant, "to_dict") else dict(vars(quant))+ method = str(quant.get("quant_method", "")).lower()+ if "fp8" not in method or method.startswith("mx"):+ return None+ # Experts left unquantized by the checkpoint stay 16-bit in vLLM and are shared as before.+ skipped = [+ entry+ for key in ("modules_to_not_convert", "ignore", "ignored_layers", "exclude_modules")+ for entry in (quant.get(key, None) or ())+ ]+ if any(+ isinstance(entry, str) and re.search(r"(^|[.:*])experts($|[.$*])", entry)+ for entry in skipped+ ):+ return None+ return "this unsloth_zoo cannot share FP8 MoE experts with vLLM (upgrade unsloth_zoo)"++def _zoo_supports_moe_fast_inference():# Older unsloth_zoo releases leave every expert a 1-wide placeholder in the training model.try:@@ -3900,6 +3942,17 @@ class FastBaseModel:"Load in 16-bit (load_in_4bit = False, load_in_8bit = False), or set fast_inference = False.")+ # Refuse unsupported FP8 MoE before vLLM allocates the engine.+ fp8_moe_refusal = _fp8_moe_fast_inference_refusal(model_config)+ if fp8_moe_refusal is not None:+ raise NotImplementedError(+ f"Unsloth: fast_inference = True cannot share the FP8 MoE experts of {model_type_arch} "+ f"with training: {fp8_moe_refusal}.\n"+ "Supported: block FP8 checkpoints (quant_method = 'fp8', weight_block_size = [128, 128], "+ "dynamic activations). Load the 16-bit (BF16) checkpoint with fast_inference = True, "+ "or set fast_inference = False."+ )+allowed_args = inspect.getfullargspec(load_vllm).args_vllm_model_name = resolve_vllm_model_path(model_name,