Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 MoE + fast_inference: refuse unsupported checkpoints before vLLM loads, with a hint that works (#13277)

Daniel Handc9b342a2aa2
diff --git a/tests/test_fp8_moe_fast_inference_gate.py b/tests/test_fp8_moe_fast_inference_gate.py
new file mode 100644
index 00000000000..72bfb5060d9
--- /dev/null
+++ b/tests/test_fp8_moe_fast_inference_gate.py
@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""fast_inference on FP8 sparse MoE: block FP8 is shared with vLLM by unsloth_zoo, every other FP8
+MoE format is refused before vLLM allocates the engine."""
+
+import ast
+import re
+import builtins
+import sys
+import types
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+VISION_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "vision.py"
+TREE = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))
+
+
+def _functions(*names):
+ nodes = [n for n in TREE.body if isinstance(n, ast.FunctionDef) and n.name in names]
+ assert len(nodes) == len(names)
+ namespace = {"re": re}
+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(VISION_PATH), "exec"), namespace)
+ return namespace
+
+
+def _refusal():
+ return _functions("_is_sparse_moe_config", "_fp8_moe_fast_inference_refusal")[
+ "_fp8_moe_fast_inference_refusal"
+ ]
+
+
+BLOCK = dict(quant_method = "fp8", weight_block_size = [128, 128], activation_scheme = "dynamic")
+
+
+def _moe(**quant):
+ return SimpleNamespace(
+ num_experts = 128, moe_intermediate_size = 768, quantization_config = {**BLOCK, **quant}
+ )
+
+
+def zoo_scheme(monkeypatch):
+ seen = []
+
+ def fp8_moe_checkpoint_scheme(config):
+ seen.append(config)
+ quant = config.quantization_config
+ if quant.get("weight_block_size") != [128, 128]:
+ return False, "weight_block_size is not [128, 128]"
+ return True, None
+
+ empty_model = types.ModuleType("unsloth_zoo.empty_model")
+ empty_model.fp8_moe_checkpoint_scheme = fp8_moe_checkpoint_scheme
+ zoo = types.ModuleType("unsloth_zoo")
+ zoo.empty_model = empty_model
+ monkeypatch.setitem(sys.modules, "unsloth_zoo", zoo)
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.empty_model", empty_model)
+ return seen
+
+
+def old_zoo(monkeypatch):
+ real_import = builtins.__import__
+
+ def fake_import(name, *args, **kwargs):
+ if name == "unsloth_zoo.empty_model":
+ raise ImportError(name)
+ return real_import(name, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "__import__", fake_import)
+
+
+def test_block_fp8_moe_passes_with_a_sharing_zoo(zoo_scheme):
+ assert _refusal()(_moe()) is None
+ assert len(zoo_scheme) == 1
+
+
+def test_other_fp8_moe_schemes_are_refused_with_the_zoo_reason(zoo_scheme):
+ assert "weight_block_size" in _refusal()(_moe(weight_block_size = None))
+
+
+def test_old_zoo_refuses_every_fp8_moe(old_zoo):
+ refusal = _refusal()
+ assert "upgrade unsloth_zoo" in refusal(_moe())
+ assert "upgrade unsloth_zoo" in refusal(
+ SimpleNamespace(num_experts = None, n_routed_experts = 64, quantization_config = BLOCK)
+ )
+ assert refusal(SimpleNamespace(num_experts = None, quantization_config = BLOCK)) is None
+ assert refusal(SimpleNamespace(num_experts = 8)) is None
+ assert (
+ refusal(
+ SimpleNamespace(num_experts = 8, quantization_config = {"quant_method": "bitsandbytes"})
+ )
+ is None
+ )
+ assert (
+ refusal(SimpleNamespace(num_experts = 8, quantization_config = {"quant_method": "mxfp4"}))
+ is None
+ )
+ # FP8 attention with unquantized experts: vLLM keeps them 16-bit and they are shared as on main.
+ assert (
+ refusal(_moe(weight_block_size = None, modules_to_not_convert = ["model.layers.0.mlp.experts"]))
+ is None
+ )
+ assert "upgrade" in refusal(_moe(modules_to_not_convert = ["model.layers.0.mlp.shared_experts"]))
+
+
+def test_the_refusal_runs_before_load_vllm():
+ source = VISION_PATH.read_text(encoding = "utf-8")
+ gate = source.index("fp8_moe_refusal = _fp8_moe_fast_inference_refusal(model_config)")
+ assert gate < source.index("llm = load_vllm(**load_vllm_kwargs)")
+ assert "raise NotImplementedError" in source[gate : gate + 800]
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 5ff6b30ba4e..0465f33b91e 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -1170,6 +1170,48 @@ def _is_sparse_moe_config(config):
)
+def _fp8_moe_fast_inference_refusal(config):
+ """Refusal reason for FP8 sparse MoE + fast_inference, else None (old zoo shares no FP8 experts)."""
+ try:
+ from unsloth_zoo.empty_model import fp8_moe_checkpoint_scheme
+ except Exception:
+ fp8_moe_checkpoint_scheme = None
+ if fp8_moe_checkpoint_scheme is not None:
+ scheme = fp8_moe_checkpoint_scheme(config)
+ if scheme is None or scheme[0]:
+ return None
+ return scheme[1]
+ text_config = getattr(config, "text_config", None) or config
+ if not (
+ _is_sparse_moe_config(config)
+ or getattr(text_config, "num_local_experts", None)
+ or getattr(text_config, "n_routed_experts", None)
+ ):
+ return None
+ quant = getattr(config, "quantization_config", None) or getattr(
+ text_config, "quantization_config", None
+ )
+ if not quant:
+ return None
+ if not isinstance(quant, dict):
+ quant = quant.to_dict() if hasattr(quant, "to_dict") else dict(vars(quant))
+ method = str(quant.get("quant_method", "")).lower()
+ if "fp8" not in method or method.startswith("mx"):
+ return None
+ # Experts left unquantized by the checkpoint stay 16-bit in vLLM and are shared as before.
+ skipped = [
+ entry
+ for key in ("modules_to_not_convert", "ignore", "ignored_layers", "exclude_modules")
+ for entry in (quant.get(key, None) or ())
+ ]
+ if any(
+ isinstance(entry, str) and re.search(r"(^|[.:*])experts($|[.$*])", entry)
+ for entry in skipped
+ ):
+ return None
+ return "this unsloth_zoo cannot share FP8 MoE experts with vLLM (upgrade unsloth_zoo)"
+
+
def _zoo_supports_moe_fast_inference():
# Older unsloth_zoo releases leave every expert a 1-wide placeholder in the training model.
try:
@@ -3900,6 +3942,17 @@ class FastBaseModel:
"Load in 16-bit (load_in_4bit = False, load_in_8bit = False), or set fast_inference = False."
)
+ # Refuse unsupported FP8 MoE before vLLM allocates the engine.
+ fp8_moe_refusal = _fp8_moe_fast_inference_refusal(model_config)
+ if fp8_moe_refusal is not None:
+ raise NotImplementedError(
+ f"Unsloth: fast_inference = True cannot share the FP8 MoE experts of {model_type_arch} "
+ f"with training: {fp8_moe_refusal}.\n"
+ "Supported: block FP8 checkpoints (quant_method = 'fp8', weight_block_size = [128, 128], "
+ "dynamic activations). Load the 16-bit (BF16) checkpoint with fast_inference = True, "
+ "or set fast_inference = False."
+ )
+
allowed_args = inspect.getfullargspec(load_vllm).args
_vllm_model_name = resolve_vllm_model_path(
model_name,