Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fast_inference for Qwen3.5 / 3.6 MoE and Gemma-4 MoE with LoRA on the experts (#12742)

Daniel Hanf8f746739ba3
diff --git a/tests/test_moe_fast_inference_gate.py b/tests/test_moe_fast_inference_gate.py
new file mode 100644
index 00000000000..1c33f4747eb
--- /dev/null
+++ b/tests/test_moe_fast_inference_gate.py
@@ -0,0 +1,207 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import ast
+from pathlib import Path
+from types import SimpleNamespace
+
+VISION_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "vision.py"
+TREE = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))
+
+
+def _module_constant(name):
+ for node in TREE.body:
+ if isinstance(node, ast.Assign) and any(
+ isinstance(t, ast.Name) and t.id == name for t in node.targets
+ ):
+ return ast.literal_eval(node.value)
+ raise AssertionError(f"{name} not found")
+
+
+def _is_sparse_moe_config():
+ for node in TREE.body:
+ if isinstance(node, ast.FunctionDef) and node.name == "_is_sparse_moe_config":
+ namespace = {}
+ exec(
+ compile(ast.Module(body = [node], type_ignores = []), str(VISION_PATH), "exec"),
+ namespace,
+ )
+ return namespace["_is_sparse_moe_config"]
+ raise AssertionError("_is_sparse_moe_config not found")
+
+
+GEMMA4_MOE = SimpleNamespace(text_config = SimpleNamespace(num_experts = 128, enable_moe_block = True))
+GEMMA4_DENSE = SimpleNamespace(
+ text_config = SimpleNamespace(num_experts = None, enable_moe_block = False)
+)
+QWEN3_5_MOE = SimpleNamespace(text_config = SimpleNamespace(num_experts = 256))
+QWEN3_5_DENSE = SimpleNamespace(text_config = SimpleNamespace(num_experts = None))
+
+
+def test_moe_model_types_pass_the_allowlist():
+ supported = _module_constant("VLLM_SUPPORTED_VLM")
+ for arch in ("qwen3_5_moe", "gemma4", "gemma4_text"):
+ assert arch in supported
+ assert len(supported) == len(set(supported))
+
+
+def test_sparse_moe_is_read_off_the_text_config():
+ is_moe = _is_sparse_moe_config()
+ assert is_moe(GEMMA4_MOE) is True
+ assert is_moe(QWEN3_5_MOE) is True
+ assert is_moe(GEMMA4_DENSE) is False
+ assert is_moe(QWEN3_5_DENSE) is False
+
+
+def test_a_config_without_text_config_is_read_directly():
+ is_moe = _is_sparse_moe_config()
+ assert is_moe(SimpleNamespace(num_experts = 128)) is True
+ assert is_moe(SimpleNamespace(enable_moe_block = True)) is True
+ assert is_moe(SimpleNamespace()) is False
+
+
+def test_dense_gemma4_is_refused_at_the_gate():
+ """Dense Gemma-4 shares the gemma4 model type, so the allowlist alone would admit it."""
+ assert set(_module_constant("VLLM_MOE_ONLY_VLM")) == {"gemma4", "gemma4_text"}
+ gate, parents = _gate("not the dense ones yet")
+ # text_only = True sets is_vlm_config False, so the gate must not sit under it.
+ assert not any("is_vlm_config" in test for test in parents)
+ only_moe = {"VLLM_MOE_ONLY_VLM": _module_constant("VLLM_MOE_ONLY_VLM"), "fast_inference": True}
+ assert _evaluate(gate, model_types = ["gemma4"], auto_config = GEMMA4_DENSE, **only_moe)
+ assert _evaluate(
+ gate,
+ model_types = ["gemma4_text", "gemma4"],
+ auto_config = GEMMA4_DENSE.text_config,
+ **only_moe,
+ )
+ assert not _evaluate(gate, model_types = ["gemma4"], auto_config = GEMMA4_MOE, **only_moe)
+ assert not _evaluate(
+ gate, model_types = ["gemma4_text"], auto_config = GEMMA4_MOE.text_config, **only_moe
+ )
+ assert not _evaluate(gate, model_types = ["qwen3_5"], auto_config = QWEN3_5_DENSE, **only_moe)
+ assert not _evaluate(
+ gate,
+ **{**only_moe, "fast_inference": False},
+ model_types = ["gemma4"],
+ auto_config = GEMMA4_DENSE,
+ )
+
+
+def test_the_4bit_refusal_uses_the_same_moe_test():
+ source = VISION_PATH.read_text(encoding = "utf-8")
+ assert "and _is_sparse_moe_config(model_config)" in source
+
+
+def _gate(message, condition = ""):
+ """The `if` raising `message` (test mentions `condition`) plus its enclosing `if` tests."""
+
+ def walk(node, parents):
+ for child in ast.iter_child_nodes(node):
+ if (
+ isinstance(child, ast.If)
+ and condition in ast.unparse(child.test)
+ and any(
+ isinstance(stmt, ast.Raise) and message in ast.unparse(stmt)
+ for stmt in child.body
+ )
+ ):
+ return child, parents
+ found = walk(
+ child, parents + [ast.unparse(child.test)] if isinstance(child, ast.If) else parents
+ )
+ if found:
+ return found
+ return None
+
+ found = walk(TREE, [])
+ assert found, message
+ return found
+
+
+def _evaluate(node, **names):
+ namespace = {
+ "VLLM_ZOO_MOE_VLM": _module_constant("VLLM_ZOO_MOE_VLM"),
+ "_is_sparse_moe_config": _is_sparse_moe_config(),
+ **names,
+ }
+ return eval(compile(ast.Expression(node.test), str(VISION_PATH), "eval"), namespace)
+
+
+def test_the_zoo_gate_covers_text_only_moe_loads():
+ """text_only = True sets is_vlm_config False, so the zoo gate must not sit under it."""
+ gate, parents = _gate("needs a newer unsloth_zoo", "_zoo_supports_moe_fast_inference")
+ assert not any("is_vlm_config" in test for test in parents)
+ old_zoo = lambda: False
+ for model_types, config in (
+ (["qwen3_5_moe"], QWEN3_5_MOE.text_config),
+ (["gemma4_text", "gemma4"], GEMMA4_MOE.text_config),
+ (["qwen3_5_moe"], QWEN3_5_MOE),
+ ):
+ refused = _evaluate(
+ gate,
+ fast_inference = True,
+ model_types = model_types,
+ auto_config = config,
+ _zoo_supports_moe_fast_inference = old_zoo,
+ )
+ assert refused, model_types
+ assert not _evaluate(
+ gate,
+ fast_inference = True,
+ model_types = ["gemma4_text"],
+ auto_config = GEMMA4_DENSE,
+ _zoo_supports_moe_fast_inference = old_zoo,
+ )
+ assert not _evaluate(
+ gate,
+ fast_inference = True,
+ model_types = ["gemma4"],
+ auto_config = GEMMA4_MOE,
+ _zoo_supports_moe_fast_inference = lambda: True,
+ )
+ assert not _evaluate(
+ gate,
+ fast_inference = False,
+ model_types = ["gemma4"],
+ auto_config = GEMMA4_MOE,
+ _zoo_supports_moe_fast_inference = old_zoo,
+ )
+
+
+def test_bnb_moe_loads_are_refused_without_load_in_4bit():
+ gate, _ = _gate("does not support bitsandbytes weights")
+
+ def quant_type(method):
+ return lambda config: method
+
+ def refused(
+ load_in_4bit,
+ model_name,
+ method,
+ config = GEMMA4_MOE,
+ model_types = ("gemma4",),
+ load_in_8bit = False,
+ ):
+ return bool(
+ _evaluate(
+ gate,
+ load_in_4bit = load_in_4bit,
+ load_in_8bit = load_in_8bit,
+ model_name = model_name,
+ get_quant_type = quant_type(method),
+ model_config = config,
+ model_types = list(model_types),
+ )
+ )
+
+ assert refused(True, "google/gemma-4-26B-A4B-it", None)
+ assert refused(False, "unsloth/gemma-4-26B-A4B-it-bnb-4bit", None)
+ assert refused(False, "someone/gemma-4-26b-moe-4bit", "bitsandbytes")
+ assert refused(False, "Qwen/Qwen3.6-35B-A3B", "bitsandbytes", QWEN3_5_MOE, ("qwen3_5_moe",))
+ # load_vllm has no 8-bit mode, so the experts would silently load in 16-bit.
+ assert refused(
+ False, "Qwen/Qwen3.6-35B-A3B", None, QWEN3_5_MOE, ("qwen3_5_moe",), load_in_8bit = True
+ )
+ assert not refused(False, "google/gemma-4-26B-A4B-it", None)
+ assert not refused(False, "Qwen/Qwen3.6-35B-A3B-FP8", "fp8", QWEN3_5_MOE, ("qwen3_5_moe",))
+ assert not refused(False, "unsloth/gemma-4-E2B-it-bnb-4bit", "bitsandbytes", GEMMA4_DENSE)
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 5d49ab4621d..a6cfffce87d 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -1086,6 +1086,10 @@ VLLM_SUPPORTED_VLM = [
# reaches this gate even for the text-only checkpoints.
"qwen3_5",
"idefics3",
+ # Exact-membership gate: "qwen3_5" does not match qwen3_5_moe.
+ "qwen3_5_moe",
+ "gemma4",
+ "gemma4_text",
]
@@ -1101,6 +1105,29 @@ def _zoo_supports_idefics3_fast_inference():
return False
+# Need an unsloth_zoo that rebuilds MoE blocks from vLLM.
+VLLM_ZOO_MOE_VLM = ("qwen3_5_moe", "gemma4", "gemma4_text")
+# Dense Gemma-4 shares the model type but aborts in vLLM's audio-encoder profiling, and in
+# 4-bit hits a bnb loader vLLM >= 0.28 moved out of tree. Only MoE checkpoints pass.
+VLLM_MOE_ONLY_VLM = ("gemma4", "gemma4_text")
+
+
+def _is_sparse_moe_config(config):
+ text_config = getattr(config, "text_config", None) or config
+ return bool(
+ getattr(text_config, "num_experts", None) or getattr(text_config, "enable_moe_block", False)
+ )
+
+
+def _zoo_supports_moe_fast_inference():
+ # Older unsloth_zoo releases leave every expert a 1-wide placeholder in the training model.
+ try:
+ from unsloth_zoo.empty_model import extract_moe_layers # noqa: F401
+ return True
+ except Exception:
+ return False
+
+
VLLM_NON_LORA_VLM = [
"mllama",
]
@@ -2908,6 +2935,27 @@ class FastBaseModel:
"Unsloth: Idefics3 fast_inference needs a newer unsloth_zoo. "
"Please run `pip install --upgrade unsloth_zoo`."
)
+ # Outside the VLM block: text_only = True has is_vlm_config False.
+ if (
+ fast_inference
+ and any(arch in VLLM_MOE_ONLY_VLM for arch in model_types)
+ and not _is_sparse_moe_config(auto_config)
+ ):
+ raise RuntimeError(
+ f"Unsloth: fast_inference = True is only supported for the MoE {model_type_arch} "
+ "checkpoints (such as gemma-4-26B-A4B), not the dense ones yet. "
+ "Please set fast_inference = False."
+ )
+ if (
+ fast_inference
+ and any(arch in VLLM_ZOO_MOE_VLM for arch in model_types)
+ and _is_sparse_moe_config(auto_config)
+ and not _zoo_supports_moe_fast_inference()
+ ):
+ raise RuntimeError(
+ f"Unsloth: {model_type_arch} fast_inference needs a newer unsloth_zoo. "
+ "Please run `pip install --upgrade unsloth_zoo`."
+ )
if any(arch in VLLM_NON_LORA_VLM for arch in model_types):
# mllama is still only in vllm v0, and vLLM V0 does not support LoRA on multimodal models. TODO: revisit once vLLM V1 supports Llama 3.2 (mllama).
@@ -3617,6 +3665,27 @@ class FastBaseModel:
load_in_16bit,
)
+ from unsloth_zoo.utils import get_quant_type
+
+ # Mirrors load_vllm's bnb loader test, so prequantized bnb-4bit is refused too.
+ if (
+ (
+ load_in_4bit
+ or load_in_8bit
+ or str(model_name).lower().endswith("-bnb-4bit")
+ or get_quant_type(model_config) == "bitsandbytes"
+ )
+ and any(arch in VLLM_ZOO_MOE_VLM for arch in model_types)
+ and _is_sparse_moe_config(model_config)
+ ):
+ raise NotImplementedError(
+ f"Unsloth: fast_inference = True does not support bitsandbytes weights (load_in_4bit / load_in_8bit = True "
+ "or a prequantized bnb-4bit checkpoint) for the sparse MoE "
+ f"model {model_type_arch}: vLLM's bitsandbytes MoE experts cannot be shared with the "
+ "training model, and vLLM does not serve LoRA on bitsandbytes MoE experts.\n"
+ "Load in 16-bit (load_in_4bit = False, load_in_8bit = False), or set fast_inference = False."
+ )
+
allowed_args = inspect.getfullargspec(load_vllm).args
load_vllm_kwargs = dict(
model_name = model_name,