Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fast_inference: Gemma 4 Unified, Muse Glimmer, dense Gemma 4; 4-bit fallback when no -bnb-4bit repo (#13825)

Daniel Handda71fde6357
diff --git a/tests/test_fast_inference_bnb_fallback.py b/tests/test_fast_inference_bnb_fallback.py
new file mode 100644
index 00000000000..07835e99667
--- /dev/null
+++ b/tests/test_fast_inference_bnb_fallback.py
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import ast
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+MODELS = Path(__file__).resolve().parents[1] / "unsloth" / "models"
+
+
+def _helper(monkeypatch):
+ package = types.ModuleType("_fi_models")
+ package.__path__ = [str(MODELS)]
+ spec = importlib.util.spec_from_file_location("_fi_models.mapper", MODELS / "mapper.py")
+ mapper = importlib.util.module_from_spec(spec)
+ monkeypatch.setitem(sys.modules, "_fi_models", package)
+ monkeypatch.setitem(sys.modules, "_fi_models.mapper", mapper)
+ spec.loader.exec_module(mapper)
+ tree = ast.parse((MODELS / "_utils.py").read_text(encoding = "utf-8"))
+ node = next(
+ n
+ for n in tree.body
+ if isinstance(n, ast.FunctionDef) and n.name == "_vllm_plain_bnb_or_16bit"
+ )
+ namespace = {"__package__": "_fi_models", "__name__": "_fi_models._utils"}
+ exec(compile(ast.Module(body = [node], type_ignores = []), "_utils.py", "exec"), namespace)
+ return namespace["_vllm_plain_bnb_or_16bit"]
+
+
+def test_published_plain_bnb_repo_is_kept(monkeypatch):
+ f = _helper(monkeypatch)
+ assert f("unsloth/gemma-3-12b-it-unsloth-bnb-4bit") == "unsloth/gemma-3-12b-it-bnb-4bit"
+ assert f("unsloth/Qwen3-0.6B-unsloth-bnb-4bit") == "unsloth/Qwen3-0.6B-bnb-4bit"
+
+
+def test_dynamic_only_models_fall_back_to_16bit(monkeypatch):
+ # No -bnb-4bit repo is published for these: the old rename hit a 404.
+ f = _helper(monkeypatch)
+ assert f("unsloth/Muse-Glimmer-30B-unsloth-bnb-4bit") == "unsloth/Muse-Glimmer-30B"
+ assert f("unsloth/gemma-4-E2B-it-unsloth-bnb-4bit") == "unsloth/gemma-4-E2B-it"
+ assert f("unsloth/Qwen3.8-27B-unsloth-bnb-4bit") == "unsloth/Qwen3.8-27B"
+
+
+def test_unknown_repo_keeps_the_old_rename(monkeypatch):
+ f = _helper(monkeypatch)
+ assert f("someone/foo-unsloth-bnb-4bit") == "someone/foo-bnb-4bit"
diff --git a/tests/test_moe_fast_inference_gate.py b/tests/test_moe_fast_inference_gate.py
index 1c33f4747eb..13da010a73f 100644
--- a/tests/test_moe_fast_inference_gate.py
+++ b/tests/test_moe_fast_inference_gate.py
@@ -56,34 +56,33 @@ def test_sparse_moe_is_read_off_the_text_config():
def test_a_config_without_text_config_is_read_directly():
is_moe = _is_sparse_moe_config()
assert is_moe(SimpleNamespace(num_experts = 128)) is True
+ assert is_moe(SimpleNamespace(num_local_experts = 8)) is True
+ assert is_moe(SimpleNamespace(n_routed_experts = 64)) is True
assert is_moe(SimpleNamespace(enable_moe_block = True)) is True
assert is_moe(SimpleNamespace()) is False
-def test_dense_gemma4_is_refused_at_the_gate():
- """Dense Gemma-4 shares the gemma4 model type, so the allowlist alone would admit it."""
- assert set(_module_constant("VLLM_MOE_ONLY_VLM")) == {"gemma4", "gemma4_text"}
- gate, parents = _gate("not the dense ones yet")
+def test_dense_gemma4_is_refused_only_with_an_old_zoo():
+ """Dense Gemma-4 shares the gemma4 model type; an old zoo leaves vLLM profiling its audio encoder."""
+ assert set(_module_constant("VLLM_DENSE_NEEDS_ZOO_VLM")) == {"gemma4", "gemma4_text"}
+ gate, parents = _gate("needs a newer unsloth_zoo", "_zoo_supports_dense_gemma4_fast_inference")
# text_only = True sets is_vlm_config False, so the gate must not sit under it.
assert not any("is_vlm_config" in test for test in parents)
- only_moe = {"VLLM_MOE_ONLY_VLM": _module_constant("VLLM_MOE_ONLY_VLM"), "fast_inference": True}
- assert _evaluate(gate, model_types = ["gemma4"], auto_config = GEMMA4_DENSE, **only_moe)
+ base = {
+ "VLLM_DENSE_NEEDS_ZOO_VLM": _module_constant("VLLM_DENSE_NEEDS_ZOO_VLM"),
+ "fast_inference": True,
+ }
+ old = {**base, "_zoo_supports_dense_gemma4_fast_inference": lambda: False}
+ new = {**base, "_zoo_supports_dense_gemma4_fast_inference": lambda: True}
+ assert _evaluate(gate, model_types = ["gemma4"], auto_config = GEMMA4_DENSE, **old)
assert _evaluate(
- gate,
- model_types = ["gemma4_text", "gemma4"],
- auto_config = GEMMA4_DENSE.text_config,
- **only_moe,
- )
- assert not _evaluate(gate, model_types = ["gemma4"], auto_config = GEMMA4_MOE, **only_moe)
- assert not _evaluate(
- gate, model_types = ["gemma4_text"], auto_config = GEMMA4_MOE.text_config, **only_moe
+ gate, model_types = ["gemma4_text", "gemma4"], auto_config = GEMMA4_DENSE.text_config, **old
)
- assert not _evaluate(gate, model_types = ["qwen3_5"], auto_config = QWEN3_5_DENSE, **only_moe)
+ assert not _evaluate(gate, model_types = ["gemma4"], auto_config = GEMMA4_DENSE, **new)
+ assert not _evaluate(gate, model_types = ["gemma4"], auto_config = GEMMA4_MOE, **old)
+ assert not _evaluate(gate, model_types = ["qwen3_5"], auto_config = QWEN3_5_DENSE, **old)
assert not _evaluate(
- gate,
- **{**only_moe, "fast_inference": False},
- model_types = ["gemma4"],
- auto_config = GEMMA4_DENSE,
+ gate, **{**old, "fast_inference": False}, model_types = ["gemma4"], auto_config = GEMMA4_DENSE
)
@@ -205,3 +204,116 @@ def test_bnb_moe_loads_are_refused_without_load_in_4bit():
assert not refused(False, "google/gemma-4-26B-A4B-it", None)
assert not refused(False, "Qwen/Qwen3.6-35B-A3B-FP8", "fp8", QWEN3_5_MOE, ("qwen3_5_moe",))
assert not refused(False, "unsloth/gemma-4-E2B-it-bnb-4bit", "bitsandbytes", GEMMA4_DENSE)
+
+
+def test_bnb_refusal_covers_text_sparse_moe():
+ """Qwen3-MoE 4-bit crashed in vLLM init ("output with shape [8, 1] ... [8, 768]"); vLLM serves no LoRA on bnb experts."""
+ gate, _ = _gate("does not support bitsandbytes weights")
+
+ def refused(
+ config,
+ model_types,
+ load_in_4bit = True,
+ method = None,
+ ):
+ return bool(
+ _evaluate(
+ gate,
+ load_in_4bit = load_in_4bit,
+ load_in_8bit = False,
+ model_name = "unsloth/Qwen3-30B-A3B-Instruct-2507",
+ get_quant_type = lambda config: method,
+ model_config = config,
+ model_types = list(model_types),
+ )
+ )
+
+ qwen3_moe = SimpleNamespace(num_experts = 128)
+ mixtral = SimpleNamespace(num_local_experts = 8)
+ deepseek = SimpleNamespace(n_routed_experts = 64)
+ for config, types in (
+ (qwen3_moe, ("qwen3_moe",)),
+ (mixtral, ("mixtral",)),
+ (deepseek, ("deepseek_v3",)),
+ ):
+ assert refused(config, types), types
+ assert not refused(config, types, load_in_4bit = False), types
+ assert refused(config, types, load_in_4bit = False, method = "bitsandbytes"), types
+ assert not refused(SimpleNamespace(num_experts = None), ("qwen3",))
+ assert not refused(SimpleNamespace(num_experts = 0), ("qwen3",))
+
+
+def _zoo_probe():
+ for node in TREE.body:
+ if (
+ isinstance(node, ast.FunctionDef)
+ and node.name == "_zoo_supports_dense_gemma4_fast_inference"
+ ):
+ namespace = {}
+ exec(
+ compile(ast.Module(body = [node], type_ignores = []), str(VISION_PATH), "exec"),
+ namespace,
+ )
+ return namespace["_zoo_supports_dense_gemma4_fast_inference"]
+ raise AssertionError("_zoo_supports_dense_gemma4_fast_inference not found")
+
+
+def test_dense_gemma4_zoo_probe_reads_the_clamp_bound_templates(monkeypatch):
+ import sys
+ import types
+
+ probe = _zoo_probe()
+ bound = "model.vision_tower.encoder.layers.{kk}.self_attn.q_proj.input_min"
+
+ def install(vision_layers):
+ module = types.ModuleType("unsloth_zoo.empty_model")
+ if vision_layers is not None:
+ module.get_model_layer_config = lambda: {"vision_layers": set(vision_layers)}
+ package = sys.modules.get("unsloth_zoo") or types.ModuleType("unsloth_zoo")
+ monkeypatch.setitem(sys.modules, "unsloth_zoo", package)
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.empty_model", module)
+ monkeypatch.setattr(package, "empty_model", module, raising = False)
+
+ install({bound, "model.vision_tower.encoder.layers.{kk}.self_attn.q_proj"})
+ assert probe() is True
+ install({"model.vision_tower.encoder.layers.{kk}.self_attn.q_proj"})
+ assert probe() is False
+ install(None)
+ assert probe() is False
+
+
+def _new_vlm_probe():
+ for node in TREE.body:
+ if (
+ isinstance(node, ast.FunctionDef)
+ and node.name == "_zoo_supports_new_vlm_fast_inference"
+ ):
+ namespace = {}
+ exec(
+ compile(ast.Module(body = [node], type_ignores = []), str(VISION_PATH), "exec"),
+ namespace,
+ )
+ return namespace["_zoo_supports_new_vlm_fast_inference"]
+ raise AssertionError("_zoo_supports_new_vlm_fast_inference not found")
+
+
+def test_new_vlm_zoo_gate_only_probes_unified_and_muse(monkeypatch):
+ import sys
+ import types
+
+ probe = _new_vlm_probe()
+ old_zoo = types.ModuleType("unsloth_zoo.empty_model") # neither probed helper exists
+ package = sys.modules.get("unsloth_zoo") or types.ModuleType("unsloth_zoo")
+ monkeypatch.setitem(sys.modules, "unsloth_zoo", package)
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.empty_model", old_zoo)
+ monkeypatch.setattr(package, "empty_model", old_zoo, raising = False)
+ # An older zoo must not refuse models it already rebuilds.
+ for model_types in (["qwen2_5_vl"], ["gemma4"], ["qwen3_5_moe"], ["llama"]):
+ assert probe(model_types) is True, model_types
+ for model_types in (["gemma4_unified"], ["gemma4_unified_text"], ["muse_glimmer_text"]):
+ assert probe(model_types) is False, model_types
+
+
+def test_new_vlm_zoo_gate_covers_text_only_loads():
+ gate, parents = _gate("needs a newer unsloth_zoo", "_zoo_supports_new_vlm_fast_inference")
+ assert not any("is_vlm_config" in test for test in parents)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index fad3738e871..b3ad0d2021e 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -7345,10 +7345,22 @@ def fast_inference_setup(model_name, model_config):
f"Unsloth: Switching from Unsloth dynamic quant to normal quant since\n"
f"we do not yet support fast inference for {model_name}"
)
- model_name = model_name[: -len("unsloth-bnb-4bit")] + "bnb-4bit"
+ model_name = _vllm_plain_bnb_or_16bit(model_name)
return fast_inference, model_name
+def _vllm_plain_bnb_or_16bit(model_name):
+ """`-unsloth-bnb-4bit` -> `-bnb-4bit` when that repo is published, else the 16-bit original (quantized on load)."""
+ from .mapper import FLOAT_TO_INT_MAPPER, INT_TO_FLOAT_MAPPER
+
+ plain = model_name[: -len("unsloth-bnb-4bit")] + "bnb-4bit"
+ lower = model_name.lower()
+ key = lower if lower in INT_TO_FLOAT_MAPPER else FLOAT_TO_INT_MAPPER.get(lower)
+ if key is None or (FLOAT_TO_INT_MAPPER.get(plain.lower()) or "").lower() == key.lower():
+ return plain
+ return INT_TO_FLOAT_MAPPER.get(key) or plain
+
+
def save_lora_adapter(model, save_directory, *args, **kwargs):
"""`save_pretrained` over the adapter, cast to the embedding dtype. PEFT's own selection decides what an adapter contains, so it is handed the whole state dict and only the adapter tensors are cast: filtering down to `.lora_A.`/`.lora_B.` first, as the Zoo helper does, makes PEFT raise `KeyError` looking up `modules_to_save.<adapter>.weight`, and a DoRA run loses its `lora_magnitude_vector` the same way. Both are reachable without vLLM, since `get_peft_model` adds `embed_tokens` and `lm_head` to `modules_to_save` once new tokens are trained. Non-adapter entries pass through by reference."""
from .compressed_tensors_int4 import adapter_only_state_dict
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 43163a3d95b..f45d197525c 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -1143,6 +1143,10 @@ VLLM_SUPPORTED_VLM = [
"qwen3_5_moe",
"gemma4",
"gemma4_text",
+ "gemma4_unified",
+ "gemma4_unified_text",
+ "muse_glimmer",
+ "muse_glimmer_text",
]
@@ -1158,17 +1162,53 @@ def _zoo_supports_idefics3_fast_inference():
return False
+def _zoo_supports_new_vlm_fast_inference(model_types):
+ """Only Gemma-4 Unified / Muse Glimmer loads probe the zoo; every other model passes."""
+ unified = any(t in ("gemma4_unified", "gemma4_unified_text") for t in model_types)
+ muse = any(t in ("muse_glimmer", "muse_glimmer_text") for t in model_types)
+ if not (unified or muse):
+ return True
+ try:
+ from types import SimpleNamespace
+ from unsloth_zoo.empty_model import _is_gemma4_config, get_model_layer_config
+
+ if unified and not _is_gemma4_config(SimpleNamespace(model_type = "gemma4_unified")):
+ return False
+ if muse and (
+ "model.vision_tower.layers.{kk}.attn.proj"
+ not in get_model_layer_config()["vision_layers"]
+ ):
+ return False
+ return True
+ except Exception:
+ return False
+
+
# Need an unsloth_zoo that rebuilds MoE blocks from vLLM.
VLLM_ZOO_MOE_VLM = ("qwen3_5_moe", "gemma4", "gemma4_text")
-# Dense Gemma-4 shares the model type but aborts in vLLM's audio-encoder profiling, and in
-# 4-bit hits a bnb loader vLLM >= 0.28 moved out of tree. Only MoE checkpoints pass.
-VLLM_MOE_ONLY_VLM = ("gemma4", "gemma4_text")
+# Dense Gemma-4 (E2B / E4B) needs a zoo that turns vLLM's audio profiling off (audio limit 0).
+VLLM_DENSE_NEEDS_ZOO_VLM = ("gemma4", "gemma4_text")
+
+
+def _zoo_supports_dense_gemma4_fast_inference():
+ # Older unsloth_zoo leaves the clipped vision linears of dense Gemma-4 clamping at +-inf.
+ try:
+ from unsloth_zoo.empty_model import get_model_layer_config
+ return (
+ "model.vision_tower.encoder.layers.{kk}.self_attn.q_proj.input_min"
+ in get_model_layer_config()["vision_layers"]
+ )
+ except Exception:
+ return False
def _is_sparse_moe_config(config):
text_config = getattr(config, "text_config", None) or config
return bool(
- getattr(text_config, "num_experts", None) or getattr(text_config, "enable_moe_block", False)
+ getattr(text_config, "num_experts", None)
+ or getattr(text_config, "num_local_experts", None)
+ or getattr(text_config, "n_routed_experts", None)
+ or getattr(text_config, "enable_moe_block", False)
)
@@ -3168,15 +3208,20 @@ class FastBaseModel:
"Please run `pip install --upgrade unsloth_zoo`."
)
# Outside the VLM block: text_only = True has is_vlm_config False.
+ if fast_inference and not _zoo_supports_new_vlm_fast_inference(model_types):
+ raise RuntimeError(
+ f"Unsloth: {model_type_arch} fast_inference needs a newer unsloth_zoo. "
+ "Please run `pip install --upgrade unsloth_zoo`."
+ )
if (
fast_inference
- and any(arch in VLLM_MOE_ONLY_VLM for arch in model_types)
+ and any(arch in VLLM_DENSE_NEEDS_ZOO_VLM for arch in model_types)
and not _is_sparse_moe_config(auto_config)
+ and not _zoo_supports_dense_gemma4_fast_inference()
):
raise RuntimeError(
- f"Unsloth: fast_inference = True is only supported for the MoE {model_type_arch} "
- "checkpoints (such as gemma-4-26B-A4B), not the dense ones yet. "
- "Please set fast_inference = False."
+ f"Unsloth: dense {model_type_arch} fast_inference needs a newer unsloth_zoo. "
+ "Please run `pip install --upgrade unsloth_zoo`, or set fast_inference = False."
)
if (
fast_inference
@@ -3928,16 +3973,14 @@ class FastBaseModel:
from unsloth_zoo.utils import get_quant_type
# Mirrors load_vllm's bnb loader test, so prequantized bnb-4bit is refused too.
+ # Every sparse MoE, text ones too: vLLM's MoE LoRA layer rejects the bnb MoE
+ # method ("Old-style modular kernel not supported", vLLM 0.31 lora/layers/fused_moe.py).
if (
- (
- load_in_4bit
- or load_in_8bit
- or str(model_name).lower().endswith("-bnb-4bit")
- or get_quant_type(model_config) == "bitsandbytes"
- )
- and any(arch in VLLM_ZOO_MOE_VLM for arch in model_types)
- and _is_sparse_moe_config(model_config)
- ):
+ load_in_4bit
+ or load_in_8bit
+ or str(model_name).lower().endswith("-bnb-4bit")
+ or get_quant_type(model_config) == "bitsandbytes"
+ ) and _is_sparse_moe_config(model_config):
raise NotImplementedError(
f"Unsloth: fast_inference = True does not support bitsandbytes weights (load_in_4bit / load_in_8bit = True "
"or a prequantized bnb-4bit checkpoint) for the sparse MoE "