Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
import ast
from pathlib import Path
from types import SimpleNamespace
VISION_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "vision.py"
TREE = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))
def _module_constant(name):
for node in TREE.body:
if isinstance(node, ast.Assign) and any(
isinstance(t, ast.Name) and t.id == name for t in node.targets
):
return ast.literal_eval(node.value)
raise AssertionError(f"{name} not found")
def _is_sparse_moe_config():
for node in TREE.body:
if isinstance(node, ast.FunctionDef) and node.name == "_is_sparse_moe_config":
namespace = {}
exec(
compile(ast.Module(body = [node], type_ignores = []), str(VISION_PATH), "exec"),
namespace,
)
return namespace["_is_sparse_moe_config"]
raise AssertionError("_is_sparse_moe_config not found")
GEMMA4_MOE = SimpleNamespace(text_config = SimpleNamespace(num_experts = 128, enable_moe_block = True))
GEMMA4_DENSE = SimpleNamespace(
text_config = SimpleNamespace(num_experts = None, enable_moe_block = False)
)
QWEN3_5_MOE = SimpleNamespace(text_config = SimpleNamespace(num_experts = 256))
QWEN3_5_DENSE = SimpleNamespace(text_config = SimpleNamespace(num_experts = None))
def test_moe_model_types_pass_the_allowlist():
supported = _module_constant("VLLM_SUPPORTED_VLM")
for arch in ("qwen3_5_moe", "gemma4", "gemma4_text"):
assert arch in supported
assert len(supported) == len(set(supported))
def test_sparse_moe_is_read_off_the_text_config():
is_moe = _is_sparse_moe_config()
assert is_moe(GEMMA4_MOE) is True
assert is_moe(QWEN3_5_MOE) is True
assert is_moe(GEMMA4_DENSE) is False
assert is_moe(QWEN3_5_DENSE) is False
def test_a_config_without_text_config_is_read_directly():
is_moe = _is_sparse_moe_config()
assert is_moe(SimpleNamespace(num_experts = 128)) is True
assert is_moe(SimpleNamespace(num_local_experts = 8)) is True
assert is_moe(SimpleNamespace(n_routed_experts = 64)) is True
assert is_moe(SimpleNamespace(enable_moe_block = True)) is True
assert is_moe(SimpleNamespace()) is False
def test_dense_gemma4_is_refused_only_with_an_old_zoo():
"""Dense Gemma-4 shares the gemma4 model type; an old zoo leaves vLLM profiling its audio encoder."""
assert set(_module_constant("VLLM_DENSE_NEEDS_ZOO_VLM")) == {"gemma4", "gemma4_text"}
gate, parents = _gate("needs a newer unsloth_zoo", "_zoo_supports_dense_gemma4_fast_inference")
# text_only = True sets is_vlm_config False, so the gate must not sit under it.
assert not any("is_vlm_config" in test for test in parents)
base = {
"VLLM_DENSE_NEEDS_ZOO_VLM": _module_constant("VLLM_DENSE_NEEDS_ZOO_VLM"),
"fast_inference": True,
}
old = {**base, "_zoo_supports_dense_gemma4_fast_inference": lambda: False}
new = {**base, "_zoo_supports_dense_gemma4_fast_inference": lambda: True}
assert _evaluate(gate, model_types = ["gemma4"], auto_config = GEMMA4_DENSE, **old)
assert _evaluate(
gate, model_types = ["gemma4_text", "gemma4"], auto_config = GEMMA4_DENSE.text_config, **old
)
assert not _evaluate(gate, model_types = ["gemma4"], auto_config = GEMMA4_DENSE, **new)
assert not _evaluate(gate, model_types = ["gemma4"], auto_config = GEMMA4_MOE, **old)
assert not _evaluate(gate, model_types = ["qwen3_5"], auto_config = QWEN3_5_DENSE, **old)
assert not _evaluate(
gate, **{**old, "fast_inference": False}, model_types = ["gemma4"], auto_config = GEMMA4_DENSE
)
def test_the_4bit_refusal_uses_the_same_moe_test():
source = VISION_PATH.read_text(encoding = "utf-8")
assert "and _is_sparse_moe_config(model_config)" in source
def _gate(message, condition = ""):
"""The `if` raising `message` (test mentions `condition`) plus its enclosing `if` tests."""
def walk(node, parents):
for child in ast.iter_child_nodes(node):
if (
isinstance(child, ast.If)
and condition in ast.unparse(child.test)
and any(
isinstance(stmt, ast.Raise) and message in ast.unparse(stmt)
for stmt in child.body
)
):
return child, parents
found = walk(
child, parents + [ast.unparse(child.test)] if isinstance(child, ast.If) else parents
)
if found:
return found
return None
found = walk(TREE, [])
assert found, message
return found
def _evaluate(node, **names):
namespace = {
"VLLM_ZOO_MOE_VLM": _module_constant("VLLM_ZOO_MOE_VLM"),
"_is_sparse_moe_config": _is_sparse_moe_config(),
**names,
}
return eval(compile(ast.Expression(node.test), str(VISION_PATH), "eval"), namespace)
def test_the_zoo_gate_covers_text_only_moe_loads():
"""text_only = True sets is_vlm_config False, so the zoo gate must not sit under it."""
gate, parents = _gate("needs a newer unsloth_zoo", "_zoo_supports_moe_fast_inference")
assert not any("is_vlm_config" in test for test in parents)
old_zoo = lambda: False
for model_types, config in (
(["qwen3_5_moe"], QWEN3_5_MOE.text_config),
(["gemma4_text", "gemma4"], GEMMA4_MOE.text_config),
(["qwen3_5_moe"], QWEN3_5_MOE),
):
refused = _evaluate(
gate,
fast_inference = True,
model_types = model_types,
auto_config = config,
_zoo_supports_moe_fast_inference = old_zoo,
)
assert refused, model_types
assert not _evaluate(
gate,
fast_inference = True,
model_types = ["gemma4_text"],
auto_config = GEMMA4_DENSE,
_zoo_supports_moe_fast_inference = old_zoo,
)
assert not _evaluate(
gate,
fast_inference = True,
model_types = ["gemma4"],
auto_config = GEMMA4_MOE,
_zoo_supports_moe_fast_inference = lambda: True,
)
assert not _evaluate(
gate,
fast_inference = False,
model_types = ["gemma4"],
auto_config = GEMMA4_MOE,
_zoo_supports_moe_fast_inference = old_zoo,
)
def test_bnb_moe_loads_are_refused_without_load_in_4bit():
gate, _ = _gate("does not support bitsandbytes weights")
def quant_type(method):
return lambda config: method
def refused(
load_in_4bit,
model_name,
method,
config = GEMMA4_MOE,
model_types = ("gemma4",),
load_in_8bit = False,
):
return bool(
_evaluate(
gate,
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
model_name = model_name,
get_quant_type = quant_type(method),
model_config = config,
model_types = list(model_types),
)
)
assert refused(True, "google/gemma-4-26B-A4B-it", None)
assert refused(False, "unsloth/gemma-4-26B-A4B-it-bnb-4bit", None)
assert refused(False, "someone/gemma-4-26b-moe-4bit", "bitsandbytes")
assert refused(False, "Qwen/Qwen3.6-35B-A3B", "bitsandbytes", QWEN3_5_MOE, ("qwen3_5_moe",))
# load_vllm has no 8-bit mode, so the experts would silently load in 16-bit.
assert refused(
False, "Qwen/Qwen3.6-35B-A3B", None, QWEN3_5_MOE, ("qwen3_5_moe",), load_in_8bit = True
)
assert not refused(False, "google/gemma-4-26B-A4B-it", None)
assert not refused(False, "Qwen/Qwen3.6-35B-A3B-FP8", "fp8", QWEN3_5_MOE, ("qwen3_5_moe",))
assert not refused(False, "unsloth/gemma-4-E2B-it-bnb-4bit", "bitsandbytes", GEMMA4_DENSE)
def test_bnb_refusal_covers_text_sparse_moe():
"""Qwen3-MoE 4-bit crashed in vLLM init ("output with shape [8, 1] ... [8, 768]"); vLLM serves no LoRA on bnb experts."""
gate, _ = _gate("does not support bitsandbytes weights")
def refused(
config,
model_types,
load_in_4bit = True,
method = None,
):
return bool(
_evaluate(
gate,
load_in_4bit = load_in_4bit,
load_in_8bit = False,
model_name = "unsloth/Qwen3-30B-A3B-Instruct-2507",
get_quant_type = lambda config: method,
model_config = config,
model_types = list(model_types),
)
)
qwen3_moe = SimpleNamespace(num_experts = 128)
mixtral = SimpleNamespace(num_local_experts = 8)
deepseek = SimpleNamespace(n_routed_experts = 64)
for config, types in (
(qwen3_moe, ("qwen3_moe",)),
(mixtral, ("mixtral",)),
(deepseek, ("deepseek_v3",)),
):
assert refused(config, types), types
assert not refused(config, types, load_in_4bit = False), types
assert refused(config, types, load_in_4bit = False, method = "bitsandbytes"), types
assert not refused(SimpleNamespace(num_experts = None), ("qwen3",))
assert not refused(SimpleNamespace(num_experts = 0), ("qwen3",))
def _zoo_probe():
for node in TREE.body:
if (
isinstance(node, ast.FunctionDef)
and node.name == "_zoo_supports_dense_gemma4_fast_inference"
):
namespace = {}
exec(
compile(ast.Module(body = [node], type_ignores = []), str(VISION_PATH), "exec"),
namespace,
)
return namespace["_zoo_supports_dense_gemma4_fast_inference"]
raise AssertionError("_zoo_supports_dense_gemma4_fast_inference not found")
def test_dense_gemma4_zoo_probe_reads_the_clamp_bound_templates(monkeypatch):
import sys
import types
probe = _zoo_probe()
bound = "model.vision_tower.encoder.layers.{kk}.self_attn.q_proj.input_min"
def install(vision_layers):
module = types.ModuleType("unsloth_zoo.empty_model")
if vision_layers is not None:
module.get_model_layer_config = lambda: {"vision_layers": set(vision_layers)}
package = sys.modules.get("unsloth_zoo") or types.ModuleType("unsloth_zoo")
monkeypatch.setitem(sys.modules, "unsloth_zoo", package)
monkeypatch.setitem(sys.modules, "unsloth_zoo.empty_model", module)
monkeypatch.setattr(package, "empty_model", module, raising = False)
install({bound, "model.vision_tower.encoder.layers.{kk}.self_attn.q_proj"})
assert probe() is True
install({"model.vision_tower.encoder.layers.{kk}.self_attn.q_proj"})
assert probe() is False
install(None)
assert probe() is False
def _new_vlm_probe():
for node in TREE.body:
if (
isinstance(node, ast.FunctionDef)
and node.name == "_zoo_supports_new_vlm_fast_inference"
):
namespace = {}
exec(
compile(ast.Module(body = [node], type_ignores = []), str(VISION_PATH), "exec"),
namespace,
)
return namespace["_zoo_supports_new_vlm_fast_inference"]
raise AssertionError("_zoo_supports_new_vlm_fast_inference not found")
def test_new_vlm_zoo_gate_only_probes_unified_and_muse(monkeypatch):
import sys
import types
probe = _new_vlm_probe()
old_zoo = types.ModuleType("unsloth_zoo.empty_model") # neither probed helper exists
package = sys.modules.get("unsloth_zoo") or types.ModuleType("unsloth_zoo")
monkeypatch.setitem(sys.modules, "unsloth_zoo", package)
monkeypatch.setitem(sys.modules, "unsloth_zoo.empty_model", old_zoo)
monkeypatch.setattr(package, "empty_model", old_zoo, raising = False)
# An older zoo must not refuse models it already rebuilds.
for model_types in (["qwen2_5_vl"], ["gemma4"], ["qwen3_5_moe"], ["llama"]):
assert probe(model_types) is True, model_types
for model_types in (["gemma4_unified"], ["gemma4_unified_text"], ["muse_glimmer_text"]):
assert probe(model_types) is False, model_types
def test_new_vlm_zoo_gate_covers_text_only_loads():
gate, parents = _gate("needs a newer unsloth_zoo", "_zoo_supports_new_vlm_fast_inference")
assert not any("is_vlm_config" in test for test in parents)