Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep opted-in bnb 4-bit MoE experts outside the compiled block (#13946)
Daniel Hancb7a0473644d
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pyindex 63d3b93562e..fbf51a15862 100644--- a/tests/test_block_compile.py+++ b/tests/test_block_compile.py@@ -713,3 +713,88 @@ def test_moe_experts_run_outside_the_block(monkeypatch):third.experts = LoraWrapper()M._experts_outside(third)assert not getattr(third.experts, "_unsloth_block_outside", False)+++def test_selective_dequant_keeps_bnb4bit_experts_outside_the_block(monkeypatch):+ # The zoo plan reads token counts on the host and declines in a trace; opted-in experts stay eager.+ M = _load()++ class Params4bit(torch.nn.Parameter):+ pass++ class ToyExperts(torch.nn.Module):+ def __init__(self, quantized):+ super().__init__()+ weight = torch.zeros(2, 4, 4)+ self.gate_up_proj = (+ Params4bit(weight, requires_grad = False) if quantized else torch.nn.Parameter(weight)+ )++ def forward(self, x):+ return x * 2++ def layer(quantized):+ out = torch.nn.Module()+ out.mlp = torch.nn.Module()+ out.mlp.experts = ToyExperts(quantized)+ return out++ monkeypatch.setattr(M, "_lora_would_fold", lambda experts: False)+ for key in (+ "UNSLOTH_BLOCK_MOE",+ "UNSLOTH_MOE_SELECTIVE_DEQUANT",+ "UNSLOTH_MOE_DEQUANT_CHUNK_EXPERTS",+ ):+ monkeypatch.delenv(key, raising = False)+ default = layer(True)+ M._experts_outside(default)+ assert not getattr(default.mlp.experts, "_unsloth_block_outside", False)+ for key, value in (+ ("UNSLOTH_MOE_SELECTIVE_DEQUANT", "1"),+ ("UNSLOTH_MOE_DEQUANT_CHUNK_EXPERTS", "32"),+ ):+ monkeypatch.setenv(key, value)+ quantized, plain = layer(True), layer(False)+ M._experts_outside(quantized)+ M._experts_outside(plain)+ assert quantized.mlp.experts._unsloth_block_outside+ assert not getattr(plain.mlp.experts, "_unsloth_block_outside", False)+ monkeypatch.delenv(key)+ monkeypatch.setenv("UNSLOTH_MOE_DEQUANT_CHUNK_EXPERTS", "abc")+ bad = layer(True)+ M._experts_outside(bad)+ assert not getattr(bad.mlp.experts, "_unsloth_block_outside", False)+++def test_selective_dequant_off_never_reads_a_packed_expert_stack(monkeypatch):+ # MXFP4 experts expose gate_up_proj as a property that dequantizes the whole stack.+ M = _load()+ reads = []++ class PackedExperts(torch.nn.Module):+ @property+ def gate_up_proj(self):+ reads.append(1)+ return torch.zeros(1)++ def forward(self, x):+ return x++ monkeypatch.setattr(M, "_lora_would_fold", lambda experts: False)+ for key in (+ "UNSLOTH_BLOCK_MOE",+ "UNSLOTH_MOE_SELECTIVE_DEQUANT",+ "UNSLOTH_MOE_DEQUANT_CHUNK_EXPERTS",+ ):+ monkeypatch.delenv(key, raising = False)+ layer = torch.nn.Module()+ layer.mlp = torch.nn.Module()+ layer.mlp.experts = PackedExperts()+ M._experts_outside(layer)+ assert reads == []+ # Opted in, a packed stack is still not a bnb 4-bit one: never read, never moved out.+ monkeypatch.setenv("UNSLOTH_MOE_SELECTIVE_DEQUANT", "1")+ layer.__dict__.pop("_unsloth_block_moe_done", None)+ M._experts_outside(layer)+ assert reads == []+ assert not getattr(layer.mlp.experts, "_unsloth_block_outside", False)diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex bbbbab30d3c..297317d9d26 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -712,6 +712,19 @@ def _lora_would_fold(experts):return reads is False+def _selective_dequant_requested(experts):+ # zoo's selective / chunked bnb 4-bit expert dequant plans from host-read token counts, so it+ # declines inside a trace: experts it should run on must stay out of the compiled block.+ # Never getattr: on packed experts (MXFP4) gate_up_proj is a property that dequantizes the stack.+ try:+ chunked = int(os.environ.get("UNSLOTH_MOE_DEQUANT_CHUNK_EXPERTS", "0")) > 0+ except ValueError:+ chunked = False+ if os.environ.get("UNSLOTH_MOE_SELECTIVE_DEQUANT", "0") != "1" and not chunked:+ return False+ return type(experts._parameters.get("gate_up_proj")).__name__ == "Params4bit"++def _experts_outside(layer):# At the layer's first call, after get_peft_model wrapped the experts.layer.__dict__["_unsloth_block_moe_done"] = True@@ -724,7 +737,11 @@ def _experts_outside(layer):experts = _experts_of(module)if experts is None: # named_modules() is pre-order: the outermost wrapper comes firstcontinue- if mode != "outside" and not _lora_would_fold(experts):+ if (+ mode != "outside"+ and not _lora_would_fold(experts)+ and not _selective_dequant_requested(experts)+ ):continuemodule.forward = torch.compiler.disable(module.forward)for inner in module.modules():