Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep opted-in bnb 4-bit MoE experts outside the compiled block (#13946)

Daniel Hancb7a0473644d
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
index 63d3b93562e..fbf51a15862 100644
--- a/tests/test_block_compile.py
+++ b/tests/test_block_compile.py
@@ -713,3 +713,88 @@ def test_moe_experts_run_outside_the_block(monkeypatch):
third.experts = LoraWrapper()
M._experts_outside(third)
assert not getattr(third.experts, "_unsloth_block_outside", False)
+
+
+def test_selective_dequant_keeps_bnb4bit_experts_outside_the_block(monkeypatch):
+ # The zoo plan reads token counts on the host and declines in a trace; opted-in experts stay eager.
+ M = _load()
+
+ class Params4bit(torch.nn.Parameter):
+ pass
+
+ class ToyExperts(torch.nn.Module):
+ def __init__(self, quantized):
+ super().__init__()
+ weight = torch.zeros(2, 4, 4)
+ self.gate_up_proj = (
+ Params4bit(weight, requires_grad = False) if quantized else torch.nn.Parameter(weight)
+ )
+
+ def forward(self, x):
+ return x * 2
+
+ def layer(quantized):
+ out = torch.nn.Module()
+ out.mlp = torch.nn.Module()
+ out.mlp.experts = ToyExperts(quantized)
+ return out
+
+ monkeypatch.setattr(M, "_lora_would_fold", lambda experts: False)
+ for key in (
+ "UNSLOTH_BLOCK_MOE",
+ "UNSLOTH_MOE_SELECTIVE_DEQUANT",
+ "UNSLOTH_MOE_DEQUANT_CHUNK_EXPERTS",
+ ):
+ monkeypatch.delenv(key, raising = False)
+ default = layer(True)
+ M._experts_outside(default)
+ assert not getattr(default.mlp.experts, "_unsloth_block_outside", False)
+ for key, value in (
+ ("UNSLOTH_MOE_SELECTIVE_DEQUANT", "1"),
+ ("UNSLOTH_MOE_DEQUANT_CHUNK_EXPERTS", "32"),
+ ):
+ monkeypatch.setenv(key, value)
+ quantized, plain = layer(True), layer(False)
+ M._experts_outside(quantized)
+ M._experts_outside(plain)
+ assert quantized.mlp.experts._unsloth_block_outside
+ assert not getattr(plain.mlp.experts, "_unsloth_block_outside", False)
+ monkeypatch.delenv(key)
+ monkeypatch.setenv("UNSLOTH_MOE_DEQUANT_CHUNK_EXPERTS", "abc")
+ bad = layer(True)
+ M._experts_outside(bad)
+ assert not getattr(bad.mlp.experts, "_unsloth_block_outside", False)
+
+
+def test_selective_dequant_off_never_reads_a_packed_expert_stack(monkeypatch):
+ # MXFP4 experts expose gate_up_proj as a property that dequantizes the whole stack.
+ M = _load()
+ reads = []
+
+ class PackedExperts(torch.nn.Module):
+ @property
+ def gate_up_proj(self):
+ reads.append(1)
+ return torch.zeros(1)
+
+ def forward(self, x):
+ return x
+
+ monkeypatch.setattr(M, "_lora_would_fold", lambda experts: False)
+ for key in (
+ "UNSLOTH_BLOCK_MOE",
+ "UNSLOTH_MOE_SELECTIVE_DEQUANT",
+ "UNSLOTH_MOE_DEQUANT_CHUNK_EXPERTS",
+ ):
+ monkeypatch.delenv(key, raising = False)
+ layer = torch.nn.Module()
+ layer.mlp = torch.nn.Module()
+ layer.mlp.experts = PackedExperts()
+ M._experts_outside(layer)
+ assert reads == []
+ # Opted in, a packed stack is still not a bnb 4-bit one: never read, never moved out.
+ monkeypatch.setenv("UNSLOTH_MOE_SELECTIVE_DEQUANT", "1")
+ layer.__dict__.pop("_unsloth_block_moe_done", None)
+ M._experts_outside(layer)
+ assert reads == []
+ assert not getattr(layer.mlp.experts, "_unsloth_block_outside", False)
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index bbbbab30d3c..297317d9d26 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -712,6 +712,19 @@ def _lora_would_fold(experts):
return reads is False
+def _selective_dequant_requested(experts):
+ # zoo's selective / chunked bnb 4-bit expert dequant plans from host-read token counts, so it
+ # declines inside a trace: experts it should run on must stay out of the compiled block.
+ # Never getattr: on packed experts (MXFP4) gate_up_proj is a property that dequantizes the stack.
+ try:
+ chunked = int(os.environ.get("UNSLOTH_MOE_DEQUANT_CHUNK_EXPERTS", "0")) > 0
+ except ValueError:
+ chunked = False
+ if os.environ.get("UNSLOTH_MOE_SELECTIVE_DEQUANT", "0") != "1" and not chunked:
+ return False
+ return type(experts._parameters.get("gate_up_proj")).__name__ == "Params4bit"
+
+
def _experts_outside(layer):
# At the layer's first call, after get_peft_model wrapped the experts.
layer.__dict__["_unsloth_block_moe_done"] = True
@@ -724,7 +737,11 @@ def _experts_outside(layer):
experts = _experts_of(module)
if experts is None: # named_modules() is pre-order: the outermost wrapper comes first
continue
- if mode != "outside" and not _lora_would_fold(experts):
+ if (
+ mode != "outside"
+ and not _lora_would_fold(experts)
+ and not _selective_dequant_requested(experts)
+ ):
continue
module.forward = torch.compiler.disable(module.forward)
for inner in module.modules():