Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

NF4 bitsandbytes GEMM gate: return before its lazy setup under torch.compile (#14608)

Daniel Han356721b31ebe
diff --git a/tests/test_bnb_integration_compile.py b/tests/test_bnb_integration_compile.py
index 9637454d43a..542f02a067a 100644
--- a/tests/test_bnb_integration_compile.py
+++ b/tests/test_bnb_integration_compile.py
@@ -477,6 +477,23 @@ def test_fast_linear_forward_compiles_fullgraph(nf4_kernels, bsz):
_assert_compiled_matches(compiled, eager, backend = "aot_eager")
+# No eager warmup: the GEMM gate's first call must not run its lazy setup while tracing.
[email protected]("bsz", [8, 16])
+def test_fast_linear_forward_cold_compile_fullgraph(nf4_kernels, monkeypatch, bsz):
+ monkeypatch.setattr(U, "_BNB_GEMM_4BIT", None)
+ monkeypatch.setattr(U, "_BNB_GEMM_4BIT_DEVICE", {})
+ monkeypatch.setattr(U, "_BNB_GEMM_4BIT_ARCHS", {torch.cuda.get_device_capability()})
+ _, block = _lora_block(dtype = torch.float16)
+ fn = lambda x: U.fast_linear_forward(block.o_proj, x)
+ X = torch.randn(bsz, 1, 256, dtype = torch.float16, device = DEVICE)
+ torch._dynamo.reset()
+ with torch.no_grad():
+ compiled = torch.compile(fn, fullgraph = _dynamo_traces_params4bit(), backend = "aot_eager")(X)
+ eager = fn(X)
+ torch._dynamo.reset()
+ _assert_compiled_matches(compiled, eager, backend = "aot_eager")
+
+
def test_eager_gemv_gate_off_takes_bitsandbytes(monkeypatch):
"""Where the Triton GEMV is not the eager choice (HIP before Triton 3.6, or
UNSLOTH_TRITON_GEMV_EAGER=0), eager decode takes bitsandbytes' GEMV, on the live stream.
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index bebeb09eec1..6d36ce8cf02 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -1521,6 +1521,9 @@ def _bnb_gemm_4bit_device_ok(device):
def _bnb_gemm_4bit_ok(X, W, quant_state):
global _BNB_GEMM_4BIT, _BNB_GEMM_4BIT_DTYPES
+ # Before the lazy setup: dynamo cannot trace torch.cuda.device_count().
+ if _is_compiling() or torch.is_grad_enabled():
+ return False
if _BNB_GEMM_4BIT is None:
if os.environ.get("UNSLOTH_BNB_GEMM_4BIT") == "1":
_BNB_GEMM_4BIT_DTYPES = (torch.float16, torch.bfloat16)
@@ -1548,8 +1551,6 @@ def _bnb_gemm_4bit_ok(X, W, quant_state):
and quant_state.shape[-1] % quant_state.blocksize == 0
# matmul_4bit raises for any other nested blocksize.
and (not quant_state.nested or quant_state.state2.blocksize == 256)
- and not torch.is_grad_enabled()
- and not _is_compiling()
)