Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
NF4 bitsandbytes GEMM gate: return before its lazy setup under torch.compile (#14608)
Daniel Han356721b31ebe
diff --git a/tests/test_bnb_integration_compile.py b/tests/test_bnb_integration_compile.pyindex 9637454d43a..542f02a067a 100644--- a/tests/test_bnb_integration_compile.py+++ b/tests/test_bnb_integration_compile.py@@ -477,6 +477,23 @@ def test_fast_linear_forward_compiles_fullgraph(nf4_kernels, bsz):_assert_compiled_matches(compiled, eager, backend = "aot_eager")+# No eager warmup: the GEMM gate's first call must not run its lazy setup while tracing.[email protected]("bsz", [8, 16])+def test_fast_linear_forward_cold_compile_fullgraph(nf4_kernels, monkeypatch, bsz):+ monkeypatch.setattr(U, "_BNB_GEMM_4BIT", None)+ monkeypatch.setattr(U, "_BNB_GEMM_4BIT_DEVICE", {})+ monkeypatch.setattr(U, "_BNB_GEMM_4BIT_ARCHS", {torch.cuda.get_device_capability()})+ _, block = _lora_block(dtype = torch.float16)+ fn = lambda x: U.fast_linear_forward(block.o_proj, x)+ X = torch.randn(bsz, 1, 256, dtype = torch.float16, device = DEVICE)+ torch._dynamo.reset()+ with torch.no_grad():+ compiled = torch.compile(fn, fullgraph = _dynamo_traces_params4bit(), backend = "aot_eager")(X)+ eager = fn(X)+ torch._dynamo.reset()+ _assert_compiled_matches(compiled, eager, backend = "aot_eager")++def test_eager_gemv_gate_off_takes_bitsandbytes(monkeypatch):"""Where the Triton GEMV is not the eager choice (HIP before Triton 3.6, orUNSLOTH_TRITON_GEMV_EAGER=0), eager decode takes bitsandbytes' GEMV, on the live stream.diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex bebeb09eec1..6d36ce8cf02 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -1521,6 +1521,9 @@ def _bnb_gemm_4bit_device_ok(device):def _bnb_gemm_4bit_ok(X, W, quant_state):global _BNB_GEMM_4BIT, _BNB_GEMM_4BIT_DTYPES+ # Before the lazy setup: dynamo cannot trace torch.cuda.device_count().+ if _is_compiling() or torch.is_grad_enabled():+ return Falseif _BNB_GEMM_4BIT is None:if os.environ.get("UNSLOTH_BNB_GEMM_4BIT") == "1":_BNB_GEMM_4BIT_DTYPES = (torch.float16, torch.bfloat16)@@ -1548,8 +1551,6 @@ def _bnb_gemm_4bit_ok(X, W, quant_state):and quant_state.shape[-1] % quant_state.blocksize == 0# matmul_4bit raises for any other nested blocksize.and (not quant_state.nested or quant_state.state2.blocksize == 256)- and not torch.is_grad_enabled()- and not _is_compiling())