Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fall back to PEFT's LoRA forward for float32 base weights (#7020)
Daniel Han5a60350e05e9
diff --git a/tests/test_float32_lora_fallback.py b/tests/test_float32_lora_fallback.pynew file mode 100644index 00000000000..3e89a2b3cc8--- /dev/null+++ b/tests/test_float32_lora_fallback.py@@ -0,0 +1,64 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""float32 base weights skip the fused LoRA kernels, and fp32 NF4 dequantizes with the fp32 kernel."""++from __future__ import annotations++import types++import pytest+import torch+import unsloth # noqa: F401++from real_accelerator import has_real_cuda+from unsloth.models.llama import _base_weight_dtype, _fused_lora_skip_reason+++def _proj(weight):+ return types.SimpleNamespace(base_layer = types.SimpleNamespace(weight = weight))++[email protected]("dtype", [torch.float32, torch.bfloat16, torch.float16])+def test_base_weight_dtype_plain(dtype):+ assert _base_weight_dtype(_proj(torch.zeros(2, 2, dtype = dtype))) == dtype+++def test_base_weight_dtype_reads_quant_state():+ weight = torch.zeros(2, 2, dtype = torch.uint8)+ weight.quant_state = types.SimpleNamespace(dtype = torch.float32)+ assert _base_weight_dtype(_proj(weight)) == torch.float32+++def test_float32_skip_reason():+ assert _fused_lora_skip_reason(0, "none") == ""+ assert "float32" in _fused_lora_skip_reason(0, "none", float32_base = True)++[email protected](not has_real_cuda(), reason = "needs a real CUDA device")+def test_fp32_nf4_dequant_matches_bitsandbytes():+ import bitsandbytes as bnb+ from unsloth.kernels.utils import fast_dequantize++ W = torch.randn(256, 128) * 0.02+ p = bnb.nn.Params4bit(W, requires_grad = False, quant_type = "nf4").cuda()+ assert p.quant_state.dtype == torch.float32+ ref = bnb.functional.dequantize_4bit(p.data, p.quant_state)+ torch.testing.assert_close(fast_dequantize(p, p.quant_state), ref, rtol = 0, atol = 0)++[email protected](not has_real_cuda(), reason = "needs a real CUDA device")+def test_fp32_nf4_single_token_decode():+ import bitsandbytes as bnb+ from unsloth.kernels.utils import fast_linear_forward++ layer = bnb.nn.Linear4bit(128, 256, bias = False, compute_dtype = torch.float32, quant_type = "nf4")+ layer.weight = bnb.nn.Params4bit(+ torch.randn(256, 128) * 0.02, requires_grad = False, quant_type = "nf4"+ )+ layer = layer.cuda()+ X = torch.randn(1, 1, 128, device = "cuda")+ ref = X @ bnb.functional.dequantize_4bit(layer.weight.data, layer.weight.quant_state).t()+ torch.testing.assert_close(fast_linear_forward(layer, X), ref, rtol = 1e-4, atol = 1e-4)diff --git a/tests/test_lora_dropout_patch_summary.py b/tests/test_lora_dropout_patch_summary.pyindex b6943816a41..72f66675ab5 100644--- a/tests/test_lora_dropout_patch_summary.py+++ b/tests/test_lora_dropout_patch_summary.py@@ -86,16 +86,16 @@ def _gate_tests(source: str) -> list[str]:return out-def test_patch_peft_model_still_gates_the_fused_kernels_on_the_same_two_values():+def test_patch_peft_model_still_gates_the_fused_kernels_on_the_same_values():"""If the gate grows a third condition, _fused_lora_skip_reason has to grow with it orthe summary starts reporting zero counts with no reason again."""source = inspect.getsource(llama_module.FastLlamaModel.patch_peft_model)- assert _gate_tests(source) == ["lora_dropout == 0 and bias == 'none'"]+ assert _gate_tests(source) == ["lora_dropout == 0 and bias == 'none' and (not float32_base)"]def test_the_summary_call_carries_the_reason():source = inspect.getsource(llama_module.FastLlamaModel.patch_peft_model)- assert "unfused_reason = _fused_lora_skip_reason(lora_dropout, bias)" in source+ assert "unfused_reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base)" in sourceassert "MLP layers.{unfused_reason}" in sourcediff --git a/unsloth/bnb_availability.py b/unsloth/bnb_availability.pyindex 7960e37409e..5e120ac2bb4 100644--- a/unsloth/bnb_availability.py+++ b/unsloth/bnb_availability.py@@ -36,6 +36,7 @@ _C_SYMBOLS = ("cdequantize_blockwise_fp32","cdequantize_blockwise_fp16_nf4","cdequantize_blockwise_bf16_nf4",+ "cdequantize_blockwise_fp32_nf4",)# 4bit inference is a gemv on xpu and a naive gemm elsewhere; probing the wrong pair would write off# a perfectly good wheel.diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 45cb36a722f..441bbc4c7fe 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -270,12 +270,14 @@ if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):cdequantize_blockwise_fp32 = _bnb_requiredcdequantize_blockwise_fp16_nf4 = _bnb_requiredcdequantize_blockwise_bf16_nf4 = _bnb_required+ cdequantize_blockwise_fp32_nf4 = _bnb_requiredcgemm_4bit_inference_naive_fp16 = _bnb_requiredcgemm_4bit_inference_naive_bf16 = _bnb_requiredelse:cdequantize_blockwise_fp32 = bnb_functional.lib.cdequantize_blockwise_fp32cdequantize_blockwise_fp16_nf4 = bnb_functional.lib.cdequantize_blockwise_fp16_nf4cdequantize_blockwise_bf16_nf4 = bnb_functional.lib.cdequantize_blockwise_bf16_nf4+ cdequantize_blockwise_fp32_nf4 = bnb_functional.lib.cdequantize_blockwise_fp32_nf4if DEVICE_TYPE == "xpu":# xpu inference gemv, per bitsandbytes backends/xpu/ops.py#L115.@@ -569,9 +571,12 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:)out_absmax += offset+ # Kernel must match `out`'s dtype: bf16 bits in an fp32 buffer corrupt silently.fx = (cdequantize_blockwise_fp16_nf4if dtype == torch_float16+ else cdequantize_blockwise_fp32_nf4+ if dtype == torch_float32else cdequantize_blockwise_bf16_nf4)fx(@@ -680,6 +685,8 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:fx = (cdequantize_blockwise_fp16_nf4if dtype == torch_float16+ else cdequantize_blockwise_fp32_nf4+ if dtype == torch_float32else cdequantize_blockwise_bf16_nf4)fx(@@ -756,6 +763,8 @@ else:fx = (cdequantize_blockwise_fp16_nf4if dtype == torch_float16+ else cdequantize_blockwise_fp32_nf4+ if dtype == torch_float32else cdequantize_blockwise_bf16_nf4)fx(@@ -1108,6 +1117,12 @@ def fast_gemv(return _fast_gemv_bnb(X, W, quant_state, out = out)+def _quant_state_dtype(quant_state):+ if type(quant_state) is list:+ return quant_state[2]+ return getattr(quant_state, "dtype", None)++def fast_linear_forward(proj,X,@@ -1131,7 +1146,8 @@ def fast_linear_forward(out = fp8_linear(X, W, W_quant)elif type(W_quant) is Int4QuantState:out = _int4_matmul(X, W, W_quant, out = out)- elif bsz == 1 and q_len == 1:+ elif bsz == 1 and q_len == 1 and _quant_state_dtype(W_quant) != torch_float32:+ # The 4bit gemv kernels are fp16/bf16 only.out = fast_gemv(X, W, W_quant, out = out)else:W = fast_dequantize(W.t(), W_quant, use_global_buffer = True)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex b197f2a68a0..8852f8458a6 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2289,7 +2289,11 @@ def _vllm_will_load_weights(fast_inference, num_labels = None):return True-def _fused_lora_skip_reason(lora_dropout, bias) -> str:+def _fused_lora_skip_reason(+ lora_dropout,+ bias,+ float32_base = False,+) -> str:"""Why patch_peft_model skipped the fused LoRA kernels, for the patched layers summary.Returns "" when nothing disabled them, so the common summary line is unchanged. The@@ -2300,6 +2304,8 @@ def _fused_lora_skip_reason(lora_dropout, bias) -> str:reasons.append(f"lora_dropout = {lora_dropout}")if bias != "none":reasons.append(f"bias = '{bias}'")+ if float32_base:+ reasons.append("the base weights are float32")if not reasons:return ""return (@@ -2308,6 +2314,12 @@ def _fused_lora_skip_reason(lora_dropout, bias) -> str:)+def _base_weight_dtype(proj):+ weight = getattr(proj, "base_layer", proj).weight+ quant_state = getattr(weight, "quant_state", None)+ return quant_state.dtype if quant_state is not None else weight.dtype++class FastLlamaModel:@staticmethoddef _prepare_for_qat(model, qat_scheme):@@ -3918,7 +3930,12 @@ class FastLlamaModel:else apply_lora_mlp)- if lora_dropout == 0 and bias == "none":+ # The fused kernels assume 16-bit base weights; float32 falls back to PEFT's own forward.+ float32_base = (+ _base_weight_dtype(model.model.model.layers[0].self_attn.q_proj) == torch.float32+ )++ if lora_dropout == 0 and bias == "none" and not float32_base:for idx, layer in enumerate(model.model.model.layers):if model_type != "falcon_h1":# LoRAMLP.apply has no gate/down multiplier support yet, so falcon h1 is not patched for now.@@ -3997,7 +4014,7 @@ class FastLlamaModel:)# A zero count reads as a failure, so say why the fused kernels were skipped.- unfused_reason = _fused_lora_skip_reason(lora_dropout, bias)+ unfused_reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base)logger.warning_once(f"Unsloth {__version__} patched {len(model.model.model.layers)} layers with "f"{n_qkv} QKV layers, {n_o} O layers and {n_mlp} MLP layers.{unfused_reason}",