Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fall back to PEFT's LoRA forward for float32 base weights (#7020)

Daniel Han5a60350e05e9
diff --git a/tests/test_float32_lora_fallback.py b/tests/test_float32_lora_fallback.py
new file mode 100644
index 00000000000..3e89a2b3cc8
--- /dev/null
+++ b/tests/test_float32_lora_fallback.py
@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""float32 base weights skip the fused LoRA kernels, and fp32 NF4 dequantizes with the fp32 kernel."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+import torch
+import unsloth # noqa: F401
+
+from real_accelerator import has_real_cuda
+from unsloth.models.llama import _base_weight_dtype, _fused_lora_skip_reason
+
+
+def _proj(weight):
+ return types.SimpleNamespace(base_layer = types.SimpleNamespace(weight = weight))
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16, torch.float16])
+def test_base_weight_dtype_plain(dtype):
+ assert _base_weight_dtype(_proj(torch.zeros(2, 2, dtype = dtype))) == dtype
+
+
+def test_base_weight_dtype_reads_quant_state():
+ weight = torch.zeros(2, 2, dtype = torch.uint8)
+ weight.quant_state = types.SimpleNamespace(dtype = torch.float32)
+ assert _base_weight_dtype(_proj(weight)) == torch.float32
+
+
+def test_float32_skip_reason():
+ assert _fused_lora_skip_reason(0, "none") == ""
+ assert "float32" in _fused_lora_skip_reason(0, "none", float32_base = True)
+
+
[email protected](not has_real_cuda(), reason = "needs a real CUDA device")
+def test_fp32_nf4_dequant_matches_bitsandbytes():
+ import bitsandbytes as bnb
+ from unsloth.kernels.utils import fast_dequantize
+
+ W = torch.randn(256, 128) * 0.02
+ p = bnb.nn.Params4bit(W, requires_grad = False, quant_type = "nf4").cuda()
+ assert p.quant_state.dtype == torch.float32
+ ref = bnb.functional.dequantize_4bit(p.data, p.quant_state)
+ torch.testing.assert_close(fast_dequantize(p, p.quant_state), ref, rtol = 0, atol = 0)
+
+
[email protected](not has_real_cuda(), reason = "needs a real CUDA device")
+def test_fp32_nf4_single_token_decode():
+ import bitsandbytes as bnb
+ from unsloth.kernels.utils import fast_linear_forward
+
+ layer = bnb.nn.Linear4bit(128, 256, bias = False, compute_dtype = torch.float32, quant_type = "nf4")
+ layer.weight = bnb.nn.Params4bit(
+ torch.randn(256, 128) * 0.02, requires_grad = False, quant_type = "nf4"
+ )
+ layer = layer.cuda()
+ X = torch.randn(1, 1, 128, device = "cuda")
+ ref = X @ bnb.functional.dequantize_4bit(layer.weight.data, layer.weight.quant_state).t()
+ torch.testing.assert_close(fast_linear_forward(layer, X), ref, rtol = 1e-4, atol = 1e-4)
diff --git a/tests/test_lora_dropout_patch_summary.py b/tests/test_lora_dropout_patch_summary.py
index b6943816a41..72f66675ab5 100644
--- a/tests/test_lora_dropout_patch_summary.py
+++ b/tests/test_lora_dropout_patch_summary.py
@@ -86,16 +86,16 @@ def _gate_tests(source: str) -> list[str]:
return out
-def test_patch_peft_model_still_gates_the_fused_kernels_on_the_same_two_values():
+def test_patch_peft_model_still_gates_the_fused_kernels_on_the_same_values():
"""If the gate grows a third condition, _fused_lora_skip_reason has to grow with it or
the summary starts reporting zero counts with no reason again."""
source = inspect.getsource(llama_module.FastLlamaModel.patch_peft_model)
- assert _gate_tests(source) == ["lora_dropout == 0 and bias == 'none'"]
+ assert _gate_tests(source) == ["lora_dropout == 0 and bias == 'none' and (not float32_base)"]
def test_the_summary_call_carries_the_reason():
source = inspect.getsource(llama_module.FastLlamaModel.patch_peft_model)
- assert "unfused_reason = _fused_lora_skip_reason(lora_dropout, bias)" in source
+ assert "unfused_reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base)" in source
assert "MLP layers.{unfused_reason}" in source
diff --git a/unsloth/bnb_availability.py b/unsloth/bnb_availability.py
index 7960e37409e..5e120ac2bb4 100644
--- a/unsloth/bnb_availability.py
+++ b/unsloth/bnb_availability.py
@@ -36,6 +36,7 @@ _C_SYMBOLS = (
"cdequantize_blockwise_fp32",
"cdequantize_blockwise_fp16_nf4",
"cdequantize_blockwise_bf16_nf4",
+ "cdequantize_blockwise_fp32_nf4",
)
# 4bit inference is a gemv on xpu and a naive gemm elsewhere; probing the wrong pair would write off
# a perfectly good wheel.
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 45cb36a722f..441bbc4c7fe 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -270,12 +270,14 @@ if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):
cdequantize_blockwise_fp32 = _bnb_required
cdequantize_blockwise_fp16_nf4 = _bnb_required
cdequantize_blockwise_bf16_nf4 = _bnb_required
+ cdequantize_blockwise_fp32_nf4 = _bnb_required
cgemm_4bit_inference_naive_fp16 = _bnb_required
cgemm_4bit_inference_naive_bf16 = _bnb_required
else:
cdequantize_blockwise_fp32 = bnb_functional.lib.cdequantize_blockwise_fp32
cdequantize_blockwise_fp16_nf4 = bnb_functional.lib.cdequantize_blockwise_fp16_nf4
cdequantize_blockwise_bf16_nf4 = bnb_functional.lib.cdequantize_blockwise_bf16_nf4
+ cdequantize_blockwise_fp32_nf4 = bnb_functional.lib.cdequantize_blockwise_fp32_nf4
if DEVICE_TYPE == "xpu":
# xpu inference gemv, per bitsandbytes backends/xpu/ops.py#L115.
@@ -569,9 +571,12 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
)
out_absmax += offset
+ # Kernel must match `out`'s dtype: bf16 bits in an fp32 buffer corrupt silently.
fx = (
cdequantize_blockwise_fp16_nf4
if dtype == torch_float16
+ else cdequantize_blockwise_fp32_nf4
+ if dtype == torch_float32
else cdequantize_blockwise_bf16_nf4
)
fx(
@@ -680,6 +685,8 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
fx = (
cdequantize_blockwise_fp16_nf4
if dtype == torch_float16
+ else cdequantize_blockwise_fp32_nf4
+ if dtype == torch_float32
else cdequantize_blockwise_bf16_nf4
)
fx(
@@ -756,6 +763,8 @@ else:
fx = (
cdequantize_blockwise_fp16_nf4
if dtype == torch_float16
+ else cdequantize_blockwise_fp32_nf4
+ if dtype == torch_float32
else cdequantize_blockwise_bf16_nf4
)
fx(
@@ -1108,6 +1117,12 @@ def fast_gemv(
return _fast_gemv_bnb(X, W, quant_state, out = out)
+def _quant_state_dtype(quant_state):
+ if type(quant_state) is list:
+ return quant_state[2]
+ return getattr(quant_state, "dtype", None)
+
+
def fast_linear_forward(
proj,
X,
@@ -1131,7 +1146,8 @@ def fast_linear_forward(
out = fp8_linear(X, W, W_quant)
elif type(W_quant) is Int4QuantState:
out = _int4_matmul(X, W, W_quant, out = out)
- elif bsz == 1 and q_len == 1:
+ elif bsz == 1 and q_len == 1 and _quant_state_dtype(W_quant) != torch_float32:
+ # The 4bit gemv kernels are fp16/bf16 only.
out = fast_gemv(X, W, W_quant, out = out)
else:
W = fast_dequantize(W.t(), W_quant, use_global_buffer = True)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index b197f2a68a0..8852f8458a6 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2289,7 +2289,11 @@ def _vllm_will_load_weights(fast_inference, num_labels = None):
return True
-def _fused_lora_skip_reason(lora_dropout, bias) -> str:
+def _fused_lora_skip_reason(
+ lora_dropout,
+ bias,
+ float32_base = False,
+) -> str:
"""Why patch_peft_model skipped the fused LoRA kernels, for the patched layers summary.
Returns "" when nothing disabled them, so the common summary line is unchanged. The
@@ -2300,6 +2304,8 @@ def _fused_lora_skip_reason(lora_dropout, bias) -> str:
reasons.append(f"lora_dropout = {lora_dropout}")
if bias != "none":
reasons.append(f"bias = '{bias}'")
+ if float32_base:
+ reasons.append("the base weights are float32")
if not reasons:
return ""
return (
@@ -2308,6 +2314,12 @@ def _fused_lora_skip_reason(lora_dropout, bias) -> str:
)
+def _base_weight_dtype(proj):
+ weight = getattr(proj, "base_layer", proj).weight
+ quant_state = getattr(weight, "quant_state", None)
+ return quant_state.dtype if quant_state is not None else weight.dtype
+
+
class FastLlamaModel:
@staticmethod
def _prepare_for_qat(model, qat_scheme):
@@ -3918,7 +3930,12 @@ class FastLlamaModel:
else apply_lora_mlp
)
- if lora_dropout == 0 and bias == "none":
+ # The fused kernels assume 16-bit base weights; float32 falls back to PEFT's own forward.
+ float32_base = (
+ _base_weight_dtype(model.model.model.layers[0].self_attn.q_proj) == torch.float32
+ )
+
+ if lora_dropout == 0 and bias == "none" and not float32_base:
for idx, layer in enumerate(model.model.model.layers):
if model_type != "falcon_h1":
# LoRAMLP.apply has no gate/down multiplier support yet, so falcon h1 is not patched for now.
@@ -3997,7 +4014,7 @@ class FastLlamaModel:
)
# A zero count reads as a failure, so say why the fused kernels were skipped.
- unfused_reason = _fused_lora_skip_reason(lora_dropout, bias)
+ unfused_reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base)
logger.warning_once(
f"Unsloth {__version__} patched {len(model.model.model.layers)} layers with "
f"{n_qkv} QKV layers, {n_o} O layers and {n_mlp} MLP layers.{unfused_reason}",