Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fused Triton NF4 dequant and GEMV: bit-exact, stream-safe, torch.compile traceable (#12113)

Daniel Han338d23f6b2f9
diff --git a/tests/test_bnb_integration_compile.py b/tests/test_bnb_integration_compile.py
new file mode 100644
index 00000000000..cfd14991a08
--- /dev/null
+++ b/tests/test_bnb_integration_compile.py
@@ -0,0 +1,568 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""The 4bit paths in unsloth.kernels.utils (fast_dequantize, fast_gemv, matmul_lora and the
+fast_lora autograd functions) must match bitsandbytes byte for byte, run on whatever stream is
+current, survive CUDA graph capture, and trace under torch.compile without graph breaks.
+
+Every correctness test runs twice: on the NF4 kernels and on the bitsandbytes ctypes fallback
+(UNSLOTH_BNB_TRITON=0).
+"""
+
+import pytest
+import torch
+
+pytest.importorskip("bitsandbytes")
+if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA/HIP device", allow_module_level = True)
+
+import unsloth # noqa: F401 (sets UNSLOTH_IS_PRESENT before transformers)
+import bitsandbytes as bnb
+import bitsandbytes.functional as F
+from unsloth.kernels import utils as U
+from unsloth.kernels import fast_lora
+from unsloth.kernels.fast_lora import (
+ LoRA_W,
+ apply_lora_mlp_swiglu,
+ apply_lora_o,
+ apply_lora_qkv,
+)
+
+_NF4_KERNELS_AVAILABLE = U._USE_NF4_KERNELS
+DEVICE = "cuda"
+# Unsloth runs fp16 on GPUs without native bf16 (pre-Ampere, e.g. T4), and Inductor cannot re-emit a
+# bf16 Triton kernel there, so the single-graph checks use the dtype Unsloth would.
+CDTYPE = (
+ torch.bfloat16
+ if torch.version.hip or torch.cuda.get_device_capability()[0] >= 8
+ else torch.float16
+)
+
+
[email protected](params = ["nf4_kernels", "ctypes_fallback"])
+def path(request, monkeypatch):
+ use = request.param == "nf4_kernels"
+ if use and not _NF4_KERNELS_AVAILABLE:
+ pytest.skip("NF4 kernels unavailable or disabled")
+ monkeypatch.setattr(U, "_USE_NF4_KERNELS", use)
+ # Exercise the Triton GEMV even where eager decode would keep bitsandbytes' (Triton < 3.7).
+ monkeypatch.setattr(U, "_TRITON_GEMV_EAGER", use)
+ U._SCRATCH.clear()
+ yield request.param
+ U._SCRATCH.clear()
+
+
+def nf4_kernels(monkeypatch):
+ if not _NF4_KERNELS_AVAILABLE:
+ pytest.skip("NF4 kernels unavailable or disabled")
+ monkeypatch.setattr(U, "_USE_NF4_KERNELS", True)
+ monkeypatch.setattr(U, "_TRITON_GEMV_EAGER", True)
+ torch._dynamo.reset()
+ yield
+ torch._dynamo.reset()
+
+
+def _quantize(
+ shape,
+ dtype,
+ nested = True,
+ seed = 0,
+):
+ g = torch.Generator(device = DEVICE).manual_seed(seed)
+ W = torch.randn(shape, dtype = dtype, device = DEVICE, generator = g)
+ return F.quantize_4bit(W, quant_type = "nf4", compress_statistics = nested)
+
+
+def _as_list(s):
+ # The pre TimDettmers/bitsandbytes#763 layout fast_dequantize still accepts.
+ state2 = [s.state2.absmax, s.state2.code, s.state2.blocksize, None, None, None, None]
+ return [s.absmax, s.shape, s.dtype, s.blocksize, [s.offset, state2], s.quant_type, s.code]
+
+
+_CONTROL = {}
+
+
+def _torch_compiles_lora_exactly(backend):
+ """Some torch versions (2.7) decompose the in-place LoRA addmm_/addmv_ differently when
+ compiled, even on a plain 16bit weight with no 4bit op in sight. Measure that on a control so
+ compiled-vs-eager checks stay exact wherever torch itself is exact."""
+ if backend not in _CONTROL:
+ g = torch.Generator(device = DEVICE).manual_seed(1)
+ r = lambda *s: torch.randn(*s, dtype = CDTYPE, device = DEVICE, generator = g)
+ W, A, B, X, x = r(512, 256), r(8, 256), r(512, 8), r(32, 256), r(256)
+
+ def control(X, x):
+ out = X @ W.t()
+ out.addmm_(X @ A.t(), B.t(), alpha = 2.0)
+ v = W @ x
+ v.addmv_(B, A @ x, alpha = 2.0)
+ return out, v
+
+ torch._dynamo.reset()
+ compiled = torch.compile(control, fullgraph = True, backend = backend)(X, x)
+ torch._dynamo.reset()
+ _CONTROL[backend] = all(torch.equal(a, b) for a, b in zip(compiled, control(X, x)))
+ return _CONTROL[backend]
+
+
+_TRACES_PARAMS4BIT = []
+
+
[email protected]_op("unsloth_test::passthrough", mutates_args = ())
+def _passthrough(x: torch.Tensor) -> torch.Tensor:
+ return x.clone()
+
+
+@_passthrough.register_fake
+def _(x):
+ return torch.empty_like(x)
+
+
+def _dynamo_traces_params4bit():
+ """Older Dynamo (torch 2.7) cannot hand a Params4bit weight to any torch.library op (nor call
+ some of its methods) without a graph break, whatever the op does. There the fast_lora paths
+ can only be checked for correctness under torch.compile, not for a single graph. Probed with
+ a throwaway op so a regression in the 4bit ops cannot turn the full-graph checks off."""
+ if not _TRACES_PARAMS4BIT:
+ lin = bnb.nn.Linear4bit(64, 64, bias = False, quant_type = "nf4").to(DEVICE)
+ fn = lambda x: x + _passthrough(lin.weight)[0, 0] + lin.weight.t()[0, 0]
+ torch._dynamo.reset()
+ breaks = torch._dynamo.explain(fn)(torch.ones(1, device = DEVICE)).graph_break_count
+ _TRACES_PARAMS4BIT.append(breaks == 0)
+ torch._dynamo.reset()
+ return _TRACES_PARAMS4BIT[0]
+
+
+def _assert_compiled_matches(
+ compiled,
+ eager,
+ backend = "inductor",
+ exact = None,
+):
+ if exact is None:
+ exact = _torch_compiles_lora_exactly(backend)
+ if exact:
+ assert torch.equal(compiled, eager)
+ return
+ scale = eager.float().abs().max().clamp_min(1e-6)
+ err = ((compiled.float() - eager.float()).abs().max() / scale).item()
+ assert err < 2e-2, err
+
+
+def _bits(t):
+ return t.contiguous().view(torch.int16 if t.element_size() == 2 else torch.int32)
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("nested", [True, False])
[email protected]("shape", [(1024, 768), (130, 70), (4096, 1024)])
[email protected]("global_buffer", [False, True])
+def test_dequantize_matches_bitsandbytes(path, dtype, nested, shape, global_buffer):
+ q, s = _quantize(shape, dtype, nested)
+ ref = F.dequantize_4bit(q, s)
+ out = U.fast_dequantize(q, s, use_global_buffer = global_buffer)
+ assert torch.equal(_bits(out), _bits(ref))
+ if nested:
+ out_list = U.fast_dequantize(q, _as_list(s), use_global_buffer = global_buffer)
+ assert torch.equal(_bits(out_list), _bits(ref))
+ # The backward passes dequantize W.t(); a single packed column must come back transposed.
+ out_t = U.fast_dequantize(q.t(), s)
+ assert out_t.shape == (shape[1], shape[0])
+ assert torch.equal(_bits(out_t.t()), _bits(ref))
+
+
+def test_explicit_out(path):
+ q, s = _quantize((512, 256), torch.bfloat16)
+ out = torch.empty((512, 256), dtype = torch.bfloat16, device = DEVICE)
+ got = U.fast_dequantize(q, s, out = out)
+ assert got.data_ptr() == out.data_ptr()
+ assert torch.equal(_bits(out), _bits(F.dequantize_4bit(q, s)))
+
+
+def test_fp4_state_uses_its_own_codebook(path):
+ W = torch.randn(256, 128, dtype = torch.bfloat16, device = DEVICE)
+ q, s = F.quantize_4bit(W, quant_type = "fp4", compress_statistics = True)
+ assert torch.equal(_bits(U.fast_dequantize(q, s)), _bits(F.dequantize_4bit(q, s)))
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
+def test_gemv_matches_reference(path, dtype):
+ q, s = _quantize((2048, 1024), dtype)
+ X = torch.randn(1, 1, 1024, dtype = dtype, device = DEVICE)
+ got = U.fast_gemv(X, q, s)
+ ref = X.float() @ F.dequantize_4bit(q, s).float().t()
+ assert got.shape == (1, 1, 2048)
+ rel = ((got.float() - ref).norm() / ref.norm()).item()
+ assert rel < 1e-2, rel
+
+
[email protected]("storage", [torch.bfloat16, torch.float32], ids = str)
+def test_float_packed_storage(path, storage):
+ """FSDP-QLoRA packs the 4bit weight into float storage (quant_storage); both the dequant and
+ the GEMV must read it as bytes, as the ctypes path always did."""
+ W = torch.randn(1024, 512, dtype = torch.bfloat16, device = DEVICE)
+ q, s = F.quantize_4bit(W, quant_type = "nf4", compress_statistics = True, quant_storage = storage)
+ ref = F.dequantize_4bit(q, s)
+ assert torch.equal(_bits(U.fast_dequantize(q, s)), _bits(ref))
+ X = torch.randn(1, 1, 512, dtype = torch.bfloat16, device = DEVICE)
+ got = U.fast_gemv(X, q, s).float()
+ want = (X @ ref.t()).float()
+ assert ((got - want).abs().max() / want.abs().max()).item() < 1e-2
+
+
+def test_side_streams_never_share_the_scratch(path):
+ """Two streams dequantizing different weights with use_global_buffer at the same time: each
+ must read its own weight back. Only the default stream owns a scratch, so a program making a
+ new stream per step cannot pin a weight-sized buffer per stream."""
+ (q1, s1), (q2, s2) = (
+ _quantize((4096, 4096), torch.bfloat16, seed = 1),
+ _quantize((4096, 4096), torch.bfloat16, seed = 2),
+ )
+ X = torch.randn(64, 4096, dtype = torch.bfloat16, device = DEVICE)
+ refs = [X @ F.dequantize_4bit(q, s).t() for q, s in ((q1, s1), (q2, s2))]
+ outs = [None, None]
+ torch.cuda.synchronize()
+ for _ in range(10):
+ for i, (q, s) in enumerate(((q1, s1), (q2, s2))):
+ with torch.cuda.stream(torch.cuda.Stream()):
+ W = U.fast_dequantize(q, s, use_global_buffer = True)
+ outs[i] = X @ W.t()
+ torch.cuda.synchronize()
+ for got, ref in zip(outs, refs):
+ assert torch.equal(got, ref)
+ assert not U._SCRATCH
+ U.fast_dequantize(q1, s1, use_global_buffer = True)
+ assert {k[0] for k in U._SCRATCH} <= {"weight", "absmax"}
+ assert len([k for k in U._SCRATCH if k[0] == "weight"]) == 1
+
+
[email protected]("shape", [(512, 96), (256, 4160), (130, 100)])
+def test_gemv_rows_not_starting_a_block_fall_back_to_dequant(path, shape):
+ """Both GEMV kernels assume each row starts a quantization block; other shapes must still be
+ right (bitsandbytes' own GEMV is wrong for them too)."""
+ n, k = shape
+ q, s = _quantize(shape, torch.bfloat16)
+ X = torch.randn(1, 1, k, dtype = torch.bfloat16, device = DEVICE)
+ ref = (X @ F.dequantize_4bit(q, s).t()).float()
+ got = U.fast_gemv(X, q, s).float()
+ assert got.shape == (1, 1, n)
+ assert ((got - ref).abs().max() / ref.abs().max()).item() < 1e-2
+
+
+def test_gemv_side_stream_and_cuda_graph(path):
+ q, s = _quantize((4096, 4096), torch.bfloat16)
+ X = torch.randn(1, 1, 4096, dtype = torch.bfloat16, device = DEVICE)
+ expected = U.fast_gemv(X, q, s)
+ side = torch.cuda.Stream()
+ side.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(side):
+ x = X.clone() * 1 # produced on the side stream, consumed there, no host sync
+ results = [U.fast_gemv(x, q, s) for _ in range(10)]
+ for _ in range(2):
+ U.fast_gemv(x, q, s)
+ torch.cuda.current_stream().wait_stream(side)
+ torch.cuda.synchronize()
+ assert all(torch.equal(r, expected) for r in results)
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ static_out = U.fast_gemv(x, q, s)
+ x.copy_(X * 2)
+ graph.replay()
+ torch.cuda.synchronize()
+ assert torch.equal(static_out, U.fast_gemv(X * 2, q, s))
+
+
+def test_kernel_failure_falls_back_to_bitsandbytes(nf4_kernels, monkeypatch):
+ """A Triton compile failure on an untested GPU switches to the bitsandbytes kernels."""
+ q, s = _quantize((512, 256), torch.bfloat16)
+
+ def broken(*args, **kwargs):
+ raise RuntimeError("PTX compile failed")
+
+ for name in ("dequantize_nf4", "dequantize_nf4_planned", "gemv_nf4", "gemv_nf4_planned"):
+ monkeypatch.setattr(U, name, broken)
+ assert torch.equal(_bits(U.fast_dequantize(q, s)), _bits(F.dequantize_4bit(q, s)))
+ assert U._USE_NF4_KERNELS is False
+ monkeypatch.setattr(U, "_USE_NF4_KERNELS", True)
+ X = torch.randn(1, 1, 256, dtype = torch.bfloat16, device = DEVICE)
+ assert U.fast_gemv(X, q, s).shape == (1, 1, 512)
+ assert U._USE_NF4_KERNELS is False
+ # Batch > 1 decode dequantizes into the scratch buffer through the planned launch.
+ monkeypatch.setattr(U, "_USE_NF4_KERNELS", True)
+ got = U.fast_dequantize(q.t(), s, use_global_buffer = True).t()
+ assert torch.equal(_bits(got), _bits(F.dequantize_4bit(q, s)))
+ assert U._USE_NF4_KERNELS is False
+
+
+def test_out_of_memory_is_not_mistaken_for_a_kernel_failure(nf4_kernels, monkeypatch):
+ q, s = _quantize((512, 256), torch.bfloat16)
+
+ def oom(*args, **kwargs):
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")
+
+ monkeypatch.setattr(U, "dequantize_nf4", oom)
+ with pytest.raises(torch.cuda.OutOfMemoryError):
+ U.fast_dequantize(q, s)
+ assert U._USE_NF4_KERNELS is True
+
+
+def test_scratch_first_allocated_under_inference_mode_still_trains(path):
+ """generate() runs under torch.inference_mode(); the scratch it allocates first must stay
+ usable by the training step that follows."""
+ q, s = _quantize((512, 256), torch.bfloat16)
+ A = (torch.randn(8, 256, dtype = torch.bfloat16, device = DEVICE) * 0.02).requires_grad_()
+ B = (torch.randn(512, 8, dtype = torch.bfloat16, device = DEVICE) * 0.02).requires_grad_()
+ X = torch.randn(2, 4, 256, dtype = torch.bfloat16, device = DEVICE)
+ with torch.inference_mode():
+ U.matmul_lora(X, q, s, A.detach(), B.detach(), 2.0)
+ assert all(not t.is_inference() for t in U._SCRATCH.values())
+
+ def step():
+ x = X.clone().requires_grad_()
+ LoRA_W.apply(x, q, s, A, B, 2.0).float().pow(2).mean().backward()
+ grads = (x.grad.clone(), A.grad.clone(), B.grad.clone())
+ A.grad = B.grad = None
+ return grads
+
+ after_generate = step()
+ U._SCRATCH.clear()
+ fresh = step()
+ for a, b in zip(after_generate, fresh):
+ assert torch.equal(a, b)
+
+
+def test_matmul_lora_cuda_graph_capture_and_replay(path):
+ q1, s1 = _quantize((1024, 512), torch.bfloat16, seed = 3)
+ q2, s2 = _quantize((1024, 512), torch.bfloat16, seed = 4)
+ A = torch.randn(8, 512, dtype = torch.bfloat16, device = DEVICE) * 0.02
+ B = torch.randn(1024, 8, dtype = torch.bfloat16, device = DEVICE) * 0.02
+ static_x = torch.randn(4, 512, dtype = torch.bfloat16, device = DEVICE)
+
+ side = torch.cuda.Stream()
+ side.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(side):
+ for _ in range(3):
+ U.matmul_lora(static_x, q1, s1, A, B, 2.0)
+ U.matmul_lora(static_x, q2, s2, A, B, 2.0)
+ torch.cuda.current_stream().wait_stream(side)
+
+ g1, g2 = torch.cuda.CUDAGraph(), torch.cuda.CUDAGraph()
+ with torch.cuda.graph(g1):
+ static_y1 = U.matmul_lora(static_x, q1, s1, A, B, 2.0)
+ with torch.cuda.graph(g2):
+ static_y2 = U.matmul_lora(static_x, q2, s2, A, B, 2.0)
+
+ for seed in range(3):
+ static_x.copy_(torch.randn(4, 512, dtype = torch.bfloat16, device = DEVICE))
+ g1.replay()
+ # An eager call in between may grow or reuse the eager scratch; the graphs must not care.
+ big_q, big_s = _quantize((4096, 2048), torch.bfloat16, seed = 10 + seed)
+ U.fast_dequantize(big_q, big_s, use_global_buffer = True)
+ g2.replay()
+ torch.cuda.synchronize()
+ assert torch.equal(static_y1, U.matmul_lora(static_x, q1, s1, A, B, 2.0))
+ assert torch.equal(static_y2, U.matmul_lora(static_x, q2, s2, A, B, 2.0))
+
+
+def _lora_block(
+ dtype = torch.bfloat16,
+ D = 256,
+ H = 512,
+):
+ import torch.nn as nn
+ from peft import LoraConfig, get_peft_model
+
+ torch.manual_seed(0)
+
+ class Block(nn.Module):
+ def __init__(self):
+ super().__init__()
+ L = lambda i, o: bnb.nn.Linear4bit(
+ i, o, bias = False, compute_dtype = dtype, quant_type = "nf4"
+ )
+ self.gate_proj, self.up_proj, self.down_proj = L(D, H), L(D, H), L(H, D)
+ self.q_proj, self.k_proj, self.v_proj, self.o_proj = (
+ L(D, D),
+ L(D, 128),
+ L(D, 128),
+ L(D, D),
+ )
+
+ block = Block()
+ for module in block.modules():
+ if isinstance(module, bnb.nn.Linear4bit):
+ module.weight = bnb.nn.Params4bit(
+ (torch.randn(module.out_features, module.in_features) * 0.05).to(dtype),
+ requires_grad = False,
+ quant_type = "nf4",
+ compress_statistics = True,
+ )
+ targets = ["gate_proj", "up_proj", "down_proj", "q_proj", "k_proj", "v_proj", "o_proj"]
+ model = get_peft_model(block.to(DEVICE), LoraConfig(r = 8, lora_alpha = 16, target_modules = targets))
+ for name, p in model.named_parameters():
+ if "lora_B" in name:
+ torch.nn.init.normal_(p, std = 0.02)
+ return model, model.base_model.model
+
+
+_LORA_FNS = {
+ "mlp": lambda block: (lambda x: apply_lora_mlp_swiglu(block, x)),
+ "qkv": lambda block: (lambda x: apply_lora_qkv(block, x, inplace = False)),
+ "o": lambda block: (lambda x: apply_lora_o(block, x)),
+}
+
+
+def _fwd_bwd(model, fn, X):
+ x = X.clone().requires_grad_()
+ out = fn(x)
+ outs = out if isinstance(out, tuple) else (out,)
+ sum(o.float().pow(2).mean() for o in outs).backward()
+ grads = [x.grad] + [p.grad for p in model.parameters() if p.grad is not None]
+ result = [o.detach().clone() for o in outs] + [g.clone() for g in grads]
+ model.zero_grad(set_to_none = True)
+ return result
+
+
[email protected]("which", ["mlp", "qkv", "o"])
+def test_fast_lora_compiles_fullgraph(nf4_kernels, which):
+ model, block = _lora_block(dtype = CDTYPE)
+ fn = _LORA_FNS[which](block)
+ X = torch.randn(2, 16, 256, dtype = CDTYPE, device = DEVICE)
+ eager = _fwd_bwd(model, fn, X)
+ # Before torch 2.11 the LoRA Functions stay opaque to torch.compile (fast_lora.TRACE_LORA_FUNCTIONS).
+ fullgraph = _dynamo_traces_params4bit() and fast_lora.TRACE_LORA_FUNCTIONS
+ if fullgraph:
+ explained = torch._dynamo.explain(fn)(X.clone().requires_grad_())
+ assert explained.graph_break_count == 0, explained.break_reasons
+ torch._dynamo.reset()
+ compiled = _fwd_bwd(model, torch.compile(fn, fullgraph = fullgraph), X)
+ # The dequant is exact under compile; the plain torch LoRA matmuls around it need not be.
+ assert len(compiled) == len(eager)
+ for a, b in zip(eager, compiled):
+ _assert_compiled_matches(b, a, exact = False)
+
+
+def test_primitives_compile_fullgraph(nf4_kernels):
+ q, s = _quantize((512, 256), CDTYPE)
+ X1 = torch.randn(1, 1, 256, dtype = CDTYPE, device = DEVICE)
+ X = torch.randn(2, 16, 256, dtype = CDTYPE, device = DEVICE)
+ A = torch.randn(8, 256, dtype = CDTYPE, device = DEVICE) * 0.02
+ B = torch.randn(512, 8, dtype = CDTYPE, device = DEVICE) * 0.02
+ cases = [
+ (lambda q: U.fast_dequantize(q, s, use_global_buffer = True) * 1, (q,), True),
+ (lambda q: U.fast_dequantize(q.t(), s) * 1, (q,), True),
+ # Inductor re-emits the GEMV reduction and may round a partial sum 1 ulp differently.
+ (lambda x: U.fast_gemv(x, q, s) * 1, (X1,), "close"),
+ (lambda x: U.matmul_lora(x, q, s, A, B, 2.0) * 1, (X,), False),
+ ]
+ for fn, args, always_exact in cases:
+ explained = torch._dynamo.explain(fn)(*args)
+ assert explained.graph_break_count == 0, explained.break_reasons
+ torch._dynamo.reset()
+ compiled, eager = torch.compile(fn, fullgraph = True)(*args), fn(*args)
+ if always_exact == "close":
+ _assert_compiled_matches(compiled, eager, exact = False)
+ elif always_exact:
+ assert torch.equal(compiled, eager)
+ else:
+ _assert_compiled_matches(compiled, eager)
+ torch._dynamo.reset()
+
+
[email protected]("bsz", [1, 2])
+def test_fast_linear_forward_compiles_fullgraph(nf4_kernels, bsz):
+ _, block = _lora_block(dtype = CDTYPE)
+ fn = lambda x: U.fast_linear_forward(block.o_proj, x)
+ X = torch.randn(bsz, 1, 256, dtype = CDTYPE, device = DEVICE)
+ with torch.inference_mode():
+ eager = fn(X).clone()
+ fullgraph = _dynamo_traces_params4bit()
+ if fullgraph:
+ explained = torch._dynamo.explain(fn)(X)
+ assert explained.graph_break_count == 0, explained.break_reasons
+ torch._dynamo.reset()
+ compiled = torch.compile(fn, fullgraph = fullgraph, backend = "aot_eager")(X)
+ # aot_eager: inductor may lower the bsz=1 LoRA addmv one bf16 ulp differently.
+ _assert_compiled_matches(compiled, eager, backend = "aot_eager")
+
+
+def test_eager_gemv_gate_off_takes_bitsandbytes(monkeypatch):
+ """Where the Triton GEMV is not the eager choice (HIP before Triton 3.7), eager decode takes
+ bitsandbytes' GEMV, on the live stream. On CUDA the Triton GEMV is the eager choice."""
+ from unsloth.kernels import nf4_gemv
+
+ if torch.version.hip is None and _NF4_KERNELS_AVAILABLE:
+ assert nf4_gemv.triton_gemv_eager()
+ if not _NF4_KERNELS_AVAILABLE:
+ pytest.skip("NF4 kernels unavailable or disabled")
+ monkeypatch.setattr(U, "_USE_NF4_KERNELS", True)
+ monkeypatch.setattr(U, "_TRITON_GEMV_EAGER", False)
+ q, s = _quantize((512, 256), torch.bfloat16)
+ X = torch.randn(1, 1, 256, dtype = torch.bfloat16, device = DEVICE)
+ stream = torch.cuda.Stream()
+ stream.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(stream):
+ out = U.fast_gemv(X, q, s)
+ stream.synchronize()
+ assert torch.equal(out, U._fast_gemv_ctypes(X, q, s))
+
+
+def test_graph_break_is_not_mistaken_for_a_kernel_failure(nf4_kernels, monkeypatch):
+ """A Dynamo exception surfacing through fast_dequantize (e.g. an older Dynamo tracing the
+ fallback's try/except) must re-raise, not switch the kernels off for the process."""
+ q, s = _quantize((512, 256), torch.bfloat16)
+
+ def unsupported(*args, **kwargs):
+ raise torch._dynamo.exc.Unsupported("Data-dependent branching")
+
+ monkeypatch.setattr(U, "dequantize_nf4", unsupported)
+ with pytest.raises(torch._dynamo.exc.Unsupported):
+ U.fast_dequantize(q, s)
+ assert U._USE_NF4_KERNELS is True
+
+
+# Runs as a module-level script: only that shape reproduced the torch 2.10 stale reads.
+_STALE_MEMORY_SCRIPT = """
+import sys, torch
+sys.path.insert(0, sys.argv[1])
+from unsloth.kernels import fast_lora
+import test_bnb_integration_compile as t
+
+model, block = t._lora_block(dtype = getattr(torch, sys.argv[3]))
+fn = lambda x: getattr(fast_lora, sys.argv[2])(block, x)
+X = torch.randn(2, 16, 256, dtype = getattr(torch, sys.argv[3]), device = "cuda")
+compiled = torch.compile(fn)
+eager = t._fwd_bwd(model, fn, X)
+bad = 0
+for _ in range(10):
+ poison = [torch.full((n,), float("nan"), device = "cuda") for n in (2048, 4096, 8192, 16384, 65536, 131072)]
+ del poison
+ # The previous step's outputs stay alive until this one returns, as in a training loop.
+ out = t._fwd_bwd(model, compiled, X)
+ bad += sum((~torch.isfinite(o)).sum().item() for o in out)
+print("NONFINITE", bad)
+"""
+
+
[email protected]("mlp", ["swiglu", "geglu_exact", "geglu_approx"])
+def test_compiled_lora_mlp_backward_never_reads_stale_memory(mlp):
+ """torch 2.10 miscompiles the traced LoRA_MLP backward: with freed NaN memory around, LoRA
+ gradients came back non-finite. There the LoRA Functions must stay opaque to torch.compile."""
+ import os, subprocess, sys
+
+ tests_dir = os.path.dirname(os.path.abspath(__file__))
+ run = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ _STALE_MEMORY_SCRIPT,
+ tests_dir,
+ f"apply_lora_mlp_{mlp}",
+ str(CDTYPE).split(".")[-1],
+ ],
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ )
+ assert "NONFINITE 0" in run.stdout, run.stdout[-2000:] + run.stderr[-3000:]
diff --git a/tests/test_lora_traced_inplace.py b/tests/test_lora_traced_inplace.py
new file mode 100644
index 00000000000..1bf9f0078f3
--- /dev/null
+++ b/tests/test_lora_traced_inplace.py
@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Traced LoRA backward and forward without copies: the accumulating addmm_ runs in place under
+torch.compile (unsloth::addmm_), and the SwiGLU backward is plain torch math there, so Inductor
+neither copies the accumulator nor the saved activations. Both must match eager."""
+
+import pytest
+import torch
+
+if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA GPU", allow_module_level = True)
+pytest.importorskip("triton")
+
+from unsloth.kernels import utils
+from unsloth.kernels.swiglu import swiglu_DWf_DW_dfg_kernel
+
+
+def _inputs(seed = 0):
+ g = torch.Generator(device = "cuda").manual_seed(seed)
+ d = dict(device = "cuda", dtype = torch.bfloat16, generator = g)
+ return (
+ torch.randn(64, 256, **d),
+ torch.randn(512, 256, **d) / 16,
+ torch.randn(8, 256, **d) / 16,
+ torch.randn(512, 8, **d) / 16,
+ )
+
+
+def _lora_forward(X, W, A, B):
+ out = X @ W.t()
+ return utils.addmm_(out, X @ A.t(), B.t(), alpha = 2.0)
+
+
[email protected](not utils._HAS_ADDMM_OP, reason = "torch.library.custom_op needs torch 2.4+")
+def test_traced_addmm_is_the_in_place_op_and_matches_eager():
+ torch._dynamo.reset()
+ graphs = []
+
+ def backend(gm, example_inputs):
+ graphs.append(gm)
+ return gm
+
+ X, W, A, B = _inputs()
+ with torch.no_grad():
+ compiled = torch.compile(_lora_forward, backend = backend, fullgraph = True)(X, W, A, B)
+ inductor = torch.compile(_lora_forward, fullgraph = True)(X, W, A, B)
+ eager = _lora_forward(X, W, A, B)
+ targets = [node.target for gm in graphs for node in gm.graph.nodes]
+ assert torch.ops.unsloth.addmm_.default in targets or torch.ops.unsloth.addmm_ in targets
+ assert torch.equal(compiled, eager) and torch.equal(inductor, eager)
+
+
+def test_addmm_keeps_autograd_with_grad_enabled():
+ # The custom op has no derivative; with grad enabled the plain addmm_ must be used.
+ torch._dynamo.reset()
+ X, W, A, B = _inputs(1)
+ W.requires_grad_(True)
+ out = torch.compile(_lora_forward, fullgraph = True)(X, W, A, B)
+ out.float().sum().backward()
+ assert W.grad is not None and torch.isfinite(W.grad).all()
+
+
+# bf16 only where Unsloth trains in it (compute capability 8+); T4 trains in fp16.
+DTYPES = [torch.float16] + ([torch.bfloat16] if torch.cuda.get_device_capability()[0] >= 8 else [])
+
+
[email protected]("dtype", DTYPES, ids = str)
+def test_traced_swiglu_backward_matches_the_kernel(dtype):
+ torch._dynamo.reset()
+ g = torch.Generator(device = "cuda").manual_seed(2)
+ d = dict(device = "cuda", dtype = dtype, generator = g)
+ DW, e, gate = (
+ torch.randn(256, 1024, **d),
+ torch.randn(256, 1024, **d),
+ torch.randn(256, 1024, **d),
+ )
+ ref = swiglu_DWf_DW_dfg_kernel(DW.clone(), e.clone(), gate.clone())
+ traced = torch.compile(swiglu_DWf_DW_dfg_kernel, fullgraph = True)(DW, e, gate)
+ for name, a, b in zip(("h", "df", "de"), traced, ref):
+ # Inductor keeps the intermediate products in fp32 where the kernel rounds them to dtype.
+ err = ((a.float() - b.float()).abs().max() / b.float().abs().max()).item()
+ assert err < 1e-2, (name, err)
diff --git a/tests/test_nf4_triton_dequant.py b/tests/test_nf4_triton_dequant.py
new file mode 100644
index 00000000000..6cc2f1835f6
--- /dev/null
+++ b/tests/test_nf4_triton_dequant.py
@@ -0,0 +1,317 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""The fused Triton NF4 dequant kernel must be byte-identical to bitsandbytes. Streams, CUDA
+graphs and torch.compile through fast_dequantize are covered in test_bnb_integration_compile.py."""
+
+import hashlib
+import os
+
+import pytest
+import torch
+
+if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA device", allow_module_level = True)
+
+pytest.importorskip("triton")
+bnb_functional = pytest.importorskip("bitsandbytes.functional")
+
+from unsloth.kernels import nf4 as nf4_mod
+from unsloth.kernels import nf4_gemv as nf4_gemv_mod
+from unsloth.kernels.nf4 import dequantize_nf4
+
+_INT_VIEW = {torch.float16: torch.int16, torch.bfloat16: torch.int16, torch.float32: torch.int32}
+
+# nf4_dequantization_lut in bitsandbytes csrc/kernels.cu.
+CSRC_NF4 = (
+ -1.0,
+ -0.6961928009986877,
+ -0.5250730514526367,
+ -0.39491748809814453,
+ -0.28444138169288635,
+ -0.18477343022823334,
+ -0.09105003625154495,
+ 0.0,
+ 0.07958029955625534,
+ 0.16093020141124725,
+ 0.24611230194568634,
+ 0.33791524171829224,
+ 0.44070982933044434,
+ 0.5626170039176941,
+ 0.7229568362236023,
+ 1.0,
+)
+
+
+def _bytes_equal(a, b):
+ assert a.shape == b.shape and a.dtype == b.dtype
+ iv = _INT_VIEW[a.dtype]
+ return torch.equal(a.contiguous().view(iv), b.contiguous().view(iv))
+
+
+def _quantize(
+ shape,
+ dtype,
+ blocksize = 64,
+ nested = True,
+ seed = 0,
+):
+ g = torch.Generator(device = "cuda").manual_seed(seed)
+ W = torch.randn(shape, dtype = dtype, device = "cuda", generator = g)
+ return bnb_functional.quantize_4bit(
+ W, blocksize = blocksize, quant_type = "nf4", compress_statistics = nested
+ )
+
+
+def _args(q, s):
+ if s.nested:
+ return (
+ q,
+ s.absmax,
+ s.state2.code,
+ s.state2.absmax,
+ s.offset,
+ s.code,
+ s.blocksize,
+ s.state2.blocksize,
+ s.shape,
+ s.dtype,
+ )
+ return (q, s.absmax, None, None, None, s.code, s.blocksize, 0, s.shape, s.dtype)
+
+
+SHAPES = [(1, 4096), (4096, 1), (17, 33), (4096, 4096), (14336, 4096), (4096, 14336)]
+
+
[email protected]("shape", SHAPES, ids = lambda s: f"{s[0]}x{s[1]}")
[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32], ids = str)
[email protected]("blocksize", [64, 128])
[email protected]("nested", [True, False], ids = ["nested", "flat"])
+def test_bytes_match_bitsandbytes(shape, dtype, blocksize, nested):
+ q, s = _quantize(shape, dtype, blocksize, nested)
+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), bnb_functional.dequantize_4bit(q, s))
+
+
+def test_vocab_sized_weight_matches_bitsandbytes():
+ if torch.cuda.mem_get_info()[0] < 8 * 2**30:
+ pytest.skip("not enough free GPU memory for the vocab-sized weight")
+ q, s = _quantize((128256, 4096), torch.bfloat16)
+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), bnb_functional.dequantize_4bit(q, s))
+
+
+def test_quant_state_code_is_the_kernel_table():
+ """fast_dequantize passes quant_state.code as the table; bitsandbytes' own kernel uses the
+ hardcoded csrc table, so the two must be the same fp32 values."""
+ for dtype in (torch.float16, torch.bfloat16):
+ _, s = _quantize((64, 64), dtype)
+ csrc = torch.tensor(CSRC_NF4, dtype = torch.float32)
+ assert torch.equal(s.code.cpu().view(torch.int32), csrc.view(torch.int32))
+
+
+def test_explicit_out_is_written_and_returned():
+ q, s = _quantize((256, 512), torch.bfloat16)
+ out = torch.full(s.shape, 7.0, dtype = s.dtype, device = "cuda")
+ assert dequantize_nf4(*_args(q, s), out = out).data_ptr() == out.data_ptr()
+ assert _bytes_equal(out, bnb_functional.dequantize_4bit(q, s))
+
+
+def test_compiled_out_variant():
+ q, s = _quantize((256, 512), torch.float16, seed = 8)
+
+ def f(q, absmax, out):
+ args = list(_args(q, s))
+ args[1] = absmax
+ dequantize_nf4(*args, out = out)
+ return out * 1
+
+ torch._dynamo.reset()
+ out = torch.empty(s.shape, dtype = s.dtype, device = "cuda")
+ res = torch.compile(f, fullgraph = True)(q, s.absmax, out)
+ ref = bnb_functional.dequantize_4bit(q, s)
+ assert _bytes_equal(out, ref) and _bytes_equal(res, ref)
+
+
+def test_rejects_unsupported_dtype_and_blocksize():
+ q, s = _quantize((64, 64), torch.bfloat16)
+ args = list(_args(q, s))
+ with pytest.raises(TypeError):
+ dequantize_nf4(*args[:-1], torch.int8)
+ args[6] = 48
+ with pytest.raises(ValueError):
+ dequantize_nf4(*args)
+
+
+def test_fma_contraction_would_be_caught():
+ """Control: a contracted fma(code2, absmax2, offset) differs from the two-rounding result
+ somewhere, so the bytes test above can tell a fused kernel from a correct one."""
+ _, s = _quantize((4096, 4096), torch.float32)
+ code2, absmax2 = s.state2.code, s.state2.absmax
+ blk2 = torch.arange(s.absmax.numel(), device = "cuda") // s.state2.blocksize
+ two_roundings = (code2[s.absmax.long()] * absmax2[blk2]) + s.offset
+ one_rounding = (
+ code2[s.absmax.long()].double() * absmax2[blk2].double() + s.offset.double()
+ ).float()
+ assert not torch.equal(two_roundings, one_rounding)
+
+
[email protected](not nf4_mod._HAS_MUL_RN, reason = "already the no-mul_rn route")
+def test_hip_route_without_mul_rn_is_also_exact(monkeypatch):
+ """The HIP route (plain multiply, fp fusion disabled at launch) exercised on CUDA."""
+ monkeypatch.setattr(nf4_mod, "_HAS_MUL_RN", False)
+ for dtype in (torch.float32, torch.bfloat16):
+ q, s = _quantize((4096, 4096), dtype, seed = 3)
+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), bnb_functional.dequantize_4bit(q, s))
+
+
[email protected]("fp_fusion", [False, True], ids = ["fusion_off", "fusion_on"])
+def test_exact_with_fp_fusion_on_or_off(fp_fusion):
+ """Eager launches with fp fusion off; torch.compile re-emits the kernel with it on. Both must
+ keep code2 * absmax2 and + offset as two roundings (fp32 output shows a 1 ulp scale error)."""
+ if fp_fusion and not nf4_mod._HAS_MUL_RN:
+ # Without libdevice.mul_rn (HIP, the interpreter) the kernel never runs with fusion on.
+ pytest.skip("no mul_rn: this route never launches with fp fusion on")
+ for dtype in (torch.float32, torch.bfloat16):
+ q, s = _quantize((4096, 4096), dtype, seed = 3)
+ out = torch.empty(4096, 4096, dtype = dtype, device = "cuda")
+ nf4_mod._launch(
+ nf4_mod._nf4_dequant_kernel,
+ q,
+ s.absmax,
+ s.state2.code,
+ s.state2.absmax,
+ s.offset,
+ s.code,
+ s.blocksize,
+ s.state2.blocksize,
+ out,
+ fp_fusion = fp_fusion,
+ )
+ assert _bytes_equal(out, bnb_functional.dequantize_4bit(q, s))
+
+
[email protected](not nf4_mod._HAS_MUL_RN, reason = "the plain multiply keeps subnormals")
[email protected]("nested", [True, False], ids = ["nested", "flat"])
+def test_subnormal_products_flush_like_bitsandbytes(nested):
+ """bitsandbytes' kernels flush subnormal products to zero, in the nested absmax decode and in
+ code * absmax; the Triton kernel must too."""
+ q, s = _quantize((256, 1024), torch.float32, nested = nested, seed = 5)
+ if nested:
+ s.state2.absmax.mul_(1e-36)
+ s.offset.zero_()
+ else:
+ s.absmax.fill_(2e-38)
+ ref = bnb_functional.dequantize_4bit(q, s)
+ assert ref.abs().max() > 0 and (ref == 0).float().mean() > 0.1
+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), ref)
+
+
[email protected]("storage", [torch.bfloat16, torch.float16, torch.float32], ids = str)
+def test_float_packed_storage_is_read_as_bytes(storage):
+ """FSDP-QLoRA packs the 4bit weight into float storage (quant_storage); it is still bytes."""
+ g = torch.Generator(device = "cuda").manual_seed(4)
+ W = torch.randn(512, 256, dtype = torch.bfloat16, device = "cuda", generator = g)
+ q, s = bnb_functional.quantize_4bit(
+ W, quant_type = "nf4", compress_statistics = True, quant_storage = storage
+ )
+ assert q.dtype == storage
+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), bnb_functional.dequantize_4bit(q, s))
+
+
[email protected]("lut_mode", [0, 1, 2])
[email protected]("words", [False, True])
[email protected]("evict", [False, True])
[email protected]("shape", [(17, 33), (5, 4096), (1024, 4096)], ids = str)
+def test_every_launch_config_is_exact(shape, words, evict, lut_mode, monkeypatch):
+ # The per-GPU launch table only picks speed; every knob combination must stay bit-exact.
+ for dtype in (torch.float16, torch.bfloat16):
+ for blocksize in (64, 128):
+ q, s = _quantize(shape, dtype, blocksize)
+ ref = bnb_functional.dequantize_4bit(q, s)
+ for target in (256, 2048):
+ monkeypatch.setattr(
+ nf4_mod, "_CONFIG_OVERRIDE", (target, 4, words, evict, lut_mode)
+ )
+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), ref), (dtype, blocksize, target)
+
+
+def test_every_table_config_is_exact_here(monkeypatch):
+ # Each GPU's table entry, run on this GPU: exactness does not depend on the GPU it was tuned on.
+ configs = {row[1:] for table in nf4_mod._CONFIGS.values() for row in table}
+ for shape in [(17, 33), (1024, 4096)]:
+ for dtype in (torch.float16, torch.bfloat16):
+ q, s = _quantize(shape, dtype, 64)
+ ref = bnb_functional.dequantize_4bit(q, s)
+ for config in sorted(configs):
+ monkeypatch.setattr(nf4_mod, "_CONFIG_OVERRIDE", config)
+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), ref), (shape, dtype, config)
+
+
+def test_compile_cache_key_covers_the_kernel_source():
+ """Inductor's FX graph cache keys a triton_op call without the Triton source behind it: with a
+ warm cache, an edited kernel came back as the previous kernel's code. The kernel files' hashes
+ in cache_key_tag make an edited kernel a cache miss."""
+ if not hasattr(getattr(torch.compiler, "config", None), "cache_key_tag"):
+ pytest.skip("torch has no compile cache key tag")
+ tags = torch.compiler.config.cache_key_tag.split(",")
+ for mod in (nf4_mod, nf4_gemv_mod):
+ with open(mod.__file__, "rb") as file:
+ digest = hashlib.sha256(file.read()).hexdigest()[:16]
+ assert f"unsloth/{os.path.basename(mod.__file__)}:{digest}" in tags
+
+
+def _utils_with_kernels():
+ from unsloth.kernels import utils
+ if not utils._USE_NF4_KERNELS:
+ pytest.skip("the fused NF4 kernels are off here (UNSLOTH_BNB_TRITON=0 or no Triton)")
+ return utils
+
+
[email protected]("nested", [True, False], ids = ["nested", "flat"])
+def test_decode_plan_reruns_match_bitsandbytes(nested):
+ """Decode dequantizes each weight into the shared scratch buffer through a launch planned on
+ its first call. Every rerun, weights interleaved as in a decoder, stays byte-identical."""
+ from unsloth.kernels import triton_launch
+
+ utils = _utils_with_kernels()
+ shapes = [(512, 256), (256, 1024), (768, 256)]
+ weights = [_quantize(s, torch.bfloat16, nested = nested, seed = i) for i, s in enumerate(shapes)]
+ for _ in range(3):
+ for q, s in weights:
+ got = utils.fast_dequantize(q.t(), s, use_global_buffer = True).t().clone()
+ assert _bytes_equal(got, bnb_functional.dequantize_4bit(q, s))
+ if triton_launch._ENABLED:
+ assert all(getattr(s, "_unsloth_nf4_plan", None) for _, s in weights)
+
+
+def test_decode_plan_is_rebuilt_when_the_quant_state_changes():
+ """QuantState.to() swaps in new tensors and a model cast changes its dtype: the planned launch
+ would read the old absmax or write the old dtype, so it must be rebuilt."""
+ utils = _utils_with_kernels()
+ q, s = _quantize((512, 256), torch.bfloat16, nested = False)
+ for _ in range(2):
+ utils.fast_dequantize(q.t(), s, use_global_buffer = True)
+ s.absmax = s.absmax * 2
+ got = utils.fast_dequantize(q.t(), s, use_global_buffer = True).t().clone()
+ assert _bytes_equal(got, bnb_functional.dequantize_4bit(q, s))
+ s.dtype = torch.float16
+ got = utils.fast_dequantize(q.t(), s, use_global_buffer = True).t().clone()
+ assert got.dtype == torch.float16
+ assert _bytes_equal(got, bnb_functional.dequantize_4bit(q, s))
+
+
+def test_decode_plan_is_not_copied_or_pickled_with_the_model():
+ """The plan holds a closure and the scratch buffer: deepcopy (reference models) and pickling
+ must drop it, and the copy rebuilds its own on first use."""
+ import copy
+ import pickle
+
+ utils = _utils_with_kernels()
+ q, s = _quantize((512, 256), torch.bfloat16)
+ for _ in range(2):
+ utils.fast_dequantize(q.t(), s, use_global_buffer = True)
+ s2 = copy.deepcopy(s)
+ assert getattr(s2, "_unsloth_nf4_plan", None) is None
+ assert getattr(pickle.loads(pickle.dumps(s)), "_unsloth_nf4_plan", None) is None
+ q2 = q.clone()
+ got = utils.fast_dequantize(q2.t(), s2, use_global_buffer = True).t().clone()
+ assert _bytes_equal(got, bnb_functional.dequantize_4bit(q, s))
diff --git a/tests/test_nf4_triton_gemv.py b/tests/test_nf4_triton_gemv.py
new file mode 100644
index 00000000000..7778639f2e3
--- /dev/null
+++ b/tests/test_nf4_triton_gemv.py
@@ -0,0 +1,169 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Batch-1 NF4 GEMV kernel (unsloth/kernels/nf4_gemv.py). It reduces each quantization block
+before scaling it, so it is held to an fp32 reference with a tolerance and must never be worse
+than bitsandbytes' own GEMV. Streams, CUDA graphs and torch.compile through fast_gemv are covered
+in test_bnb_integration_compile.py."""
+
+import pytest
+import torch
+
+if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA GPU", allow_module_level = True)
+pytest.importorskip("triton")
+F = pytest.importorskip("bitsandbytes.functional")
+
+from unsloth.kernels import nf4_gemv
+from unsloth.kernels.nf4_gemv import gemv_nf4
+from unsloth.kernels.utils import _fast_gemv_ctypes
+
+SHAPES = [(4096, 4096), (14336, 4096), (4096, 14336), (1024, 4096)]
+DTYPES = [torch.float16, torch.bfloat16]
+
+
+def _quant(
+ n,
+ k,
+ dtype,
+ nested = True,
+ seed = 0,
+):
+ g = torch.Generator(device = "cuda").manual_seed(seed)
+ W = torch.randn(n, k, dtype = dtype, device = "cuda", generator = g)
+ return F.quantize_4bit(W, quant_type = "nf4", compress_statistics = nested)
+
+
+def _args(q, s):
+ if s.nested:
+ return (
+ q,
+ s.absmax,
+ s.state2.code,
+ s.state2.absmax,
+ s.offset,
+ s.code,
+ s.blocksize,
+ s.state2.blocksize,
+ s.shape,
+ s.dtype,
+ )
+ return (q, s.absmax, None, None, None, s.code, s.blocksize, None, s.shape, s.dtype)
+
+
+def _reference(X, q, s):
+ n, k = s.shape
+ return (X.float().view(1, k) @ F.dequantize_4bit(q, s).float().t()).view(1, 1, n)
+
+
+def _rel_err(a, ref):
+ return ((a.float() - ref).abs().max() / ref.abs().max()).item()
+
+
+# bf16 output rounding alone is about 4e-3 relative; bitsandbytes lands at the same level.
+TOL = {torch.float16: 2e-3, torch.bfloat16: 1.2e-2}
+
+
[email protected](params = [None, "narrow", "bytes", "words"], ids = ["auto", "narrow", "bytes", "words"])
+def kernel(request, monkeypatch):
+ # Both kernel forms, whichever this Triton version and GPU would pick on its own, and the
+ # T4 / L4 large-weight config on this GPU.
+ if request.param == "narrow":
+ capability = torch.cuda.get_device_capability()
+ monkeypatch.setattr(nf4_gemv, "_NARROW_CAPS", (capability,))
+ nf4_gemv._gemv_config.cache_clear()
+ request.addfinalizer(nf4_gemv._gemv_config.cache_clear)
+ else:
+ monkeypatch.setattr(nf4_gemv, "_FORCE_KERNEL", request.param)
+ return request.param
+
+
[email protected]("nested", [True, False], ids = ["nested", "flat"])
[email protected]("dtype", DTYPES, ids = str)
[email protected]("shape", SHAPES, ids = lambda s: f"{s[0]}x{s[1]}")
+def test_matches_fp32_reference_and_bitsandbytes(shape, dtype, nested, kernel):
+ n, k = shape
+ q, s = _quant(n, k, dtype, nested)
+ X = torch.randn(1, 1, k, dtype = dtype, device = "cuda")
+ ref = _reference(X, q, s)
+ out = gemv_nf4(X, *_args(q, s))
+ assert out.shape == (1, 1, n) and out.dtype == dtype
+ assert _rel_err(out, ref) < TOL[dtype]
+ if nested: # the ctypes GEMV only takes double quantized states
+ assert _rel_err(out, ref) <= _rel_err(_fast_gemv_ctypes(X, q, s), ref) + 4e-3
+
+
+def test_unaligned_weight_view_uses_the_byte_kernel(monkeypatch):
+ # The words kernel reads int32; a weight view starting off a 4 byte boundary must not reach it.
+ monkeypatch.setattr(nf4_gemv, "_FORCE_KERNEL", "words")
+ q, s = _quant(1024, 4096, torch.float16)
+ buf = torch.empty(q.numel() + 2, dtype = torch.uint8, device = "cuda")
+ buf[2:].copy_(q.view(-1))
+ Wv = buf[2:].view(q.shape)
+ assert not nf4_gemv._word_aligned(Wv)
+ X = torch.randn(1, 1, 4096, dtype = torch.float16, device = "cuda")
+ assert _rel_err(gemv_nf4(X, Wv, *_args(q, s)[1:]), _reference(X, q, s)) < TOL[torch.float16]
+
+
+def test_vocab_sized_weight():
+ q, s = _quant(128256, 4096, torch.bfloat16)
+ X = torch.randn(1, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ assert _rel_err(gemv_nf4(X, *_args(q, s)), _reference(X, q, s)) < TOL[torch.bfloat16]
+
+
+def test_explicit_out_is_written_and_returned():
+ q, s = _quant(1024, 4096, torch.bfloat16)
+ X = torch.randn(1, 1, 4096, dtype = torch.bfloat16, device = "cuda")
+ out = torch.empty(1, 1, 1024, dtype = torch.bfloat16, device = "cuda")
+ assert gemv_nf4(X, *_args(q, s), out = out).data_ptr() == out.data_ptr()
+ assert torch.equal(out, gemv_nf4(X, *_args(q, s)))
+
+
+def _utils_with_kernels():
+ from unsloth.kernels import utils
+ if not (utils._USE_NF4_KERNELS and utils._TRITON_GEMV_EAGER):
+ pytest.skip("eager decode does not take the Triton GEMV here")
+ return utils
+
+
[email protected]("nested", [True, False], ids = ["nested", "flat"])
+def test_decode_plan_reruns_match_the_kernel(nested):
+ """Eager decode repeats the launch planned on each weight's first call. With weights
+ interleaved as in a decoder, every rerun equals a fresh gemv_nf4 launch bit for bit."""
+ from unsloth.kernels import triton_launch
+
+ utils = _utils_with_kernels()
+ weights = [
+ _quant(n, k, torch.bfloat16, nested = nested, seed = i)
+ for i, (n, k) in enumerate([(512, 256), (256, 1024), (768, 512)])
+ ]
+ for step in range(3):
+ for i, (q, s) in enumerate(weights):
+ X = torch.randn(1, 1, s.shape[1], dtype = torch.bfloat16, device = "cuda")
+ got = utils.fast_gemv(X, q, s)
+ want = gemv_nf4(X, *_args(q, s))
+ assert torch.equal(got, want), (step, i)
+ if triton_launch._ENABLED:
+ assert all(getattr(s, "_unsloth_gemv_plan", None) for _, s in weights)
+
+
+def test_decode_plan_is_rebuilt_or_bypassed_when_inputs_change():
+ """New absmax (QuantState.to), another activation dtype, or an activation that is not 16 byte
+ aligned would not fit the planned launch: each must still give the kernel's own result."""
+ import copy
+ import pickle
+
+ utils = _utils_with_kernels()
+ q, s = _quant(512, 256, torch.bfloat16, nested = False)
+ X = torch.randn(1, 1, 256, dtype = torch.bfloat16, device = "cuda")
+ for _ in range(2):
+ utils.fast_gemv(X, q, s)
+ s.absmax = s.absmax * 2
+ assert torch.equal(utils.fast_gemv(X, q, s), gemv_nf4(X, *_args(q, s)))
+ X16 = X.half()
+ assert torch.equal(utils.fast_gemv(X16, q, s), gemv_nf4(X16, *_args(q, s)))
+ base = torch.randn(1, 1, 257, dtype = torch.bfloat16, device = "cuda")
+ Xm = base[:, :, 1:] # 2 bytes past a 16 byte boundary
+ assert torch.equal(utils.fast_gemv(Xm, q, s), gemv_nf4(Xm.contiguous(), *_args(q, s)))
+ assert getattr(copy.deepcopy(s), "_unsloth_gemv_plan", None) is None
+ assert getattr(pickle.loads(pickle.dumps(s)), "_unsloth_gemv_plan", None) is None
diff --git a/tests/test_triton_launch.py b/tests/test_triton_launch.py
new file mode 100644
index 00000000000..ee678379cb6
--- /dev/null
+++ b/tests/test_triton_launch.py
@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""unsloth/kernels/triton_launch.py launches a cached compiled Triton kernel directly. It must give
+the same result as Triton's own launch for every specialization (alignment, integers equal to 1 or
+divisible by 16), launch on the current stream, and step aside when a launch hook is registered."""
+
+import pytest
+import torch
+
+triton = pytest.importorskip("triton")
+import triton.language as tl
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
+
+from unsloth.kernels import triton_launch
+
+
+def _axpy(x_ptr, y_ptr, out_ptr, n, scale, BLOCK: tl.constexpr, ADD: tl.constexpr):
+ i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
+ m = i < n
+ v = tl.load(x_ptr + i, mask = m) * scale
+ if ADD:
+ v += tl.load(y_ptr + i, mask = m)
+ tl.store(out_ptr + i, v, mask = m)
+
+
+def _run(
+ x,
+ y,
+ out,
+ n,
+ scale,
+ add,
+ block = 128,
+):
+ grid = (-(-n // block),)
+ triton_launch.launch(
+ _axpy,
+ grid,
+ (x, y, out, n, scale),
+ 3,
+ dict(BLOCK = block, ADD = add),
+ x.device.index,
+ num_warps = 4,
+ )
+ return out
+
+
[email protected](autouse = True)
+def _fresh_cache(monkeypatch):
+ monkeypatch.setattr(triton_launch, "_CACHE", {})
+ monkeypatch.setattr(
+ triton_launch,
+ "_ENABLED",
+ triton_launch._raw_stream is not None and torch.version.hip is None,
+ )
+
+
[email protected]("n", [1, 16, 17, 1000, 4096])
[email protected]("offset", [0, 1, 4])
[email protected]("add", [False, True])
+def test_matches_reference_for_every_specialization(n, offset, add):
+ base = torch.randn(n + 8, device = "cuda")
+ x = base[offset : offset + n]
+ y = torch.randn(n, device = "cuda")
+ for scale in (1, 3, 16):
+ # Reference: Triton's own launch of the same kernel (x * scale + y may be one FMA).
+ ref = torch.empty(n, device = "cuda")
+ _axpy[(-(-n // 128),)](x, y, ref, n, scale, BLOCK = 128, ADD = add, num_warps = 4)
+ for _ in range(3): # first call compiles, the others take the cached kernel
+ out = _run(x, y, torch.empty(n, device = "cuda"), n, scale, add)
+ torch.testing.assert_close(out, ref, rtol = 0, atol = 0)
+
+
+def test_second_call_uses_the_cache_and_distinct_specializations_get_their_own_entry():
+ if not triton_launch._ENABLED:
+ pytest.skip("fast launch is CUDA only")
+ x = torch.randn(64, device = "cuda")
+ compiled = _axpy[(1,)](x, x, torch.empty(64, device = "cuda"), 64, 2, BLOCK = 128, ADD = False)
+ if not hasattr(compiled, "packed_metadata"):
+ pytest.skip("this Triton returns no packed_metadata, so every launch takes its own path")
+ _run(x, x, torch.empty(64, device = "cuda"), 64, 2, False)
+ assert len(triton_launch._CACHE) == 1
+ _run(x, x, torch.empty(64, device = "cuda"), 64, 2, False)
+ assert len(triton_launch._CACHE) == 1
+ _run(x[1:], x[1:], torch.empty(63, device = "cuda"), 63, 1, False)
+ assert len(triton_launch._CACHE) == 2
+
+
+def test_runs_on_the_current_stream():
+ x = torch.randn(1 << 20, device = "cuda")
+ side = torch.cuda.Stream()
+ for _ in range(2):
+ out = torch.zeros_like(x)
+ with torch.cuda.stream(side):
+ torch.cuda._sleep(1_000_000)
+ _run(x, x, out, x.numel(), 2, False)
+ side.synchronize()
+ torch.testing.assert_close(out, x * 2)
+
+
+def test_registered_hook_keeps_tritons_own_launch(monkeypatch):
+ knobs = getattr(triton, "knobs", None)
+ if knobs is None or not hasattr(knobs.runtime.launch_enter_hook, "add"):
+ pytest.skip("this Triton has no HookChain launch hooks")
+ x = torch.randn(64, device = "cuda")
+ _run(x, x, torch.empty(64, device = "cuda"), 64, 2, False)
+ seen = []
+ hook = lambda metadata: seen.append(1)
+ knobs.runtime.launch_enter_hook.add(hook)
+ try:
+ out = _run(x, x, torch.empty(64, device = "cuda"), 64, 2, False)
+ finally:
+ knobs.runtime.launch_enter_hook.remove(hook)
+ torch.testing.assert_close(out, x * 2)
+ assert seen, "a registered launch hook must still be called"
+
+
+def test_relaunch_repeats_the_entry_and_yields_to_hooks(monkeypatch):
+ x = torch.randn(64, device = "cuda")
+ grid = (1,)
+ entry = triton_launch.launch(
+ _axpy,
+ grid,
+ (x, x, torch.empty(64, device = "cuda"), 64, 2),
+ 3,
+ dict(BLOCK = 128, ADD = False),
+ x.device.index,
+ num_warps = 4,
+ )
+ if entry is None:
+ pytest.skip("no fast launch entry on this build")
+ y = torch.randn(64, device = "cuda")
+ out = torch.empty(64, device = "cuda")
+ assert triton_launch.relaunch(entry, grid, x.device.index, (y, y, out, 64, 2))
+ torch.testing.assert_close(out, y * 2)
+ # A profiler hook must see every launch, so relaunch hands the call back to Triton's path.
+ monkeypatch.setattr(triton_launch, "_hooks_set", lambda: True)
+ assert not triton_launch.relaunch(entry, grid, x.device.index, (y, y, out, 64, 2))
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 50e44fc9ca2..36042f4efd0 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -10,9 +10,11 @@
# limitations under the License.
import torch
+from unsloth_zoo.utils import Version
from .utils import (
_has_multiple_active_adapters,
_maybe_fake_quantize_activations,
+ addmm_,
fast_dequantize,
QUANT_STATE,
get_lora_parameters,
@@ -22,6 +24,19 @@ from .utils import (
torch_amp_custom_bwd,
)
+_is_compiling = torch.compiler.is_compiling
+
+# Inductor before torch 2.11 miscompiles these Functions' traced backward (wrong LoRA gradients).
+TRACE_LORA_FUNCTIONS = Version(torch.__version__) >= Version("2.11.0")
+
+
+def _apply(function, *args):
+ return function.apply(*args)
+
+
+if not TRACE_LORA_FUNCTIONS:
+ _apply = torch._dynamo.disable(_apply)
+
class LoRA_MLP(torch.autograd.Function):
"""
@@ -179,14 +194,15 @@ class LoRA_MLP(torch.autograd.Function):
# dX = matmul_lora(df, upW.t(), ...) + matmul_lora(de, gateW.t(), ...), expanded below.
upW = fast_dequantize(upW.t(), upW_quant)
- dX = torch.matmul(df, upW.t(), out = X if ctx.inplace else None)
+ # Eager only: AOT autograd rejects a backward mutating a forward input that requires grad.
+ dX = torch.matmul(df, upW.t(), out = X if ctx.inplace and not _is_compiling() else None)
del upW
- dX.addmm_(up_dB, upA.t(), alpha = upS)
+ addmm_(dX, up_dB, upA.t(), alpha = upS)
gateW = fast_dequantize(gateW.t(), gateW_quant)
- dX.addmm_(de, gateW.t())
+ addmm_(dX, de, gateW.t())
del gateW
- dX.addmm_(gate_dB, gateA.t(), alpha = gateS)
+ addmm_(dX, gate_dB, gateA.t(), alpha = gateS)
return (
dX.view(batch, seq_len, hd),
@@ -228,7 +244,8 @@ def apply_lora_mlp_swiglu(
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)
- out = LoRA_MLP.apply(
+ out = _apply(
+ LoRA_MLP,
X,
gateW,
gateW_quant,
@@ -269,7 +286,8 @@ def apply_lora_mlp_geglu_exact(
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)
- out = LoRA_MLP.apply(
+ out = _apply(
+ LoRA_MLP,
X,
gateW,
gateW_quant,
@@ -306,7 +324,8 @@ def apply_lora_mlp_geglu_approx(self, X):
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)
- out = LoRA_MLP.apply(
+ out = _apply(
+ LoRA_MLP,
X,
gateW,
gateW_quant,
@@ -480,19 +499,19 @@ class LoRA_QKV(torch.autograd.Function):
# Combine the per-projection derivatives into dX.
QW = fast_dequantize(QW.t(), QW_quant)
- dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace else None)
+ dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace and not _is_compiling() else None)
del QW
- dX.addmm_(q_dB, QA.t(), alpha = QS)
+ addmm_(dX, q_dB, QA.t(), alpha = QS)
KW = fast_dequantize(KW.t(), KW_quant)
- dX.addmm_(dK, KW.t())
+ addmm_(dX, dK, KW.t())
del KW
- dX.addmm_(k_dB, KA.t(), alpha = KS)
+ addmm_(dX, k_dB, KA.t(), alpha = KS)
VW = fast_dequantize(VW.t(), VW_quant)
- dX.addmm_(dV, VW.t())
+ addmm_(dX, dV, VW.t())
del VW
- dX.addmm_(v_dB, VA.t(), alpha = VS)
+ addmm_(dX, v_dB, VA.t(), alpha = VS)
return (
dX.view(batch, seq_len, hd),
@@ -526,7 +545,8 @@ def apply_lora_qkv(
QW, QW_quant, QA, QB, QS = get_lora_parameters(self.q_proj)
KW, KW_quant, KA, KB, KS = get_lora_parameters(self.k_proj)
VW, VW_quant, VA, VB, VS = get_lora_parameters(self.v_proj)
- Q, K, V = LoRA_QKV.apply(
+ Q, K, V = _apply(
+ LoRA_QKV,
X,
QW,
QW_quant,
@@ -617,7 +637,7 @@ class LoRA_W(torch.autograd.Function):
W = fast_dequantize(W.t(), W_quant)
dX = dY @ W.t()
del W
- dX.addmm_(y_dB, A.t(), alpha = S)
+ addmm_(dX, y_dB, A.t(), alpha = S)
return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None
@@ -627,7 +647,7 @@ def apply_lora_o(self, X):
return self.o_proj(X)
X = _maybe_fake_quantize_activations(X, self.o_proj)
OW, OW_quant, OA, OB, OS = get_lora_parameters(self.o_proj)
- O = LoRA_W.apply(X, OW, OW_quant, OA, OB, OS)
+ O = _apply(LoRA_W, X, OW, OW_quant, OA, OB, OS)
return O
@@ -661,7 +681,7 @@ def fast_lora_forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor:
lora_B = self.lora_B[active_adapter].weight
scaling = self.scaling[active_adapter]
W = self.base_layer.weight
- return LoRA_W.apply(x, W, QUANT_STATE(W), lora_A, lora_B, scaling)
+ return _apply(LoRA_W, x, W, QUANT_STATE(W), lora_A, lora_B, scaling)
pass
pass
diff --git a/unsloth/kernels/nf4.py b/unsloth/kernels/nf4.py
new file mode 100644
index 00000000000..b2d75ea1a06
--- /dev/null
+++ b/unsloth/kernels/nf4.py
@@ -0,0 +1,449 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+# NF4 dequantization in one Triton launch on torch's current stream, bit-exact with bitsandbytes
+# kDequantizeBlockwise, which needs three launches for a double quantized weight:
+# absmax = fp32(code2[absmax_u8[k]] * absmax2[k // blocksize2]) + offset (two roundings)
+# out[j] = T(code[nibble(j)] * absmax[j // blocksize]) (high nibble first)
+
+import contextlib
+import functools
+import math
+import os
+from typing import List, Optional
+
+import torch
+import triton
+import triton.language as tl
+from triton.language.extra import libdevice
+from unsloth_zoo.utils import Version
+
+from .triton_launch import launch, relaunch, tag_compile_cache
+
+__all__ = [
+ "dequantize_nf4",
+ "dequantize_nf4_planned",
+]
+
+# libdevice mul_rn is never contracted into an FMA (Inductor re-emits kernels with fp fusion on) and
+# flushes subnormals like bitsandbytes. HIP / TRITON_INTERPRET=1 lack it: fusion off, opaque op.
+_HAS_MUL_RN = torch.version.hip is None and os.environ.get("TRITON_INTERPRET", "0") != "1"
+
+
+def _mul(a, b, USE_MUL_RN: tl.constexpr):
+ if USE_MUL_RN:
+ return libdevice.mul_rn(a, b)
+ else:
+ return a * b
+
+
+# Triton's JIT resolves every attribute in a kernel body, even in a dead constexpr branch, so
+# tl.gather (absent on Triton 3.2, torch 2.6) lives in a helper defined only where it exists.
+if hasattr(tl, "gather"):
+
+ @triton.jit
+ def _lut_gather(lut_ptr, n):
+ table = tl.load(lut_ptr + tl.arange(0, 16))
+ flat = tl.reshape(n, [n.numel])
+ return tl.reshape(tl.gather(table, flat, 0), n.shape)
+
+else:
+
+ @triton.jit
+ def _lut_gather(lut_ptr, n):
+ return tl.load(lut_ptr + n, eviction_policy = "evict_last")
+
+
+def _nf4_lut(lut_ptr, n, LUT_MODE: tl.constexpr):
+ # n: int32 nibbles. Every mode returns the same fp32 table entry, so the choice is speed only.
+ if LUT_MODE == 0:
+ # A 64-byte table gather stays in L1.
+ return tl.load(lut_ptr + n, eviction_policy = "evict_last")
+ elif LUT_MODE == 1:
+ # Select tree on the nibble bits over table values held as scalars.
+ b0 = (n & 1) != 0
+ b1 = (n & 2) != 0
+ b2 = (n & 4) != 0
+ b3 = (n & 8) != 0
+ a0 = tl.where(b0, tl.load(lut_ptr + 1), tl.load(lut_ptr + 0))
+ a1 = tl.where(b0, tl.load(lut_ptr + 3), tl.load(lut_ptr + 2))
+ a2 = tl.where(b0, tl.load(lut_ptr + 5), tl.load(lut_ptr + 4))
+ a3 = tl.where(b0, tl.load(lut_ptr + 7), tl.load(lut_ptr + 6))
+ a4 = tl.where(b0, tl.load(lut_ptr + 9), tl.load(lut_ptr + 8))
+ a5 = tl.where(b0, tl.load(lut_ptr + 11), tl.load(lut_ptr + 10))
+ a6 = tl.where(b0, tl.load(lut_ptr + 13), tl.load(lut_ptr + 12))
+ a7 = tl.where(b0, tl.load(lut_ptr + 15), tl.load(lut_ptr + 14))
+ c0 = tl.where(b1, a1, a0)
+ c1 = tl.where(b1, a3, a2)
+ c2 = tl.where(b1, a5, a4)
+ c3 = tl.where(b1, a7, a6)
+ d0 = tl.where(b2, c1, c0)
+ d1 = tl.where(b2, c3, c2)
+ return tl.where(b3, d1, d0)
+ else:
+ # Register-resident table, gathered along its only axis.
+ return _lut_gather(lut_ptr, n)
+
+
+def _nf4_dequant_kernel(
+ W_ptr, # uint8, packed: high nibble is the even element
+ absmax_ptr, # uint8 (nested) or fp32, one per absmax block
+ code2_ptr, # fp32 [256], nested only
+ absmax2_ptr, # fp32, nested only
+ offset_ptr, # fp32 [1], nested only
+ lut_ptr, # fp32 [16], the NF4 codebook
+ out_ptr,
+ n_elements,
+ n_bytes,
+ n_blocks,
+ HALF: tl.constexpr, # packed bytes per absmax block (blocksize // 2)
+ BLOCKSIZE2_SHIFT: tl.constexpr,
+ NESTED: tl.constexpr,
+ USE_MUL_RN: tl.constexpr,
+ ROWS: tl.constexpr, # absmax blocks per program
+ WORDS: tl.constexpr, # load the packed bytes as int32 words (needs n_bytes % 4 == 0)
+ EVICT: tl.constexpr, # "evict_first" streams the weight and the output past L2, else ""
+ LUT_MODE: tl.constexpr, # see _nf4_lut
+):
+ # A [ROWS, HALF] tile of packed bytes: each row is one absmax block, so its scale is
+ # decoded once per row and broadcast, instead of once per byte.
+ # FSDP-QLoRA packs the 4bit weight into float storage; it is bytes either way.
+ W_ptr = W_ptr.to(tl.pointer_type(tl.uint8))
+ pid = tl.program_id(0).to(tl.int64)
+ rows = pid * ROWS + tl.arange(0, ROWS)
+ row_mask = rows < n_blocks
+ if NESTED:
+ a_q = tl.load(absmax_ptr + rows, mask = row_mask, other = 0).to(tl.int32)
+ c2 = tl.load(code2_ptr + a_q, mask = row_mask, other = 0.0, eviction_policy = "evict_last")
+ s2 = tl.load(
+ absmax2_ptr + (rows >> BLOCKSIZE2_SHIFT),
+ mask = row_mask,
+ other = 0.0,
+ eviction_policy = "evict_last",
+ )
+ scale = _mul(c2, s2, USE_MUL_RN)
+ scale = scale + tl.load(offset_ptr)
+ else:
+ scale = tl.load(absmax_ptr + rows, mask = row_mask, other = 0.0)
+ scale = scale[:, None]
+ out_ty = out_ptr.dtype.element_ty
+
+ if WORDS:
+ # [ROWS, HALF // 4] int32 words; byte j of a word holds elements 2j (high) and 2j + 1.
+ NW: tl.constexpr = HALF // 4
+ w_offs = rows[:, None] * NW + tl.arange(0, NW)[None, :]
+ w = tl.load(
+ W_ptr.to(tl.pointer_type(tl.int32)) + w_offs,
+ mask = w_offs < (n_bytes // 4),
+ other = 0,
+ eviction_policy = EVICT,
+ )
+ i = tl.arange(0, 8)
+ shifts = (i // 2) * 8 + (1 - (i % 2)) * 4
+ nib = (w[:, :, None] >> shifts[None, None, :]) & 15
+ vals = _mul(
+ _nf4_lut(lut_ptr, tl.reshape(nib, [ROWS, 2 * HALF]), LUT_MODE), scale, USE_MUL_RN
+ )
+ vals = vals.to(out_ty)
+ else:
+ cols = tl.arange(0, HALF)
+ byte_offs = rows[:, None] * HALF + cols[None, :]
+ q = tl.load(W_ptr + byte_offs, mask = byte_offs < n_bytes, other = 0, eviction_policy = EVICT)
+ v_hi = _mul(_nf4_lut(lut_ptr, (q >> 4).to(tl.int32), LUT_MODE), scale, USE_MUL_RN).to(
+ out_ty
+ )
+ v_lo = _mul(_nf4_lut(lut_ptr, (q & 15).to(tl.int32), LUT_MODE), scale, USE_MUL_RN).to(
+ out_ty
+ )
+ vals = tl.interleave(v_hi, v_lo) # [ROWS, 2 * HALF], high nibble first
+
+ out_offs = rows[:, None] * (2 * HALF) + tl.arange(0, 2 * HALF)[None, :]
+ tl.store(out_ptr + out_offs, vals, mask = out_offs < n_elements, eviction_policy = EVICT)
+
+
+# Per compute capability (prefix match, () default) launch knobs: speed only, all bit-exact.
+_CONFIGS = {
+ (): (
+ # (max n_bytes, target bytes per program, num_warps, words, evict, lut_mode)
+ (1 << 16, 256, 2, False, False, 0),
+ (1 << 20, 1024, 2, False, False, 0),
+ (None, 2048, 4, False, False, 0),
+ ),
+ # Blackwell datacenter (B200): int32 loads + register table gather.
+ (10,): (
+ (1 << 16, 256, 2, True, True, 2),
+ (None, 1024, 2, True, True, 2),
+ ),
+}
+if Version(triton.__version__) >= Version("3.7.0"):
+ # T4: 5% to 7% faster on Triton 3.7, not on 3.6.
+ _CONFIGS[(7, 5)] = ((None, 512, 8, True, True, 0),)
+# tl.gather on a register table fails to compile on Triton 3.3; older Triton uses the L1 load.
+_HAS_TL_GATHER = Version(triton.__version__) >= Version("3.6.0")
+# Tests and the sweep script set this to a (target, num_warps, words, evict, lut_mode) tuple.
+_CONFIG_OVERRIDE = None
+
+
[email protected]_cache(maxsize = None)
+def _capability(index: int):
+ return torch.cuda.get_device_capability(index)
+
+
+def _config_for(n_bytes: int, half: int, device: torch.device):
+ if _CONFIG_OVERRIDE is not None:
+ target, num_warps, words, evict, lut_mode = _CONFIG_OVERRIDE
+ else:
+ cap = _capability(device.index if device.index is not None else torch.cuda.current_device())
+ table = _CONFIGS.get(cap) or _CONFIGS.get(cap[:1]) or _CONFIGS[()]
+ for limit, target, num_warps, words, evict, lut_mode in table:
+ if limit is None or n_bytes <= limit:
+ break
+ if lut_mode == 2 and not _HAS_TL_GATHER:
+ lut_mode = 0
+ return max(1, target // half), num_warps, words, evict, lut_mode
+
+
+def _is_pow2(x: int) -> bool:
+ return x > 0 and (x & (x - 1)) == 0
+
+
+def _launch(
+ kernel,
+ W,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ blocksize,
+ blocksize2,
+ out,
+ fp_fusion,
+ plan = False,
+):
+ nested = code2 is not None
+ n_elements = out.numel()
+ n_bytes = (n_elements + 1) // 2
+ half = blocksize // 2
+ # Integer ceil division: triton.cdiv is a JIT function on recent Triton and costs microseconds.
+ n_blocks = -(-n_elements // blocksize)
+ rows, num_warps, words, evict, lut_mode = _config_for(n_bytes, half, W.device)
+ # int32 loads need every row to start on a word and no partial trailing word.
+ words = (
+ words
+ and half % 4 == 0
+ and n_bytes % 4 == 0
+ and (W.storage_offset() * W.element_size()) % 4 == 0
+ )
+ # The nested-only pointers are never dereferenced for a flat state; absmax fills the slots.
+ args = (
+ W,
+ absmax,
+ code2 if nested else absmax,
+ absmax2 if nested else absmax,
+ offset if nested else absmax,
+ code,
+ out,
+ n_elements,
+ n_bytes,
+ n_blocks,
+ )
+ constexprs = dict(
+ HALF = half,
+ BLOCKSIZE2_SHIFT = (blocksize2.bit_length() - 1) if nested else 0,
+ NESTED = nested,
+ USE_MUL_RN = _HAS_MUL_RN,
+ ROWS = rows,
+ WORDS = words,
+ EVICT = "evict_first" if evict else "",
+ LUT_MODE = lut_mode,
+ )
+ grid = (-(-n_blocks // rows),)
+ options = (
+ {"num_warps": num_warps}
+ if fp_fusion
+ else {"num_warps": num_warps, "enable_fp_fusion": False}
+ )
+ if kernel is _nf4_dequant_kernel:
+ entry = launch(kernel, grid, args, 7, constexprs, W.device.index, **options)
+ if plan:
+ return _rerun(entry, grid, args, W.device.index)
+ else:
+ kernel[grid](*args, **constexprs, **options)
+ return out
+
+
+def _rerun(entry, grid, args, device_index):
+ if entry is None:
+ return None
+ before, after = args[:6], args[7:]
+
+ def rerun(out):
+ # out must match the first output in size, dtype and 16 byte alignment.
+ return relaunch(entry, grid, device_index, (*before, out, *after))
+
+ return rerun
+
+
+if _HAS_MUL_RN:
+ # Inductor sees the Triton kernel itself and schedules it with the surrounding graph.
+ _register = torch.library.triton_op
+ _traced_kernel = lambda: torch.library.wrap_triton(_nf4_dequant_kernel)
+ _traced_guard = lambda device: contextlib.nullcontext()
+else:
+ # HIP: an opaque custom op keeps the eager launch (fusion off) and needs its own device guard.
+ _register = torch.library.custom_op
+ _traced_kernel = lambda: _nf4_dequant_kernel
+ _traced_guard = torch.cuda.device
+
+
+@_register("unsloth::dequantize_nf4", mutates_args = ())
+def _dequantize_nf4_op(
+ W: torch.Tensor,
+ absmax: torch.Tensor,
+ code2: Optional[torch.Tensor],
+ absmax2: Optional[torch.Tensor],
+ offset: Optional[torch.Tensor],
+ code: torch.Tensor,
+ blocksize: int,
+ blocksize2: int,
+ shape: List[int],
+ dtype: torch.dtype,
+) -> torch.Tensor:
+ out = torch.empty(shape, dtype = dtype, device = W.device)
+ with _traced_guard(W.device):
+ return _launch(
+ _traced_kernel(),
+ W,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ blocksize,
+ blocksize2,
+ out,
+ fp_fusion = _HAS_MUL_RN,
+ )
+
+
+@_register("unsloth::dequantize_nf4_out", mutates_args = ["out"])
+def _dequantize_nf4_out_op(
+ W: torch.Tensor,
+ absmax: torch.Tensor,
+ code2: Optional[torch.Tensor],
+ absmax2: Optional[torch.Tensor],
+ offset: Optional[torch.Tensor],
+ code: torch.Tensor,
+ blocksize: int,
+ blocksize2: int,
+ out: torch.Tensor,
+) -> None:
+ with _traced_guard(W.device):
+ _launch(
+ _traced_kernel(),
+ W,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ blocksize,
+ blocksize2,
+ out,
+ fp_fusion = _HAS_MUL_RN,
+ )
+
+
+if not _HAS_MUL_RN:
+
+ @_dequantize_nf4_op.register_fake
+ def _dequantize_nf4_fake(
+ W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape, dtype
+ ):
+ return W.new_empty(shape, dtype = dtype)
+
+ @_dequantize_nf4_out_op.register_fake
+ def _dequantize_nf4_out_fake(
+ W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, out
+ ):
+ return None
+
+
+tag_compile_cache(__file__)
+
+
+def dequantize_nf4(
+ W: torch.Tensor,
+ absmax: torch.Tensor,
+ code2: Optional[torch.Tensor],
+ absmax2: Optional[torch.Tensor],
+ offset: Optional[torch.Tensor],
+ code: torch.Tensor,
+ blocksize: int,
+ blocksize2: int,
+ shape,
+ dtype: torch.dtype,
+ out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+ """Dequantize a bitsandbytes NF4 weight into ``out`` (allocated when None).
+
+ ``code`` is the quant state's fp32 [16] NF4 codebook. ``code2``, ``absmax2`` and the device
+ tensor ``offset`` are None when the absmax is not double quantized (it is then fp32).
+ """
+ if dtype not in (torch.float16, torch.bfloat16, torch.float32):
+ raise TypeError(f"Unsloth: dequantize_nf4 does not support dtype {dtype}")
+ if not _is_pow2(blocksize) or blocksize < 2 or (code2 is not None and not _is_pow2(blocksize2)):
+ raise ValueError(f"Unsloth: dequantize_nf4 needs power-of-two blocksizes, got {blocksize}")
+ if out is not None and (out.dtype != dtype or out.numel() != math.prod(shape)):
+ raise ValueError("Unsloth: dequantize_nf4 out has the wrong dtype or size")
+ if torch.compiler.is_compiling():
+ if out is None:
+ return torch.ops.unsloth.dequantize_nf4(
+ W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, list(shape), dtype
+ )
+ torch.ops.unsloth.dequantize_nf4_out(
+ W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, out
+ )
+ return out
+ if out is None:
+ out = torch.empty(shape, dtype = dtype, device = W.device)
+ # Eager launches skip the op dispatcher, which costs about as much as the kernel.
+ same_device = W.device.index == torch.cuda.current_device()
+ with contextlib.nullcontext() if same_device else torch.cuda.device(W.device):
+ return _launch(
+ _nf4_dequant_kernel,
+ W,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ blocksize,
+ blocksize2,
+ out,
+ fp_fusion = False,
+ )
+
+
+def dequantize_nf4_planned(W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, out):
+ """Eager dequantize_nf4 into out on the current device, returning a function that repeats
+ the launch into another buffer like out for this same weight, skipping the per-call argument
+ handling (decode dequantizes every weight once per token). None when that is unavailable."""
+ return _launch(
+ _nf4_dequant_kernel,
+ W,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ blocksize,
+ blocksize2,
+ out,
+ fp_fusion = False,
+ plan = True,
+ )
diff --git a/unsloth/kernels/nf4_gemv.py b/unsloth/kernels/nf4_gemv.py
new file mode 100644
index 00000000000..f75694e1f3b
--- /dev/null
+++ b/unsloth/kernels/nf4_gemv.py
@@ -0,0 +1,390 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+# Batch-1 NF4 GEMV for decoding, out[1, 1, N] = X[1, 1, K] @ dequant(W).T, in one Triton launch
+# (nested absmax decode, NF4 lookup and dot product fused) on torch's current stream. Registered
+# as unsloth::gemv_nf4 so torch.compile traces it and CUDA graphs capture it. Each quantization
+# block is summed before scaling, so it matches bitsandbytes to fp32 accumulation noise, not bitwise.
+# Every weight row must start a new quantization block (K % blocksize == 0); callers check.
+
+import functools
+from typing import Optional
+
+import torch
+import triton
+import triton.language as tl
+
+from .nf4 import _HAS_MUL_RN, _mul
+from .triton_launch import launch, relaunch, tag_compile_cache
+
+__all__ = [
+ "gemv_nf4",
+ "gemv_nf4_planned",
+ "triton_gemv_eager",
+]
+
+
+def _gemv_nf4_kernel(
+ X,
+ W,
+ ABSMAX,
+ CODE2,
+ ABSMAX2,
+ OFFSET,
+ LUT,
+ OUT,
+ N,
+ K,
+ BLOCKSIZE: tl.constexpr,
+ BLOCKSIZE2: tl.constexpr,
+ NESTED: tl.constexpr,
+ USE_MUL_RN: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_K: tl.constexpr,
+):
+ # FSDP-QLoRA packs the 4bit weight into float storage; it is bytes either way.
+ W = W.to(tl.pointer_type(tl.uint8))
+ pid = tl.program_id(0)
+ rows = pid * BLOCK_N + tl.arange(0, BLOCK_N)
+ rmask = rows < N
+ rows64 = rows.to(tl.int64)
+ HALF_K: tl.constexpr = BLOCK_K // 2
+ NB: tl.constexpr = BLOCK_K // BLOCKSIZE
+ half_k = tl.arange(0, HALF_K)
+ ks = tl.arange(0, BLOCK_K)
+ nbs = tl.arange(0, NB)
+ blocks_per_row = K // BLOCKSIZE
+ acc = tl.zeros([BLOCK_N], dtype = tl.float32)
+ if NESTED:
+ offset = tl.load(OFFSET)
+ for k0 in range(0, K, BLOCK_K):
+ kmask = (k0 + 2 * half_k) < K
+ m = rmask[:, None] & kmask[None, :]
+ b = tl.load(W + rows64[:, None] * (K // 2) + (k0 // 2 + half_k)[None, :], mask = m, other = 0)
+ b = b.to(tl.int32)
+ # High nibble is the earlier element, as bitsandbytes packs it.
+ hi = tl.load(LUT + (b >> 4))
+ lo = tl.load(LUT + (b & 15))
+ w = tl.reshape(tl.join(hi, lo), (BLOCK_N, BLOCK_K))
+ xk = k0 + ks
+ x = tl.load(X + xk, mask = xk < K, other = 0.0).to(tl.float32)
+ part = tl.sum(tl.reshape(w * x[None, :], (BLOCK_N, NB, BLOCKSIZE)), axis = 2)
+ bmask = rmask[:, None] & ((k0 + nbs * BLOCKSIZE) < K)[None, :]
+ blk = rows64[:, None] * blocks_per_row + (k0 // BLOCKSIZE + nbs)[None, :]
+ if NESTED:
+ q = tl.load(ABSMAX + blk, mask = bmask, other = 0).to(tl.int32)
+ c2 = tl.load(CODE2 + q, mask = bmask, other = 0.0)
+ s2 = tl.load(ABSMAX2 + blk // BLOCKSIZE2, mask = bmask, other = 0.0)
+ # Two separate fp32 roundings as in fast_dequantize, even with fp fusion on.
+ a = _mul(c2, s2, USE_MUL_RN)
+ a = a + offset
+ else:
+ a = tl.load(ABSMAX + blk, mask = bmask, other = 0.0)
+ acc += tl.sum(part * a, axis = 1)
+ tl.store(OUT + rows, acc.to(OUT.dtype.element_ty), mask = rmask)
+
+
+def _gemv_nf4_words_kernel(
+ X,
+ W,
+ ABSMAX,
+ CODE2,
+ ABSMAX2,
+ OFFSET,
+ LUT,
+ OUT,
+ N,
+ K,
+ BLOCKSIZE: tl.constexpr,
+ BLOCKSIZE2: tl.constexpr,
+ NESTED: tl.constexpr,
+ USE_MUL_RN: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_K: tl.constexpr,
+):
+ # _gemv_nf4_kernel reading int32 words of 8 nibbles; faster before Triton 3.7, slower on 3.7.
+ W = W.to(tl.pointer_type(tl.int32))
+ pid = tl.program_id(0)
+ rows = pid * BLOCK_N + tl.arange(0, BLOCK_N)
+ rmask = rows < N
+ rows64 = rows.to(tl.int64)
+ WORDS: tl.constexpr = BLOCK_K // 8
+ NB: tl.constexpr = BLOCK_K // BLOCKSIZE
+ wk = tl.arange(0, WORDS)
+ j = tl.arange(0, 8)
+ # Little endian: byte b of a word holds values 2b (high nibble) and 2b + 1 (low nibble).
+ shifts = (j // 2) * 8 + (1 - (j % 2)) * 4
+ nbs = tl.arange(0, NB)
+ words_per_row = K // 8
+ blocks_per_row = K // BLOCKSIZE
+ acc = tl.zeros([BLOCK_N], dtype = tl.float32)
+ if NESTED:
+ offset = tl.load(OFFSET)
+ for k0 in range(0, K, BLOCK_K):
+ kw = k0 // 8 + wk
+ m = rmask[:, None] & (kw < words_per_row)[None, :]
+ w = tl.load(W + rows64[:, None] * words_per_row + kw[None, :], mask = m, other = 0)
+ v = tl.load(LUT + ((w[:, :, None] >> shifts[None, None, :]) & 15))
+ xk = k0 + wk[:, None] * 8 + j[None, :]
+ x = tl.load(X + xk, mask = xk < K, other = 0.0).to(tl.float32)
+ part = tl.sum(tl.reshape(v * x[None, :, :], (BLOCK_N, NB, BLOCKSIZE)), axis = 2)
+ bmask = rmask[:, None] & ((k0 + nbs * BLOCKSIZE) < K)[None, :]
+ blk = rows64[:, None] * blocks_per_row + (k0 // BLOCKSIZE + nbs)[None, :]
+ if NESTED:
+ q = tl.load(ABSMAX + blk, mask = bmask, other = 0).to(tl.int32)
+ c2 = tl.load(CODE2 + q, mask = bmask, other = 0.0)
+ s2 = tl.load(ABSMAX2 + blk // BLOCKSIZE2, mask = bmask, other = 0.0)
+ a = _mul(c2, s2, USE_MUL_RN)
+ a = a + offset
+ else:
+ a = tl.load(ABSMAX + blk, mask = bmask, other = 0.0)
+ acc += tl.sum(part * a, axis = 1)
+ tl.store(OUT + rows, acc.to(OUT.dtype.element_ty), mask = rmask)
+
+
+_TRITON_37 = tuple(int(x) for x in triton.__version__.split(".")[:2]) >= (3, 7)
+# None picks per GPU and Triton version below; "bytes" or "words" forces one kernel (tests, sweeps).
+_FORCE_KERNEL = None
+
+
+def triton_gemv_eager(device = None):
+ """Whether eager (uncompiled) decode should take the Triton GEMV. On CUDA it does: with the
+ direct launch (triton_launch.py) it beat bitsandbytes' GEMV in batch-1 decode on T4, L4, A100,
+ RTX PRO 6000 and B200. HIP launches through Triton's own path, which only paid off from
+ Triton 3.7."""
+ return torch.version.hip is None or _TRITON_37
+
+
+def _word_aligned(W):
+ # Rows are K / 2 bytes (K % blocksize == 0), so only a view can break 4 byte alignment.
+ return (W.storage_offset() * W.element_size()) % 4 == 0
+
+
+_NARROW_CAPS = ((7, 5), (8, 9))
+
+
[email protected]_cache(maxsize = None)
+def _gemv_config(N: int, K: int, blocksize: int, capability, words_ok: bool, force):
+ """(use the words kernel, grid, BLOCK_N, BLOCK_K, num_warps).
+
+ Kernel time (CUDA graphs) over Llama 1B to 70B shapes: before Triton 3.7 the words kernel at
+ one row per program and one warp beat bitsandbytes on T4, L4, A100, RTX PRO 6000 and B200
+ (1.1x to 2.2x, lower error too); Triton 3.7 compiles the byte kernel better, best at 4 rows, BLOCK_K 2048 and
+ 4 warps (measured on B200)."""
+ block_k = max(blocksize, min(2048, triton.next_power_of_2(K)))
+ use_words = words_ok and (force == "words" or (force is None and not _TRITON_37))
+ if use_words:
+ # One row per program and BLOCK_K 2048 was the fastest config at every Llama 1B / 8B shape on
+ # T4, L4 and A100 (interleaved CUDA graph timing; T4 throttles, so sweeps must interleave).
+ return True, (N,), 1, block_k, 1
+ if force is None:
+ if capability in _NARROW_CAPS and N * K > 1 << 25 and K >= 4096:
+ # T4 / L4: 3% to 9% faster on 8B MLP shapes; slower on A100 and B200.
+ return False, (-(-N // 4),), 4, max(blocksize, 1024), 2
+ return False, (-(-N // 4),), 4, block_k, 4
+ # Forced byte kernel (tests): the pre-3.7 byte config.
+ block_k = max(blocksize, min(1024, triton.next_power_of_2(K)))
+ block_n, num_warps = (2, 4) if N <= 2048 else (4, 2)
+ return False, (-(-N // block_n),), block_n, block_k, num_warps
+
+
[email protected]_cache(maxsize = None)
+def _capability(index):
+ return torch.cuda.get_device_capability(index)
+
+
+def _launch(
+ kernels,
+ X,
+ W,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ out,
+ N,
+ K,
+ blocksize,
+ blocksize2,
+ plan = False,
+):
+ nested = code2 is not None
+ use_words, grid, block_n, block_k, num_warps = _gemv_config(
+ N, K, blocksize, _capability(X.device.index), _word_aligned(W), _FORCE_KERNEL
+ )
+ kernel = kernels[1] if use_words else kernels[0]
+ args = (
+ X,
+ W,
+ absmax,
+ code2 if nested else absmax,
+ absmax2 if nested else absmax,
+ offset if nested else absmax,
+ code,
+ out,
+ N,
+ K,
+ )
+ constexprs = dict(
+ BLOCKSIZE = blocksize,
+ BLOCKSIZE2 = blocksize2 if nested else 1,
+ NESTED = nested,
+ USE_MUL_RN = _HAS_MUL_RN,
+ BLOCK_N = block_n,
+ BLOCK_K = block_k,
+ )
+ if kernel is _gemv_nf4_kernel or kernel is _gemv_nf4_words_kernel:
+ entry = launch(kernel, grid, args, 8, constexprs, X.device.index, num_warps = num_warps)
+ if plan:
+ return _rerun(entry, grid, args, X.device.index)
+ else:
+ kernel[grid](*args, **constexprs, num_warps = num_warps)
+ return out
+
+
+def _rerun(entry, grid, args, device_index):
+ if entry is None:
+ return None
+ weights, sizes = args[1:7], args[8:]
+
+ def rerun(X, out):
+ # X and out must match the first launch's in dtype and 16 byte alignment.
+ return relaunch(entry, grid, device_index, (X, *weights, out, *sizes))
+
+ return rerun
+
+
[email protected]_op("unsloth::gemv_nf4", mutates_args = ())
+def _gemv_nf4_op(
+ X: torch.Tensor,
+ W: torch.Tensor,
+ absmax: torch.Tensor,
+ code2: Optional[torch.Tensor],
+ absmax2: Optional[torch.Tensor],
+ offset: Optional[torch.Tensor],
+ code: torch.Tensor,
+ blocksize: int,
+ blocksize2: int,
+ out_features: int,
+ in_features: int,
+) -> torch.Tensor:
+ out = torch.empty((1, 1, out_features), dtype = X.dtype, device = X.device)
+ return _launch(
+ (
+ torch.library.wrap_triton(_gemv_nf4_kernel),
+ torch.library.wrap_triton(_gemv_nf4_words_kernel),
+ ),
+ X,
+ W,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ out,
+ out_features,
+ in_features,
+ blocksize,
+ blocksize2,
+ )
+
+
+_KERNELS = (_gemv_nf4_kernel, _gemv_nf4_words_kernel)
+tag_compile_cache(__file__)
+_is_compiling = torch.compiler.is_compiling
+_current_device = torch.cuda.current_device
+
+
+def gemv_nf4(
+ X,
+ W_u8,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ blocksize,
+ blocksize2,
+ shape,
+ dtype,
+ out = None,
+):
+ """X @ dequant(W).T for a single token; returns (1, 1, shape[0]) in X's dtype.
+
+ ``code2/absmax2/offset`` are None for a quant state without double quantization, and
+ ``offset`` is bitsandbytes' 0-dim device tensor (read in the kernel, so no host sync).
+ Eager calls launch the kernel directly: the torch.library dispatcher costs about 20us
+ per call, as much as the kernel itself at decode sizes. Compiled code goes through
+ the registered op so Dynamo sees it.
+ """
+ out_features, in_features = int(shape[0]), int(shape[1])
+ if not X.is_contiguous():
+ X = X.contiguous()
+ blocksize2 = int(blocksize2) if code2 is not None else 0
+ if _is_compiling():
+ result = _gemv_nf4_op(
+ X,
+ W_u8,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ int(blocksize),
+ blocksize2,
+ out_features,
+ in_features,
+ )
+ if out is not None:
+ out.copy_(result)
+ return out
+ return result
+ if out is None:
+ out = torch.empty((1, 1, out_features), dtype = X.dtype, device = X.device)
+ args = (
+ _KERNELS,
+ X,
+ W_u8,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ out,
+ out_features,
+ in_features,
+ int(blocksize),
+ blocksize2,
+ )
+ # Triton launches on the current device's current stream, so only switch when X lives elsewhere.
+ if X.device.index == _current_device():
+ return _launch(*args)
+ with torch.cuda.device(X.device):
+ return _launch(*args)
+
+
+def gemv_nf4_planned(
+ X, W_u8, absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape, out
+):
+ """Eager gemv_nf4 into out on the current device, returning a function (X, out) that repeats
+ the launch for this same weight, skipping the per-call argument handling (decode runs every
+ weight once per token). None when that is unavailable."""
+ return _launch(
+ _KERNELS,
+ X,
+ W_u8,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ out,
+ int(shape[0]),
+ int(shape[1]),
+ int(blocksize),
+ int(blocksize2) if code2 is not None else 0,
+ plan = True,
+ )
diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.py
index cfc721d3618..2078f3266e0 100644
--- a/unsloth/kernels/swiglu.py
+++ b/unsloth/kernels/swiglu.py
@@ -103,7 +103,21 @@ def _DWf_DW_dfg_kernel(DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDE
tl.store(g + offsets, de_row, mask = mask)
+def _DWf_DW_dfg_traced(DW, e, g):
+ # Traced: Inductor fuses this in place; the in-place kernel would make it copy e and g first.
+ e_float = e.float()
+ se = torch.sigmoid(e_float)
+ f = (se * e_float).to(DW.dtype)
+ h = f * g
+ df = DW * f
+ dg = DW * g
+ de = (dg.float() * se * (1.0 + e_float * (1.0 - se))).to(DW.dtype)
+ return h, df, de
+
+
def swiglu_DWf_DW_dfg_kernel(DW, e, g):
+ if torch.compiler.is_compiling():
+ return _DWf_DW_dfg_traced(DW, e, g)
batch_seq_len, hd = e.shape # Flattened to 2D, so 1st dim is bsz * seq_len
n_elements = e.numel()
grid = lambda meta: (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),)
diff --git a/unsloth/kernels/triton_launch.py b/unsloth/kernels/triton_launch.py
new file mode 100644
index 00000000000..68225724162
--- /dev/null
+++ b/unsloth/kernels/triton_launch.py
@@ -0,0 +1,129 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+# Low overhead eager launches: the first call per specialization goes through Triton, later ones
+# launch the compiled kernel directly, skipping JITFunction's per-call argument binding. The key
+# covers what Triton specializes on: dtypes, 16 byte alignment, ints == 1 / % 16 / > int32, options.
+
+import hashlib
+import os
+
+import torch
+import triton
+
+__all__ = [
+ "launch",
+ "relaunch",
+ "tag_compile_cache",
+]
+
+# CPU-only torch builds have no CUDA stream binding.
+_raw_stream = getattr(torch._C, "_cuda_getCurrentRawStream", None)
+_ENABLED = (
+ _raw_stream is not None
+ and torch.version.hip is None
+ and os.environ.get("UNSLOTH_TRITON_FAST_LAUNCH", "1") != "0"
+)
+_CACHE = {}
+_knobs = getattr(triton, "knobs", None)
+
+
+def _active(hook):
+ # Triton 3.5+ always installs a HookChain, empty unless something registered a hook.
+ return hook is not None and getattr(hook, "calls", True) != []
+
+
+def _hooks_set():
+ # Profilers (e.g. proton) hook Triton launches; those launches keep Triton's own path.
+ if _knobs is not None:
+ runtime = _knobs.runtime
+ return _active(runtime.launch_enter_hook) or _active(runtime.launch_exit_hook)
+ from triton.compiler import CompiledKernel
+ return _active(getattr(CompiledKernel, "launch_enter_hook", None)) or _active(
+ getattr(CompiledKernel, "launch_exit_hook", None)
+ )
+
+
+def relaunch(entry, grid, device_index, args):
+ """Repeat a launch whose cache entry launch() returned, with args matching the original in
+ dtypes, 16 byte alignment and integer specialization, on device_index (the current device).
+ False when the caller must use launch() instead."""
+ if not _ENABLED or _hooks_set():
+ return False
+ compiled, tail = entry
+ compiled.run(
+ grid[0],
+ grid[1] if len(grid) > 1 else 1,
+ grid[2] if len(grid) > 2 else 1,
+ _raw_stream(device_index),
+ compiled.function,
+ compiled.packed_metadata,
+ None,
+ None,
+ None,
+ *args,
+ *tail,
+ )
+ return True
+
+
+def launch(kernel, grid, args, n_tensors, constexprs, device_index, **options):
+ """kernel[grid](*args, **constexprs, **options) on the current stream of device_index, which
+ must be the current device. args are the runtime arguments in signature order, before every
+ constexpr: n_tensors tensors, then integers. Returns the cache entry relaunch() takes, or None."""
+ global _ENABLED
+ if _ENABLED and not _hooks_set():
+ key = (
+ kernel,
+ device_index,
+ tuple(constexprs.values()),
+ tuple(options.values()),
+ tuple([(arg.dtype, arg.data_ptr() & 15) for arg in args[:n_tensors]]),
+ tuple([(arg == 1, arg & 15, arg < (1 << 31)) for arg in args[n_tensors:]]),
+ )
+ entry = _CACHE.get(key)
+ if entry is not None:
+ compiled, tail = entry
+ try:
+ compiled.run(
+ grid[0],
+ grid[1] if len(grid) > 1 else 1,
+ grid[2] if len(grid) > 2 else 1,
+ _raw_stream(device_index),
+ compiled.function,
+ compiled.packed_metadata,
+ None,
+ None,
+ None,
+ *args,
+ *tail,
+ )
+ return entry
+ except Exception:
+ # A Triton build whose launcher takes other arguments: use its own path from now.
+ _ENABLED = False
+ _CACHE.clear()
+ compiled = kernel[grid](*args, **constexprs, **options)
+ if _ENABLED and hasattr(compiled, "packed_metadata"):
+ names = kernel.arg_names[len(args) :]
+ if len(names) == len(constexprs):
+ compiled.run # loads the module now, so the fast path never does
+ entry = _CACHE[key] = (compiled, tuple(constexprs[name] for name in names))
+ return entry
+ return None
+ kernel[grid](*args, **constexprs, **options)
+ return None
+
+
+def tag_compile_cache(path):
+ """Add a hash of a kernel file to torch.compile's cache key. Inductor's FX graph cache keys a
+ torch.library.triton_op call without the Triton source behind it, so after an Unsloth upgrade
+ a warm cache would keep serving the previous kernel."""
+ config = getattr(torch.compiler, "config", None)
+ if config is None or not hasattr(config, "cache_key_tag"):
+ return
+ with open(path, "rb") as file:
+ tag = f"unsloth/{os.path.basename(path)}:{hashlib.sha256(file.read()).hexdigest()[:16]}"
+ tags = [t for t in config.cache_key_tag.split(",") if t]
+ if tag not in tags:
+ config.cache_key_tag = ",".join(tags + [tag])
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index ad0b2568a08..5027392d3c7 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -10,6 +10,7 @@
# limitations under the License.
import importlib
+import os
import triton
import ctypes
@@ -211,6 +212,8 @@ def _get_tensor_stream(tensor: torch_Tensor) -> c_void_p:
return c_void_p(_gpu_getCurrentRawStream(tensor.device.index))
+# Import-time stream snapshots and the old scratch pair. The kernels now read the live stream, but
+# these stay defined for external readers (e.g. tests/kaggle/t4_smoke/run_t4_smoke.py).
global CUDA_STREAMS
global XPU_STREAMS
global WEIGHT_BUFFERS
@@ -497,6 +500,117 @@ def _maybe_fake_quantize_activations(X: torch.Tensor, proj: torch.nn.Module) ->
return X
+def _unpack_quant_state(quant_state):
+ """(absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2) from a bnb
+ 4bit quant state, class or legacy list form. The nested fields are None (and blocksize2 0)
+ when the absmax is not double quantized."""
+ if type(quant_state) is not list:
+ # New quant_state as a class, per TimDettmers/bitsandbytes#763.
+ absmax = quant_state.absmax
+ shape = quant_state.shape
+ dtype = quant_state.dtype
+ blocksize = quant_state.blocksize
+ code = quant_state.code
+ offset = quant_state.offset
+ state2 = quant_state.state2
+ else:
+ # Old quant_state as a list of lists
+ absmax, shape, dtype, blocksize, compressed_stats, _, code = quant_state
+ offset, state2 = compressed_stats if compressed_stats is not None else (None, None)
+ if state2 is None:
+ return absmax, shape, dtype, blocksize, code, None, None, None, 0
+ if type(state2) is list:
+ absmax2, code2, blocksize2 = state2[0], state2[1], state2[2]
+ else:
+ absmax2, code2, blocksize2 = state2.absmax, state2.code, state2.blocksize
+ if not isinstance(offset, torch_Tensor):
+ offset = torch.tensor(offset, dtype = torch_float32, device = absmax.device)
+ return absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2
+
+
+def _quant_type(quant_state):
+ if type(quant_state) is list:
+ return quant_state[5]
+ return getattr(quant_state, "quant_type", "nf4")
+
+
+# Eager use_global_buffer scratch per (kind, device), default stream only (other streams, compiling
+# and graph capture get fresh outputs); allocated outside inference mode so training can write it.
+_SCRATCH = {}
+_DEFAULT_STREAMS = {}
+
+
+def _can_use_scratch(device):
+ if torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing():
+ return False
+ default = _DEFAULT_STREAMS.get(device.index)
+ if default is None:
+ default = _DEFAULT_STREAMS[device.index] = torch.cuda.default_stream(device).cuda_stream
+ return _gpu_getCurrentRawStream(device.index) == default
+
+
+def _scratch(kind, device, numel, dtype):
+ key = (kind, device.index)
+ buffer = _SCRATCH.get(key)
+ if buffer is None or buffer.dtype != dtype or buffer.numel() < numel:
+ with torch.inference_mode(False):
+ buffer = torch_empty(numel, dtype = dtype, device = device, requires_grad = False)
+ _SCRATCH[key] = buffer
+ return buffer[:numel]
+
+
+# Fused NF4 kernels, traceable by torch.compile and on the live stream by construction.
+# UNSLOTH_BNB_TRITON=0 keeps the bitsandbytes ctypes path.
+_USE_NF4_KERNELS = False
+_TRITON_GEMV_EAGER = False
+_is_compiling = torch.compiler.is_compiling
+_current_device = torch.cuda.current_device
+if (
+ DEVICE_TYPE in ("cuda", "hip")
+ and HAS_CUDA_STREAM
+ and os.environ.get("UNSLOTH_BNB_TRITON", "1") != "0"
+):
+ try:
+ import triton
+ from .nf4 import dequantize_nf4, dequantize_nf4_planned
+ from .nf4_gemv import gemv_nf4, gemv_nf4_planned, triton_gemv_eager
+
+ _USE_NF4_KERNELS = True
+ # Eager decode uses the Triton GEMV where it is faster; compiled code always does.
+ _TRITON_GEMV_EAGER = triton_gemv_eager()
+ except Exception:
+ pass
+
+
+def _no_plan():
+ return None
+
+
+class _NF4Plan(list):
+ # One weight's decode launch (see fast_dequantize). It holds a closure and the scratch buffer,
+ # so copies and pickles of the model drop it and rebuild on first use.
+ def __reduce__(self):
+ return (_no_plan, ())
+
+
+def _nf4_kernels_failed(error):
+ """A Triton compile failure on an untested GPU switches to the bitsandbytes kernels for good.
+ False (re-raise) while tracing, for Dynamo's own exceptions (a graph break is not a kernel
+ failure) and for out-of-memory, which the fallback would not fix."""
+ global _USE_NF4_KERNELS
+ if (
+ torch.compiler.is_compiling()
+ or isinstance(error, torch.cuda.OutOfMemoryError)
+ or type(error).__module__.startswith("torch._dynamo")
+ ):
+ return False
+ _USE_NF4_KERNELS = False
+ print(
+ f"Unsloth: fused NF4 kernels failed ({type(error).__name__}: {error}), using bitsandbytes."
+ )
+ return True
+
+
if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
@torch.inference_mode
@@ -613,84 +727,36 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
- @torch.inference_mode
- def fast_dequantize(
+ def _fast_dequantize_ctypes(
W,
- quant_state = None,
+ quant_state,
out = None,
use_global_buffer = False,
):
- if isinstance(W, Float8Tensor):
- return W.dequantize()
- if quant_state is None:
- return W
- if W.dtype == torch.float8_e4m3fn:
- return weight_dequant(W, quant_state)
- if type(quant_state) is NVFP4QuantState:
- return nvfp4_dequantize(
- W, quant_state.scale, quant_state.global_scale, quant_state.dtype
- )
- if type(quant_state) is not list:
- # New quant_state as a class, per TimDettmers/bitsandbytes#763.
- absmax = quant_state.absmax
- shape = quant_state.shape
- dtype = quant_state.dtype
- blocksize = quant_state.blocksize
- offset = quant_state.offset
- state2 = quant_state.state2
- absmax2 = state2.absmax
- code2 = state2.code
- blocksize2 = state2.blocksize
- else:
- # Old quant_state as a list of lists
- absmax, shape, dtype, blocksize, compressed_stats, _, _ = quant_state
- offset, state2 = compressed_stats
- absmax2, code2, blocksize2, _, _, _, _ = state2
- pass
- global CUDA_STREAMS
+ # bitsandbytes ctypes kernels, used when the NF4 kernels are unavailable or disabled.
+ absmax, shape, dtype, blocksize, _, code2, absmax2, offset, blocksize2 = (
+ _unpack_quant_state(quant_state)
+ )
+ if code2 is None or dtype not in (torch_float16, torch_bfloat16):
+ return bnb_functional.dequantize_4bit(W, quant_state, out = out)
device = W.device
- device_index = device.index
- CUDA_STREAM = CUDA_STREAMS[device_index]
+ # Each launch reads the live stream, so out_absmax += offset stays ordered between them.
+ CUDA_STREAM = _get_tensor_stream(W)
n_elements_absmax = absmax.numel()
-
- if use_global_buffer:
- size = shape[0] * shape[1]
- global WEIGHT_BUFFERS
- global ABSMAX_BUFFERS
- WEIGHT_BUFFER = WEIGHT_BUFFERS[device_index]
- ABSMAX_BUFFER = ABSMAX_BUFFERS[device_index]
- if WEIGHT_BUFFER is None or WEIGHT_BUFFER.dtype != dtype:
- WEIGHT_BUFFERS[device_index] = WEIGHT_BUFFER = torch_empty(
- size, dtype = dtype, device = device, requires_grad = False
- )
- ABSMAX_BUFFERS[device_index] = ABSMAX_BUFFER = torch_empty(
- n_elements_absmax,
- dtype = torch_float32,
- device = device,
- requires_grad = False,
- )
-
- if size > WEIGHT_BUFFER.numel():
- WEIGHT_BUFFER.resize_(size)
- if n_elements_absmax > ABSMAX_BUFFER.numel():
- ABSMAX_BUFFER.resize_(n_elements_absmax)
-
- out = WEIGHT_BUFFER[:size].view(shape)
- out_absmax = ABSMAX_BUFFER[:n_elements_absmax]
+ if out is not None:
+ assert out.shape == shape
+ assert out.dtype == dtype
+ elif use_global_buffer and _can_use_scratch(device):
+ out = _scratch("weight", device, shape[0] * shape[1], dtype).view(shape)
+ else:
+ out = torch_empty(shape, dtype = dtype, device = device, requires_grad = False)
+ if use_global_buffer and _can_use_scratch(device):
+ out_absmax = _scratch("absmax", device, n_elements_absmax, torch_float32)
else:
- if out is None:
- out = torch_empty(shape, dtype = dtype, device = device, requires_grad = False)
- else:
- assert out.shape == shape
- assert out.dtype == dtype
out_absmax = torch_empty(
- n_elements_absmax,
- dtype = torch_float32,
- device = device,
- requires_grad = False,
+ n_elements_absmax, dtype = torch_float32, device = device, requires_grad = False
)
- pass
ptr_out_absmax = get_ptr(out_absmax)
with torch_gpu_device(device):
@@ -721,12 +787,114 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
ctypes_c_int(out.numel()),
CUDA_STREAM,
)
- pass
# Careful returning transposed data.
is_transposed = True if W.shape[0] == 1 else False
return out.t() if is_transposed else out
- pass
+ def fast_dequantize(
+ W,
+ quant_state = None,
+ out = None,
+ use_global_buffer = False,
+ ):
+ if isinstance(W, Float8Tensor):
+ return W.dequantize()
+ if quant_state is None:
+ return W
+ if W.dtype == torch.float8_e4m3fn:
+ return weight_dequant(W, quant_state)
+ if type(quant_state) is NVFP4QuantState:
+ return nvfp4_dequantize(
+ W, quant_state.scale, quant_state.global_scale, quant_state.dtype
+ )
+ if _quant_type(quant_state) != "nf4":
+ # The NF4 kernels would decode an FP4 weight with the wrong codebook.
+ return bnb_functional.dequantize_4bit(W, quant_state, out = out)
+ if not _USE_NF4_KERNELS:
+ return _fast_dequantize_ctypes(W, quant_state, out, use_global_buffer)
+ device = W.device
+ # Decode dequantizes each weight into the scratch buffer once per token: repeat the first
+ # launch for this weight directly (see dequantize_nf4_planned).
+ scratch = (
+ out is None
+ and use_global_buffer
+ and _can_use_scratch(device)
+ and device.index == _current_device()
+ )
+ if scratch and type(quant_state) is not list:
+ plan = getattr(quant_state, "_unsloth_nf4_plan", None)
+ if (
+ plan is not None
+ and plan[0] == W.data_ptr()
+ and plan[1] is quant_state.absmax
+ and plan[2] is quant_state.dtype
+ ):
+ if _SCRATCH.get(("weight", device.index)) is plan[4]:
+ out = plan[5]
+ else:
+ shape = quant_state.shape
+ out = _scratch("weight", device, shape[0] * shape[1], plan[2]).view(shape)
+ plan[4], plan[5] = _SCRATCH[("weight", device.index)], out
+ try:
+ done = plan[3](out)
+ except Exception:
+ done = False
+ if done:
+ return out.t() if W.shape[0] == 1 else out
+ quant_state._unsloth_nf4_plan = None
+ out = None
+ absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2 = (
+ _unpack_quant_state(quant_state)
+ )
+ if out is not None:
+ assert out.shape == shape
+ assert out.dtype == dtype
+ elif use_global_buffer and _can_use_scratch(device):
+ out = _scratch("weight", device, shape[0] * shape[1], dtype).view(shape)
+ try:
+ if (
+ scratch
+ and dtype in (torch_float16, torch_bfloat16, torch_float32)
+ and blocksize & (blocksize - 1) == 0
+ and (code2 is None or blocksize2 & (blocksize2 - 1) == 0)
+ ):
+ # Launches once; the scratch buffer always starts 16 byte aligned, as reruns need.
+ rerun = dequantize_nf4_planned(
+ W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, out
+ )
+ if rerun is not None and type(quant_state) is not list:
+ quant_state._unsloth_nf4_plan = _NF4Plan(
+ [
+ W.data_ptr(),
+ absmax,
+ dtype,
+ rerun,
+ _SCRATCH.get(("weight", device.index)),
+ out,
+ ]
+ )
+ else:
+ out = dequantize_nf4(
+ W,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ blocksize,
+ blocksize2,
+ shape,
+ dtype,
+ out = out,
+ )
+ except Exception as error:
+ if not _nf4_kernels_failed(error):
+ raise
+ return _fast_dequantize_ctypes(W, quant_state, out, use_global_buffer)
+ # Careful returning transposed data.
+ is_transposed = True if W.shape[0] == 1 else False
+ return out.t() if is_transposed else out
+
else:
@torch.inference_mode
@@ -914,37 +1082,22 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
- def fast_gemv(
+ def _fast_gemv_ctypes(
X,
W,
quant_state,
out = None,
):
- if quant_state is None:
- return torch_matmul(X, W, out = out)
# Fast X @ W where seq_len == 1, from bitsandbytes functional.py#L1469 and TimDettmers/bitsandbytes#763.
_, q_len, hd = X.shape
-
- if type(quant_state) is not list:
- absmax = quant_state.absmax
- shape = quant_state.shape
- dtype = quant_state.dtype
- blocksize = quant_state.blocksize
- stats = quant_state.code
- offset = quant_state.offset
- state2 = quant_state.state2
- absmax2 = state2.absmax
- code2 = state2.code
- blocksize2 = state2.blocksize
- else:
- absmax, shape, dtype, blocksize, compressed_stats, quant_type, stats = quant_state
- offset, state2 = compressed_stats
- absmax2, code2, blocksize2, _, _, _, _ = state2
- pass
- global CUDA_STREAMS
+ absmax, shape, dtype, blocksize, stats, code2, absmax2, offset, blocksize2 = (
+ _unpack_quant_state(quant_state)
+ )
+ if shape[1] % blocksize != 0:
+ # The gemv kernels assume each weight row starts a new quantization block.
+ return torch_matmul(X, fast_dequantize(W, quant_state).t(), out = out)
device = W.device
- device_index = device.index
- CUDA_STREAM = CUDA_STREAMS[device_index]
+ CUDA_STREAM = c_void_p(_gpu_getCurrentRawStream(device.index))
bout = shape[0]
@@ -972,19 +1125,20 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
ldb = ctypes_c_int32(ldb)
ldc = ctypes_c_int32(ldc)
- df = torch_empty(absmax.shape, dtype = torch_float32, device = device)
with torch_gpu_device(device):
- cdequantize_blockwise_fp32(
- get_ptr(code2),
- get_ptr(absmax),
- get_ptr(absmax2),
- get_ptr(df),
- ctypes_c_int(blocksize2),
- ctypes_c_int(df.numel()),
- CUDA_STREAM,
- )
- df += offset
- absmax = df
+ if code2 is not None:
+ df = torch_empty(absmax.shape, dtype = torch_float32, device = device)
+ cdequantize_blockwise_fp32(
+ get_ptr(code2),
+ get_ptr(absmax),
+ get_ptr(absmax2),
+ get_ptr(df),
+ ctypes_c_int(blocksize2),
+ ctypes_c_int(df.numel()),
+ CUDA_STREAM,
+ )
+ df += offset
+ absmax = df
fx = (
cgemm_4bit_inference_naive_fp16
@@ -1008,11 +1162,90 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
blocksize,
CUDA_STREAM,
)
- pass
return out
- pass
+ def fast_gemv(
+ X,
+ W,
+ quant_state,
+ out = None,
+ ):
+ if quant_state is None:
+ return torch_matmul(X, W, out = out)
+ if not _USE_NF4_KERNELS or not (_TRITON_GEMV_EAGER or _is_compiling()):
+ return _fast_gemv_ctypes(X, W, quant_state, out)
+ # Eager decode runs every weight once per token: repeat the first launch for this weight
+ # directly (see gemv_nf4_planned).
+ eager = (
+ not _is_compiling()
+ and type(quant_state) is not list
+ and X.is_contiguous()
+ and X.device.index == _current_device()
+ )
+ if eager:
+ plan = getattr(quant_state, "_unsloth_gemv_plan", None)
+ if (
+ plan is not None
+ and plan[0] == W.data_ptr()
+ and plan[1] is quant_state.absmax
+ and plan[2] is X.dtype
+ and X.data_ptr() & 15 == plan[3]
+ and (out is None or (out.dtype is X.dtype and out.data_ptr() & 15 == 0))
+ ):
+ if out is None:
+ out = torch_empty((1, 1, plan[4]), dtype = X.dtype, device = X.device)
+ try:
+ done = plan[5](X, out)
+ except Exception:
+ done = False
+ if done:
+ return out
+ quant_state._unsloth_gemv_plan = None
+ absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2 = (
+ _unpack_quant_state(quant_state)
+ )
+ if shape[1] % blocksize != 0:
+ # The gemv kernels assume each weight row starts a new quantization block.
+ return torch_matmul(X, fast_dequantize(W, quant_state).t(), out = out)
+ try:
+ if (
+ eager
+ and X.dim() == 3
+ and X.shape[0] == 1
+ and X.shape[1] == 1
+ and (out is None or (out.dtype is X.dtype and out.data_ptr() & 15 == 0))
+ ):
+ if out is None:
+ out = torch_empty((1, 1, int(shape[0])), dtype = X.dtype, device = X.device)
+ # Launches once.
+ rerun = gemv_nf4_planned(
+ X, W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape, out
+ )
+ if rerun is not None:
+ quant_state._unsloth_gemv_plan = _NF4Plan(
+ [W.data_ptr(), absmax, X.dtype, X.data_ptr() & 15, int(shape[0]), rerun]
+ )
+ return out
+ return gemv_nf4(
+ X,
+ W,
+ absmax,
+ code2,
+ absmax2,
+ offset,
+ code,
+ blocksize,
+ blocksize2,
+ shape,
+ dtype,
+ out = out,
+ )
+ except Exception as error:
+ if not _nf4_kernels_failed(error):
+ raise
+ return _fast_gemv_ctypes(X, W, quant_state, out)
+
else:
def fast_gemv(
@@ -1210,6 +1443,35 @@ def fast_linear_forward(
return out
+# Traced, x.addmm_ copies x first; this op stays in place. No derivative: grad off only.
+_HAS_ADDMM_OP = hasattr(torch.library, "custom_op")
+if _HAS_ADDMM_OP:
+
+ @torch.library.custom_op("unsloth::addmm_", mutates_args = ["out"])
+ def _addmm_op(
+ out: torch.Tensor, A: torch.Tensor, B: torch.Tensor, alpha: float, beta: float
+ ) -> None:
+ out.addmm_(A, B, alpha = alpha, beta = beta)
+
+
+def addmm_(
+ out,
+ A,
+ B,
+ alpha = 1.0,
+ beta = 1.0,
+):
+ if (
+ _HAS_ADDMM_OP
+ and torch.compiler.is_compiling()
+ and not torch.is_grad_enabled()
+ and type(alpha) in (int, float)
+ ):
+ torch.ops.unsloth.addmm_(out, A, B, alpha, beta)
+ return out
+ return out.addmm_(A, B, alpha = alpha, beta = beta)
+
+
def matmul_lora(
X,
W,
@@ -1252,6 +1514,6 @@ def matmul_lora(
if A is not None:
A, B = A.t(), B.t()
XA = torch_matmul(X, A.to(dtype))
- out.addmm_(XA, B.to(dtype), alpha = s)
+ addmm_(out, XA, B.to(dtype), alpha = s)
return out.view(batch, seq_len, -1) if reshape else out