Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fused Triton NF4 dequant and GEMV: bit-exact, stream-safe, torch.compile traceable (#12113)
Daniel Han338d23f6b2f9
diff --git a/tests/test_bnb_integration_compile.py b/tests/test_bnb_integration_compile.pynew file mode 100644index 00000000000..cfd14991a08--- /dev/null+++ b/tests/test_bnb_integration_compile.py@@ -0,0 +1,568 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""The 4bit paths in unsloth.kernels.utils (fast_dequantize, fast_gemv, matmul_lora and the+fast_lora autograd functions) must match bitsandbytes byte for byte, run on whatever stream is+current, survive CUDA graph capture, and trace under torch.compile without graph breaks.++Every correctness test runs twice: on the NF4 kernels and on the bitsandbytes ctypes fallback+(UNSLOTH_BNB_TRITON=0).+"""++import pytest+import torch++pytest.importorskip("bitsandbytes")+if not torch.cuda.is_available():+ pytest.skip("needs a CUDA/HIP device", allow_module_level = True)++import unsloth # noqa: F401 (sets UNSLOTH_IS_PRESENT before transformers)+import bitsandbytes as bnb+import bitsandbytes.functional as F+from unsloth.kernels import utils as U+from unsloth.kernels import fast_lora+from unsloth.kernels.fast_lora import (+ LoRA_W,+ apply_lora_mlp_swiglu,+ apply_lora_o,+ apply_lora_qkv,+)++_NF4_KERNELS_AVAILABLE = U._USE_NF4_KERNELS+DEVICE = "cuda"+# Unsloth runs fp16 on GPUs without native bf16 (pre-Ampere, e.g. T4), and Inductor cannot re-emit a+# bf16 Triton kernel there, so the single-graph checks use the dtype Unsloth would.+CDTYPE = (+ torch.bfloat16+ if torch.version.hip or torch.cuda.get_device_capability()[0] >= 8+ else torch.float16+)++[email protected](params = ["nf4_kernels", "ctypes_fallback"])+def path(request, monkeypatch):+ use = request.param == "nf4_kernels"+ if use and not _NF4_KERNELS_AVAILABLE:+ pytest.skip("NF4 kernels unavailable or disabled")+ monkeypatch.setattr(U, "_USE_NF4_KERNELS", use)+ # Exercise the Triton GEMV even where eager decode would keep bitsandbytes' (Triton < 3.7).+ monkeypatch.setattr(U, "_TRITON_GEMV_EAGER", use)+ U._SCRATCH.clear()+ yield request.param+ U._SCRATCH.clear()+++def nf4_kernels(monkeypatch):+ if not _NF4_KERNELS_AVAILABLE:+ pytest.skip("NF4 kernels unavailable or disabled")+ monkeypatch.setattr(U, "_USE_NF4_KERNELS", True)+ monkeypatch.setattr(U, "_TRITON_GEMV_EAGER", True)+ torch._dynamo.reset()+ yield+ torch._dynamo.reset()+++def _quantize(+ shape,+ dtype,+ nested = True,+ seed = 0,+):+ g = torch.Generator(device = DEVICE).manual_seed(seed)+ W = torch.randn(shape, dtype = dtype, device = DEVICE, generator = g)+ return F.quantize_4bit(W, quant_type = "nf4", compress_statistics = nested)+++def _as_list(s):+ # The pre TimDettmers/bitsandbytes#763 layout fast_dequantize still accepts.+ state2 = [s.state2.absmax, s.state2.code, s.state2.blocksize, None, None, None, None]+ return [s.absmax, s.shape, s.dtype, s.blocksize, [s.offset, state2], s.quant_type, s.code]+++_CONTROL = {}+++def _torch_compiles_lora_exactly(backend):+ """Some torch versions (2.7) decompose the in-place LoRA addmm_/addmv_ differently when+ compiled, even on a plain 16bit weight with no 4bit op in sight. Measure that on a control so+ compiled-vs-eager checks stay exact wherever torch itself is exact."""+ if backend not in _CONTROL:+ g = torch.Generator(device = DEVICE).manual_seed(1)+ r = lambda *s: torch.randn(*s, dtype = CDTYPE, device = DEVICE, generator = g)+ W, A, B, X, x = r(512, 256), r(8, 256), r(512, 8), r(32, 256), r(256)++ def control(X, x):+ out = X @ W.t()+ out.addmm_(X @ A.t(), B.t(), alpha = 2.0)+ v = W @ x+ v.addmv_(B, A @ x, alpha = 2.0)+ return out, v++ torch._dynamo.reset()+ compiled = torch.compile(control, fullgraph = True, backend = backend)(X, x)+ torch._dynamo.reset()+ _CONTROL[backend] = all(torch.equal(a, b) for a, b in zip(compiled, control(X, x)))+ return _CONTROL[backend]+++_TRACES_PARAMS4BIT = []++[email protected]_op("unsloth_test::passthrough", mutates_args = ())+def _passthrough(x: torch.Tensor) -> torch.Tensor:+ return x.clone()+++@_passthrough.register_fake+def _(x):+ return torch.empty_like(x)+++def _dynamo_traces_params4bit():+ """Older Dynamo (torch 2.7) cannot hand a Params4bit weight to any torch.library op (nor call+ some of its methods) without a graph break, whatever the op does. There the fast_lora paths+ can only be checked for correctness under torch.compile, not for a single graph. Probed with+ a throwaway op so a regression in the 4bit ops cannot turn the full-graph checks off."""+ if not _TRACES_PARAMS4BIT:+ lin = bnb.nn.Linear4bit(64, 64, bias = False, quant_type = "nf4").to(DEVICE)+ fn = lambda x: x + _passthrough(lin.weight)[0, 0] + lin.weight.t()[0, 0]+ torch._dynamo.reset()+ breaks = torch._dynamo.explain(fn)(torch.ones(1, device = DEVICE)).graph_break_count+ _TRACES_PARAMS4BIT.append(breaks == 0)+ torch._dynamo.reset()+ return _TRACES_PARAMS4BIT[0]+++def _assert_compiled_matches(+ compiled,+ eager,+ backend = "inductor",+ exact = None,+):+ if exact is None:+ exact = _torch_compiles_lora_exactly(backend)+ if exact:+ assert torch.equal(compiled, eager)+ return+ scale = eager.float().abs().max().clamp_min(1e-6)+ err = ((compiled.float() - eager.float()).abs().max() / scale).item()+ assert err < 2e-2, err+++def _bits(t):+ return t.contiguous().view(torch.int16 if t.element_size() == 2 else torch.int32)++[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("nested", [True, False])[email protected]("shape", [(1024, 768), (130, 70), (4096, 1024)])[email protected]("global_buffer", [False, True])+def test_dequantize_matches_bitsandbytes(path, dtype, nested, shape, global_buffer):+ q, s = _quantize(shape, dtype, nested)+ ref = F.dequantize_4bit(q, s)+ out = U.fast_dequantize(q, s, use_global_buffer = global_buffer)+ assert torch.equal(_bits(out), _bits(ref))+ if nested:+ out_list = U.fast_dequantize(q, _as_list(s), use_global_buffer = global_buffer)+ assert torch.equal(_bits(out_list), _bits(ref))+ # The backward passes dequantize W.t(); a single packed column must come back transposed.+ out_t = U.fast_dequantize(q.t(), s)+ assert out_t.shape == (shape[1], shape[0])+ assert torch.equal(_bits(out_t.t()), _bits(ref))+++def test_explicit_out(path):+ q, s = _quantize((512, 256), torch.bfloat16)+ out = torch.empty((512, 256), dtype = torch.bfloat16, device = DEVICE)+ got = U.fast_dequantize(q, s, out = out)+ assert got.data_ptr() == out.data_ptr()+ assert torch.equal(_bits(out), _bits(F.dequantize_4bit(q, s)))+++def test_fp4_state_uses_its_own_codebook(path):+ W = torch.randn(256, 128, dtype = torch.bfloat16, device = DEVICE)+ q, s = F.quantize_4bit(W, quant_type = "fp4", compress_statistics = True)+ assert torch.equal(_bits(U.fast_dequantize(q, s)), _bits(F.dequantize_4bit(q, s)))++[email protected]("dtype", [torch.bfloat16, torch.float16])+def test_gemv_matches_reference(path, dtype):+ q, s = _quantize((2048, 1024), dtype)+ X = torch.randn(1, 1, 1024, dtype = dtype, device = DEVICE)+ got = U.fast_gemv(X, q, s)+ ref = X.float() @ F.dequantize_4bit(q, s).float().t()+ assert got.shape == (1, 1, 2048)+ rel = ((got.float() - ref).norm() / ref.norm()).item()+ assert rel < 1e-2, rel++[email protected]("storage", [torch.bfloat16, torch.float32], ids = str)+def test_float_packed_storage(path, storage):+ """FSDP-QLoRA packs the 4bit weight into float storage (quant_storage); both the dequant and+ the GEMV must read it as bytes, as the ctypes path always did."""+ W = torch.randn(1024, 512, dtype = torch.bfloat16, device = DEVICE)+ q, s = F.quantize_4bit(W, quant_type = "nf4", compress_statistics = True, quant_storage = storage)+ ref = F.dequantize_4bit(q, s)+ assert torch.equal(_bits(U.fast_dequantize(q, s)), _bits(ref))+ X = torch.randn(1, 1, 512, dtype = torch.bfloat16, device = DEVICE)+ got = U.fast_gemv(X, q, s).float()+ want = (X @ ref.t()).float()+ assert ((got - want).abs().max() / want.abs().max()).item() < 1e-2+++def test_side_streams_never_share_the_scratch(path):+ """Two streams dequantizing different weights with use_global_buffer at the same time: each+ must read its own weight back. Only the default stream owns a scratch, so a program making a+ new stream per step cannot pin a weight-sized buffer per stream."""+ (q1, s1), (q2, s2) = (+ _quantize((4096, 4096), torch.bfloat16, seed = 1),+ _quantize((4096, 4096), torch.bfloat16, seed = 2),+ )+ X = torch.randn(64, 4096, dtype = torch.bfloat16, device = DEVICE)+ refs = [X @ F.dequantize_4bit(q, s).t() for q, s in ((q1, s1), (q2, s2))]+ outs = [None, None]+ torch.cuda.synchronize()+ for _ in range(10):+ for i, (q, s) in enumerate(((q1, s1), (q2, s2))):+ with torch.cuda.stream(torch.cuda.Stream()):+ W = U.fast_dequantize(q, s, use_global_buffer = True)+ outs[i] = X @ W.t()+ torch.cuda.synchronize()+ for got, ref in zip(outs, refs):+ assert torch.equal(got, ref)+ assert not U._SCRATCH+ U.fast_dequantize(q1, s1, use_global_buffer = True)+ assert {k[0] for k in U._SCRATCH} <= {"weight", "absmax"}+ assert len([k for k in U._SCRATCH if k[0] == "weight"]) == 1++[email protected]("shape", [(512, 96), (256, 4160), (130, 100)])+def test_gemv_rows_not_starting_a_block_fall_back_to_dequant(path, shape):+ """Both GEMV kernels assume each row starts a quantization block; other shapes must still be+ right (bitsandbytes' own GEMV is wrong for them too)."""+ n, k = shape+ q, s = _quantize(shape, torch.bfloat16)+ X = torch.randn(1, 1, k, dtype = torch.bfloat16, device = DEVICE)+ ref = (X @ F.dequantize_4bit(q, s).t()).float()+ got = U.fast_gemv(X, q, s).float()+ assert got.shape == (1, 1, n)+ assert ((got - ref).abs().max() / ref.abs().max()).item() < 1e-2+++def test_gemv_side_stream_and_cuda_graph(path):+ q, s = _quantize((4096, 4096), torch.bfloat16)+ X = torch.randn(1, 1, 4096, dtype = torch.bfloat16, device = DEVICE)+ expected = U.fast_gemv(X, q, s)+ side = torch.cuda.Stream()+ side.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(side):+ x = X.clone() * 1 # produced on the side stream, consumed there, no host sync+ results = [U.fast_gemv(x, q, s) for _ in range(10)]+ for _ in range(2):+ U.fast_gemv(x, q, s)+ torch.cuda.current_stream().wait_stream(side)+ torch.cuda.synchronize()+ assert all(torch.equal(r, expected) for r in results)+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ static_out = U.fast_gemv(x, q, s)+ x.copy_(X * 2)+ graph.replay()+ torch.cuda.synchronize()+ assert torch.equal(static_out, U.fast_gemv(X * 2, q, s))+++def test_kernel_failure_falls_back_to_bitsandbytes(nf4_kernels, monkeypatch):+ """A Triton compile failure on an untested GPU switches to the bitsandbytes kernels."""+ q, s = _quantize((512, 256), torch.bfloat16)++ def broken(*args, **kwargs):+ raise RuntimeError("PTX compile failed")++ for name in ("dequantize_nf4", "dequantize_nf4_planned", "gemv_nf4", "gemv_nf4_planned"):+ monkeypatch.setattr(U, name, broken)+ assert torch.equal(_bits(U.fast_dequantize(q, s)), _bits(F.dequantize_4bit(q, s)))+ assert U._USE_NF4_KERNELS is False+ monkeypatch.setattr(U, "_USE_NF4_KERNELS", True)+ X = torch.randn(1, 1, 256, dtype = torch.bfloat16, device = DEVICE)+ assert U.fast_gemv(X, q, s).shape == (1, 1, 512)+ assert U._USE_NF4_KERNELS is False+ # Batch > 1 decode dequantizes into the scratch buffer through the planned launch.+ monkeypatch.setattr(U, "_USE_NF4_KERNELS", True)+ got = U.fast_dequantize(q.t(), s, use_global_buffer = True).t()+ assert torch.equal(_bits(got), _bits(F.dequantize_4bit(q, s)))+ assert U._USE_NF4_KERNELS is False+++def test_out_of_memory_is_not_mistaken_for_a_kernel_failure(nf4_kernels, monkeypatch):+ q, s = _quantize((512, 256), torch.bfloat16)++ def oom(*args, **kwargs):+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")++ monkeypatch.setattr(U, "dequantize_nf4", oom)+ with pytest.raises(torch.cuda.OutOfMemoryError):+ U.fast_dequantize(q, s)+ assert U._USE_NF4_KERNELS is True+++def test_scratch_first_allocated_under_inference_mode_still_trains(path):+ """generate() runs under torch.inference_mode(); the scratch it allocates first must stay+ usable by the training step that follows."""+ q, s = _quantize((512, 256), torch.bfloat16)+ A = (torch.randn(8, 256, dtype = torch.bfloat16, device = DEVICE) * 0.02).requires_grad_()+ B = (torch.randn(512, 8, dtype = torch.bfloat16, device = DEVICE) * 0.02).requires_grad_()+ X = torch.randn(2, 4, 256, dtype = torch.bfloat16, device = DEVICE)+ with torch.inference_mode():+ U.matmul_lora(X, q, s, A.detach(), B.detach(), 2.0)+ assert all(not t.is_inference() for t in U._SCRATCH.values())++ def step():+ x = X.clone().requires_grad_()+ LoRA_W.apply(x, q, s, A, B, 2.0).float().pow(2).mean().backward()+ grads = (x.grad.clone(), A.grad.clone(), B.grad.clone())+ A.grad = B.grad = None+ return grads++ after_generate = step()+ U._SCRATCH.clear()+ fresh = step()+ for a, b in zip(after_generate, fresh):+ assert torch.equal(a, b)+++def test_matmul_lora_cuda_graph_capture_and_replay(path):+ q1, s1 = _quantize((1024, 512), torch.bfloat16, seed = 3)+ q2, s2 = _quantize((1024, 512), torch.bfloat16, seed = 4)+ A = torch.randn(8, 512, dtype = torch.bfloat16, device = DEVICE) * 0.02+ B = torch.randn(1024, 8, dtype = torch.bfloat16, device = DEVICE) * 0.02+ static_x = torch.randn(4, 512, dtype = torch.bfloat16, device = DEVICE)++ side = torch.cuda.Stream()+ side.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(side):+ for _ in range(3):+ U.matmul_lora(static_x, q1, s1, A, B, 2.0)+ U.matmul_lora(static_x, q2, s2, A, B, 2.0)+ torch.cuda.current_stream().wait_stream(side)++ g1, g2 = torch.cuda.CUDAGraph(), torch.cuda.CUDAGraph()+ with torch.cuda.graph(g1):+ static_y1 = U.matmul_lora(static_x, q1, s1, A, B, 2.0)+ with torch.cuda.graph(g2):+ static_y2 = U.matmul_lora(static_x, q2, s2, A, B, 2.0)++ for seed in range(3):+ static_x.copy_(torch.randn(4, 512, dtype = torch.bfloat16, device = DEVICE))+ g1.replay()+ # An eager call in between may grow or reuse the eager scratch; the graphs must not care.+ big_q, big_s = _quantize((4096, 2048), torch.bfloat16, seed = 10 + seed)+ U.fast_dequantize(big_q, big_s, use_global_buffer = True)+ g2.replay()+ torch.cuda.synchronize()+ assert torch.equal(static_y1, U.matmul_lora(static_x, q1, s1, A, B, 2.0))+ assert torch.equal(static_y2, U.matmul_lora(static_x, q2, s2, A, B, 2.0))+++def _lora_block(+ dtype = torch.bfloat16,+ D = 256,+ H = 512,+):+ import torch.nn as nn+ from peft import LoraConfig, get_peft_model++ torch.manual_seed(0)++ class Block(nn.Module):+ def __init__(self):+ super().__init__()+ L = lambda i, o: bnb.nn.Linear4bit(+ i, o, bias = False, compute_dtype = dtype, quant_type = "nf4"+ )+ self.gate_proj, self.up_proj, self.down_proj = L(D, H), L(D, H), L(H, D)+ self.q_proj, self.k_proj, self.v_proj, self.o_proj = (+ L(D, D),+ L(D, 128),+ L(D, 128),+ L(D, D),+ )++ block = Block()+ for module in block.modules():+ if isinstance(module, bnb.nn.Linear4bit):+ module.weight = bnb.nn.Params4bit(+ (torch.randn(module.out_features, module.in_features) * 0.05).to(dtype),+ requires_grad = False,+ quant_type = "nf4",+ compress_statistics = True,+ )+ targets = ["gate_proj", "up_proj", "down_proj", "q_proj", "k_proj", "v_proj", "o_proj"]+ model = get_peft_model(block.to(DEVICE), LoraConfig(r = 8, lora_alpha = 16, target_modules = targets))+ for name, p in model.named_parameters():+ if "lora_B" in name:+ torch.nn.init.normal_(p, std = 0.02)+ return model, model.base_model.model+++_LORA_FNS = {+ "mlp": lambda block: (lambda x: apply_lora_mlp_swiglu(block, x)),+ "qkv": lambda block: (lambda x: apply_lora_qkv(block, x, inplace = False)),+ "o": lambda block: (lambda x: apply_lora_o(block, x)),+}+++def _fwd_bwd(model, fn, X):+ x = X.clone().requires_grad_()+ out = fn(x)+ outs = out if isinstance(out, tuple) else (out,)+ sum(o.float().pow(2).mean() for o in outs).backward()+ grads = [x.grad] + [p.grad for p in model.parameters() if p.grad is not None]+ result = [o.detach().clone() for o in outs] + [g.clone() for g in grads]+ model.zero_grad(set_to_none = True)+ return result++[email protected]("which", ["mlp", "qkv", "o"])+def test_fast_lora_compiles_fullgraph(nf4_kernels, which):+ model, block = _lora_block(dtype = CDTYPE)+ fn = _LORA_FNS[which](block)+ X = torch.randn(2, 16, 256, dtype = CDTYPE, device = DEVICE)+ eager = _fwd_bwd(model, fn, X)+ # Before torch 2.11 the LoRA Functions stay opaque to torch.compile (fast_lora.TRACE_LORA_FUNCTIONS).+ fullgraph = _dynamo_traces_params4bit() and fast_lora.TRACE_LORA_FUNCTIONS+ if fullgraph:+ explained = torch._dynamo.explain(fn)(X.clone().requires_grad_())+ assert explained.graph_break_count == 0, explained.break_reasons+ torch._dynamo.reset()+ compiled = _fwd_bwd(model, torch.compile(fn, fullgraph = fullgraph), X)+ # The dequant is exact under compile; the plain torch LoRA matmuls around it need not be.+ assert len(compiled) == len(eager)+ for a, b in zip(eager, compiled):+ _assert_compiled_matches(b, a, exact = False)+++def test_primitives_compile_fullgraph(nf4_kernels):+ q, s = _quantize((512, 256), CDTYPE)+ X1 = torch.randn(1, 1, 256, dtype = CDTYPE, device = DEVICE)+ X = torch.randn(2, 16, 256, dtype = CDTYPE, device = DEVICE)+ A = torch.randn(8, 256, dtype = CDTYPE, device = DEVICE) * 0.02+ B = torch.randn(512, 8, dtype = CDTYPE, device = DEVICE) * 0.02+ cases = [+ (lambda q: U.fast_dequantize(q, s, use_global_buffer = True) * 1, (q,), True),+ (lambda q: U.fast_dequantize(q.t(), s) * 1, (q,), True),+ # Inductor re-emits the GEMV reduction and may round a partial sum 1 ulp differently.+ (lambda x: U.fast_gemv(x, q, s) * 1, (X1,), "close"),+ (lambda x: U.matmul_lora(x, q, s, A, B, 2.0) * 1, (X,), False),+ ]+ for fn, args, always_exact in cases:+ explained = torch._dynamo.explain(fn)(*args)+ assert explained.graph_break_count == 0, explained.break_reasons+ torch._dynamo.reset()+ compiled, eager = torch.compile(fn, fullgraph = True)(*args), fn(*args)+ if always_exact == "close":+ _assert_compiled_matches(compiled, eager, exact = False)+ elif always_exact:+ assert torch.equal(compiled, eager)+ else:+ _assert_compiled_matches(compiled, eager)+ torch._dynamo.reset()++[email protected]("bsz", [1, 2])+def test_fast_linear_forward_compiles_fullgraph(nf4_kernels, bsz):+ _, block = _lora_block(dtype = CDTYPE)+ fn = lambda x: U.fast_linear_forward(block.o_proj, x)+ X = torch.randn(bsz, 1, 256, dtype = CDTYPE, device = DEVICE)+ with torch.inference_mode():+ eager = fn(X).clone()+ fullgraph = _dynamo_traces_params4bit()+ if fullgraph:+ explained = torch._dynamo.explain(fn)(X)+ assert explained.graph_break_count == 0, explained.break_reasons+ torch._dynamo.reset()+ compiled = torch.compile(fn, fullgraph = fullgraph, backend = "aot_eager")(X)+ # aot_eager: inductor may lower the bsz=1 LoRA addmv one bf16 ulp differently.+ _assert_compiled_matches(compiled, eager, backend = "aot_eager")+++def test_eager_gemv_gate_off_takes_bitsandbytes(monkeypatch):+ """Where the Triton GEMV is not the eager choice (HIP before Triton 3.7), eager decode takes+ bitsandbytes' GEMV, on the live stream. On CUDA the Triton GEMV is the eager choice."""+ from unsloth.kernels import nf4_gemv++ if torch.version.hip is None and _NF4_KERNELS_AVAILABLE:+ assert nf4_gemv.triton_gemv_eager()+ if not _NF4_KERNELS_AVAILABLE:+ pytest.skip("NF4 kernels unavailable or disabled")+ monkeypatch.setattr(U, "_USE_NF4_KERNELS", True)+ monkeypatch.setattr(U, "_TRITON_GEMV_EAGER", False)+ q, s = _quantize((512, 256), torch.bfloat16)+ X = torch.randn(1, 1, 256, dtype = torch.bfloat16, device = DEVICE)+ stream = torch.cuda.Stream()+ stream.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(stream):+ out = U.fast_gemv(X, q, s)+ stream.synchronize()+ assert torch.equal(out, U._fast_gemv_ctypes(X, q, s))+++def test_graph_break_is_not_mistaken_for_a_kernel_failure(nf4_kernels, monkeypatch):+ """A Dynamo exception surfacing through fast_dequantize (e.g. an older Dynamo tracing the+ fallback's try/except) must re-raise, not switch the kernels off for the process."""+ q, s = _quantize((512, 256), torch.bfloat16)++ def unsupported(*args, **kwargs):+ raise torch._dynamo.exc.Unsupported("Data-dependent branching")++ monkeypatch.setattr(U, "dequantize_nf4", unsupported)+ with pytest.raises(torch._dynamo.exc.Unsupported):+ U.fast_dequantize(q, s)+ assert U._USE_NF4_KERNELS is True+++# Runs as a module-level script: only that shape reproduced the torch 2.10 stale reads.+_STALE_MEMORY_SCRIPT = """+import sys, torch+sys.path.insert(0, sys.argv[1])+from unsloth.kernels import fast_lora+import test_bnb_integration_compile as t++model, block = t._lora_block(dtype = getattr(torch, sys.argv[3]))+fn = lambda x: getattr(fast_lora, sys.argv[2])(block, x)+X = torch.randn(2, 16, 256, dtype = getattr(torch, sys.argv[3]), device = "cuda")+compiled = torch.compile(fn)+eager = t._fwd_bwd(model, fn, X)+bad = 0+for _ in range(10):+ poison = [torch.full((n,), float("nan"), device = "cuda") for n in (2048, 4096, 8192, 16384, 65536, 131072)]+ del poison+ # The previous step's outputs stay alive until this one returns, as in a training loop.+ out = t._fwd_bwd(model, compiled, X)+ bad += sum((~torch.isfinite(o)).sum().item() for o in out)+print("NONFINITE", bad)+"""++[email protected]("mlp", ["swiglu", "geglu_exact", "geglu_approx"])+def test_compiled_lora_mlp_backward_never_reads_stale_memory(mlp):+ """torch 2.10 miscompiles the traced LoRA_MLP backward: with freed NaN memory around, LoRA+ gradients came back non-finite. There the LoRA Functions must stay opaque to torch.compile."""+ import os, subprocess, sys++ tests_dir = os.path.dirname(os.path.abspath(__file__))+ run = subprocess.run(+ [+ sys.executable,+ "-c",+ _STALE_MEMORY_SCRIPT,+ tests_dir,+ f"apply_lora_mlp_{mlp}",+ str(CDTYPE).split(".")[-1],+ ],+ capture_output = True,+ text = True,+ timeout = 600,+ )+ assert "NONFINITE 0" in run.stdout, run.stdout[-2000:] + run.stderr[-3000:]diff --git a/tests/test_lora_traced_inplace.py b/tests/test_lora_traced_inplace.pynew file mode 100644index 00000000000..1bf9f0078f3--- /dev/null+++ b/tests/test_lora_traced_inplace.py@@ -0,0 +1,83 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Traced LoRA backward and forward without copies: the accumulating addmm_ runs in place under+torch.compile (unsloth::addmm_), and the SwiGLU backward is plain torch math there, so Inductor+neither copies the accumulator nor the saved activations. Both must match eager."""++import pytest+import torch++if not torch.cuda.is_available():+ pytest.skip("needs a CUDA GPU", allow_module_level = True)+pytest.importorskip("triton")++from unsloth.kernels import utils+from unsloth.kernels.swiglu import swiglu_DWf_DW_dfg_kernel+++def _inputs(seed = 0):+ g = torch.Generator(device = "cuda").manual_seed(seed)+ d = dict(device = "cuda", dtype = torch.bfloat16, generator = g)+ return (+ torch.randn(64, 256, **d),+ torch.randn(512, 256, **d) / 16,+ torch.randn(8, 256, **d) / 16,+ torch.randn(512, 8, **d) / 16,+ )+++def _lora_forward(X, W, A, B):+ out = X @ W.t()+ return utils.addmm_(out, X @ A.t(), B.t(), alpha = 2.0)++[email protected](not utils._HAS_ADDMM_OP, reason = "torch.library.custom_op needs torch 2.4+")+def test_traced_addmm_is_the_in_place_op_and_matches_eager():+ torch._dynamo.reset()+ graphs = []++ def backend(gm, example_inputs):+ graphs.append(gm)+ return gm++ X, W, A, B = _inputs()+ with torch.no_grad():+ compiled = torch.compile(_lora_forward, backend = backend, fullgraph = True)(X, W, A, B)+ inductor = torch.compile(_lora_forward, fullgraph = True)(X, W, A, B)+ eager = _lora_forward(X, W, A, B)+ targets = [node.target for gm in graphs for node in gm.graph.nodes]+ assert torch.ops.unsloth.addmm_.default in targets or torch.ops.unsloth.addmm_ in targets+ assert torch.equal(compiled, eager) and torch.equal(inductor, eager)+++def test_addmm_keeps_autograd_with_grad_enabled():+ # The custom op has no derivative; with grad enabled the plain addmm_ must be used.+ torch._dynamo.reset()+ X, W, A, B = _inputs(1)+ W.requires_grad_(True)+ out = torch.compile(_lora_forward, fullgraph = True)(X, W, A, B)+ out.float().sum().backward()+ assert W.grad is not None and torch.isfinite(W.grad).all()+++# bf16 only where Unsloth trains in it (compute capability 8+); T4 trains in fp16.+DTYPES = [torch.float16] + ([torch.bfloat16] if torch.cuda.get_device_capability()[0] >= 8 else [])++[email protected]("dtype", DTYPES, ids = str)+def test_traced_swiglu_backward_matches_the_kernel(dtype):+ torch._dynamo.reset()+ g = torch.Generator(device = "cuda").manual_seed(2)+ d = dict(device = "cuda", dtype = dtype, generator = g)+ DW, e, gate = (+ torch.randn(256, 1024, **d),+ torch.randn(256, 1024, **d),+ torch.randn(256, 1024, **d),+ )+ ref = swiglu_DWf_DW_dfg_kernel(DW.clone(), e.clone(), gate.clone())+ traced = torch.compile(swiglu_DWf_DW_dfg_kernel, fullgraph = True)(DW, e, gate)+ for name, a, b in zip(("h", "df", "de"), traced, ref):+ # Inductor keeps the intermediate products in fp32 where the kernel rounds them to dtype.+ err = ((a.float() - b.float()).abs().max() / b.float().abs().max()).item()+ assert err < 1e-2, (name, err)diff --git a/tests/test_nf4_triton_dequant.py b/tests/test_nf4_triton_dequant.pynew file mode 100644index 00000000000..6cc2f1835f6--- /dev/null+++ b/tests/test_nf4_triton_dequant.py@@ -0,0 +1,317 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""The fused Triton NF4 dequant kernel must be byte-identical to bitsandbytes. Streams, CUDA+graphs and torch.compile through fast_dequantize are covered in test_bnb_integration_compile.py."""++import hashlib+import os++import pytest+import torch++if not torch.cuda.is_available():+ pytest.skip("needs a CUDA device", allow_module_level = True)++pytest.importorskip("triton")+bnb_functional = pytest.importorskip("bitsandbytes.functional")++from unsloth.kernels import nf4 as nf4_mod+from unsloth.kernels import nf4_gemv as nf4_gemv_mod+from unsloth.kernels.nf4 import dequantize_nf4++_INT_VIEW = {torch.float16: torch.int16, torch.bfloat16: torch.int16, torch.float32: torch.int32}++# nf4_dequantization_lut in bitsandbytes csrc/kernels.cu.+CSRC_NF4 = (+ -1.0,+ -0.6961928009986877,+ -0.5250730514526367,+ -0.39491748809814453,+ -0.28444138169288635,+ -0.18477343022823334,+ -0.09105003625154495,+ 0.0,+ 0.07958029955625534,+ 0.16093020141124725,+ 0.24611230194568634,+ 0.33791524171829224,+ 0.44070982933044434,+ 0.5626170039176941,+ 0.7229568362236023,+ 1.0,+)+++def _bytes_equal(a, b):+ assert a.shape == b.shape and a.dtype == b.dtype+ iv = _INT_VIEW[a.dtype]+ return torch.equal(a.contiguous().view(iv), b.contiguous().view(iv))+++def _quantize(+ shape,+ dtype,+ blocksize = 64,+ nested = True,+ seed = 0,+):+ g = torch.Generator(device = "cuda").manual_seed(seed)+ W = torch.randn(shape, dtype = dtype, device = "cuda", generator = g)+ return bnb_functional.quantize_4bit(+ W, blocksize = blocksize, quant_type = "nf4", compress_statistics = nested+ )+++def _args(q, s):+ if s.nested:+ return (+ q,+ s.absmax,+ s.state2.code,+ s.state2.absmax,+ s.offset,+ s.code,+ s.blocksize,+ s.state2.blocksize,+ s.shape,+ s.dtype,+ )+ return (q, s.absmax, None, None, None, s.code, s.blocksize, 0, s.shape, s.dtype)+++SHAPES = [(1, 4096), (4096, 1), (17, 33), (4096, 4096), (14336, 4096), (4096, 14336)]++[email protected]("shape", SHAPES, ids = lambda s: f"{s[0]}x{s[1]}")[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32], ids = str)[email protected]("blocksize", [64, 128])[email protected]("nested", [True, False], ids = ["nested", "flat"])+def test_bytes_match_bitsandbytes(shape, dtype, blocksize, nested):+ q, s = _quantize(shape, dtype, blocksize, nested)+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), bnb_functional.dequantize_4bit(q, s))+++def test_vocab_sized_weight_matches_bitsandbytes():+ if torch.cuda.mem_get_info()[0] < 8 * 2**30:+ pytest.skip("not enough free GPU memory for the vocab-sized weight")+ q, s = _quantize((128256, 4096), torch.bfloat16)+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), bnb_functional.dequantize_4bit(q, s))+++def test_quant_state_code_is_the_kernel_table():+ """fast_dequantize passes quant_state.code as the table; bitsandbytes' own kernel uses the+ hardcoded csrc table, so the two must be the same fp32 values."""+ for dtype in (torch.float16, torch.bfloat16):+ _, s = _quantize((64, 64), dtype)+ csrc = torch.tensor(CSRC_NF4, dtype = torch.float32)+ assert torch.equal(s.code.cpu().view(torch.int32), csrc.view(torch.int32))+++def test_explicit_out_is_written_and_returned():+ q, s = _quantize((256, 512), torch.bfloat16)+ out = torch.full(s.shape, 7.0, dtype = s.dtype, device = "cuda")+ assert dequantize_nf4(*_args(q, s), out = out).data_ptr() == out.data_ptr()+ assert _bytes_equal(out, bnb_functional.dequantize_4bit(q, s))+++def test_compiled_out_variant():+ q, s = _quantize((256, 512), torch.float16, seed = 8)++ def f(q, absmax, out):+ args = list(_args(q, s))+ args[1] = absmax+ dequantize_nf4(*args, out = out)+ return out * 1++ torch._dynamo.reset()+ out = torch.empty(s.shape, dtype = s.dtype, device = "cuda")+ res = torch.compile(f, fullgraph = True)(q, s.absmax, out)+ ref = bnb_functional.dequantize_4bit(q, s)+ assert _bytes_equal(out, ref) and _bytes_equal(res, ref)+++def test_rejects_unsupported_dtype_and_blocksize():+ q, s = _quantize((64, 64), torch.bfloat16)+ args = list(_args(q, s))+ with pytest.raises(TypeError):+ dequantize_nf4(*args[:-1], torch.int8)+ args[6] = 48+ with pytest.raises(ValueError):+ dequantize_nf4(*args)+++def test_fma_contraction_would_be_caught():+ """Control: a contracted fma(code2, absmax2, offset) differs from the two-rounding result+ somewhere, so the bytes test above can tell a fused kernel from a correct one."""+ _, s = _quantize((4096, 4096), torch.float32)+ code2, absmax2 = s.state2.code, s.state2.absmax+ blk2 = torch.arange(s.absmax.numel(), device = "cuda") // s.state2.blocksize+ two_roundings = (code2[s.absmax.long()] * absmax2[blk2]) + s.offset+ one_rounding = (+ code2[s.absmax.long()].double() * absmax2[blk2].double() + s.offset.double()+ ).float()+ assert not torch.equal(two_roundings, one_rounding)++[email protected](not nf4_mod._HAS_MUL_RN, reason = "already the no-mul_rn route")+def test_hip_route_without_mul_rn_is_also_exact(monkeypatch):+ """The HIP route (plain multiply, fp fusion disabled at launch) exercised on CUDA."""+ monkeypatch.setattr(nf4_mod, "_HAS_MUL_RN", False)+ for dtype in (torch.float32, torch.bfloat16):+ q, s = _quantize((4096, 4096), dtype, seed = 3)+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), bnb_functional.dequantize_4bit(q, s))++[email protected]("fp_fusion", [False, True], ids = ["fusion_off", "fusion_on"])+def test_exact_with_fp_fusion_on_or_off(fp_fusion):+ """Eager launches with fp fusion off; torch.compile re-emits the kernel with it on. Both must+ keep code2 * absmax2 and + offset as two roundings (fp32 output shows a 1 ulp scale error)."""+ if fp_fusion and not nf4_mod._HAS_MUL_RN:+ # Without libdevice.mul_rn (HIP, the interpreter) the kernel never runs with fusion on.+ pytest.skip("no mul_rn: this route never launches with fp fusion on")+ for dtype in (torch.float32, torch.bfloat16):+ q, s = _quantize((4096, 4096), dtype, seed = 3)+ out = torch.empty(4096, 4096, dtype = dtype, device = "cuda")+ nf4_mod._launch(+ nf4_mod._nf4_dequant_kernel,+ q,+ s.absmax,+ s.state2.code,+ s.state2.absmax,+ s.offset,+ s.code,+ s.blocksize,+ s.state2.blocksize,+ out,+ fp_fusion = fp_fusion,+ )+ assert _bytes_equal(out, bnb_functional.dequantize_4bit(q, s))++[email protected](not nf4_mod._HAS_MUL_RN, reason = "the plain multiply keeps subnormals")[email protected]("nested", [True, False], ids = ["nested", "flat"])+def test_subnormal_products_flush_like_bitsandbytes(nested):+ """bitsandbytes' kernels flush subnormal products to zero, in the nested absmax decode and in+ code * absmax; the Triton kernel must too."""+ q, s = _quantize((256, 1024), torch.float32, nested = nested, seed = 5)+ if nested:+ s.state2.absmax.mul_(1e-36)+ s.offset.zero_()+ else:+ s.absmax.fill_(2e-38)+ ref = bnb_functional.dequantize_4bit(q, s)+ assert ref.abs().max() > 0 and (ref == 0).float().mean() > 0.1+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), ref)++[email protected]("storage", [torch.bfloat16, torch.float16, torch.float32], ids = str)+def test_float_packed_storage_is_read_as_bytes(storage):+ """FSDP-QLoRA packs the 4bit weight into float storage (quant_storage); it is still bytes."""+ g = torch.Generator(device = "cuda").manual_seed(4)+ W = torch.randn(512, 256, dtype = torch.bfloat16, device = "cuda", generator = g)+ q, s = bnb_functional.quantize_4bit(+ W, quant_type = "nf4", compress_statistics = True, quant_storage = storage+ )+ assert q.dtype == storage+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), bnb_functional.dequantize_4bit(q, s))++[email protected]("lut_mode", [0, 1, 2])[email protected]("words", [False, True])[email protected]("evict", [False, True])[email protected]("shape", [(17, 33), (5, 4096), (1024, 4096)], ids = str)+def test_every_launch_config_is_exact(shape, words, evict, lut_mode, monkeypatch):+ # The per-GPU launch table only picks speed; every knob combination must stay bit-exact.+ for dtype in (torch.float16, torch.bfloat16):+ for blocksize in (64, 128):+ q, s = _quantize(shape, dtype, blocksize)+ ref = bnb_functional.dequantize_4bit(q, s)+ for target in (256, 2048):+ monkeypatch.setattr(+ nf4_mod, "_CONFIG_OVERRIDE", (target, 4, words, evict, lut_mode)+ )+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), ref), (dtype, blocksize, target)+++def test_every_table_config_is_exact_here(monkeypatch):+ # Each GPU's table entry, run on this GPU: exactness does not depend on the GPU it was tuned on.+ configs = {row[1:] for table in nf4_mod._CONFIGS.values() for row in table}+ for shape in [(17, 33), (1024, 4096)]:+ for dtype in (torch.float16, torch.bfloat16):+ q, s = _quantize(shape, dtype, 64)+ ref = bnb_functional.dequantize_4bit(q, s)+ for config in sorted(configs):+ monkeypatch.setattr(nf4_mod, "_CONFIG_OVERRIDE", config)+ assert _bytes_equal(dequantize_nf4(*_args(q, s)), ref), (shape, dtype, config)+++def test_compile_cache_key_covers_the_kernel_source():+ """Inductor's FX graph cache keys a triton_op call without the Triton source behind it: with a+ warm cache, an edited kernel came back as the previous kernel's code. The kernel files' hashes+ in cache_key_tag make an edited kernel a cache miss."""+ if not hasattr(getattr(torch.compiler, "config", None), "cache_key_tag"):+ pytest.skip("torch has no compile cache key tag")+ tags = torch.compiler.config.cache_key_tag.split(",")+ for mod in (nf4_mod, nf4_gemv_mod):+ with open(mod.__file__, "rb") as file:+ digest = hashlib.sha256(file.read()).hexdigest()[:16]+ assert f"unsloth/{os.path.basename(mod.__file__)}:{digest}" in tags+++def _utils_with_kernels():+ from unsloth.kernels import utils+ if not utils._USE_NF4_KERNELS:+ pytest.skip("the fused NF4 kernels are off here (UNSLOTH_BNB_TRITON=0 or no Triton)")+ return utils++[email protected]("nested", [True, False], ids = ["nested", "flat"])+def test_decode_plan_reruns_match_bitsandbytes(nested):+ """Decode dequantizes each weight into the shared scratch buffer through a launch planned on+ its first call. Every rerun, weights interleaved as in a decoder, stays byte-identical."""+ from unsloth.kernels import triton_launch++ utils = _utils_with_kernels()+ shapes = [(512, 256), (256, 1024), (768, 256)]+ weights = [_quantize(s, torch.bfloat16, nested = nested, seed = i) for i, s in enumerate(shapes)]+ for _ in range(3):+ for q, s in weights:+ got = utils.fast_dequantize(q.t(), s, use_global_buffer = True).t().clone()+ assert _bytes_equal(got, bnb_functional.dequantize_4bit(q, s))+ if triton_launch._ENABLED:+ assert all(getattr(s, "_unsloth_nf4_plan", None) for _, s in weights)+++def test_decode_plan_is_rebuilt_when_the_quant_state_changes():+ """QuantState.to() swaps in new tensors and a model cast changes its dtype: the planned launch+ would read the old absmax or write the old dtype, so it must be rebuilt."""+ utils = _utils_with_kernels()+ q, s = _quantize((512, 256), torch.bfloat16, nested = False)+ for _ in range(2):+ utils.fast_dequantize(q.t(), s, use_global_buffer = True)+ s.absmax = s.absmax * 2+ got = utils.fast_dequantize(q.t(), s, use_global_buffer = True).t().clone()+ assert _bytes_equal(got, bnb_functional.dequantize_4bit(q, s))+ s.dtype = torch.float16+ got = utils.fast_dequantize(q.t(), s, use_global_buffer = True).t().clone()+ assert got.dtype == torch.float16+ assert _bytes_equal(got, bnb_functional.dequantize_4bit(q, s))+++def test_decode_plan_is_not_copied_or_pickled_with_the_model():+ """The plan holds a closure and the scratch buffer: deepcopy (reference models) and pickling+ must drop it, and the copy rebuilds its own on first use."""+ import copy+ import pickle++ utils = _utils_with_kernels()+ q, s = _quantize((512, 256), torch.bfloat16)+ for _ in range(2):+ utils.fast_dequantize(q.t(), s, use_global_buffer = True)+ s2 = copy.deepcopy(s)+ assert getattr(s2, "_unsloth_nf4_plan", None) is None+ assert getattr(pickle.loads(pickle.dumps(s)), "_unsloth_nf4_plan", None) is None+ q2 = q.clone()+ got = utils.fast_dequantize(q2.t(), s2, use_global_buffer = True).t().clone()+ assert _bytes_equal(got, bnb_functional.dequantize_4bit(q, s))diff --git a/tests/test_nf4_triton_gemv.py b/tests/test_nf4_triton_gemv.pynew file mode 100644index 00000000000..7778639f2e3--- /dev/null+++ b/tests/test_nf4_triton_gemv.py@@ -0,0 +1,169 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Batch-1 NF4 GEMV kernel (unsloth/kernels/nf4_gemv.py). It reduces each quantization block+before scaling it, so it is held to an fp32 reference with a tolerance and must never be worse+than bitsandbytes' own GEMV. Streams, CUDA graphs and torch.compile through fast_gemv are covered+in test_bnb_integration_compile.py."""++import pytest+import torch++if not torch.cuda.is_available():+ pytest.skip("needs a CUDA GPU", allow_module_level = True)+pytest.importorskip("triton")+F = pytest.importorskip("bitsandbytes.functional")++from unsloth.kernels import nf4_gemv+from unsloth.kernels.nf4_gemv import gemv_nf4+from unsloth.kernels.utils import _fast_gemv_ctypes++SHAPES = [(4096, 4096), (14336, 4096), (4096, 14336), (1024, 4096)]+DTYPES = [torch.float16, torch.bfloat16]+++def _quant(+ n,+ k,+ dtype,+ nested = True,+ seed = 0,+):+ g = torch.Generator(device = "cuda").manual_seed(seed)+ W = torch.randn(n, k, dtype = dtype, device = "cuda", generator = g)+ return F.quantize_4bit(W, quant_type = "nf4", compress_statistics = nested)+++def _args(q, s):+ if s.nested:+ return (+ q,+ s.absmax,+ s.state2.code,+ s.state2.absmax,+ s.offset,+ s.code,+ s.blocksize,+ s.state2.blocksize,+ s.shape,+ s.dtype,+ )+ return (q, s.absmax, None, None, None, s.code, s.blocksize, None, s.shape, s.dtype)+++def _reference(X, q, s):+ n, k = s.shape+ return (X.float().view(1, k) @ F.dequantize_4bit(q, s).float().t()).view(1, 1, n)+++def _rel_err(a, ref):+ return ((a.float() - ref).abs().max() / ref.abs().max()).item()+++# bf16 output rounding alone is about 4e-3 relative; bitsandbytes lands at the same level.+TOL = {torch.float16: 2e-3, torch.bfloat16: 1.2e-2}++[email protected](params = [None, "narrow", "bytes", "words"], ids = ["auto", "narrow", "bytes", "words"])+def kernel(request, monkeypatch):+ # Both kernel forms, whichever this Triton version and GPU would pick on its own, and the+ # T4 / L4 large-weight config on this GPU.+ if request.param == "narrow":+ capability = torch.cuda.get_device_capability()+ monkeypatch.setattr(nf4_gemv, "_NARROW_CAPS", (capability,))+ nf4_gemv._gemv_config.cache_clear()+ request.addfinalizer(nf4_gemv._gemv_config.cache_clear)+ else:+ monkeypatch.setattr(nf4_gemv, "_FORCE_KERNEL", request.param)+ return request.param++[email protected]("nested", [True, False], ids = ["nested", "flat"])[email protected]("dtype", DTYPES, ids = str)[email protected]("shape", SHAPES, ids = lambda s: f"{s[0]}x{s[1]}")+def test_matches_fp32_reference_and_bitsandbytes(shape, dtype, nested, kernel):+ n, k = shape+ q, s = _quant(n, k, dtype, nested)+ X = torch.randn(1, 1, k, dtype = dtype, device = "cuda")+ ref = _reference(X, q, s)+ out = gemv_nf4(X, *_args(q, s))+ assert out.shape == (1, 1, n) and out.dtype == dtype+ assert _rel_err(out, ref) < TOL[dtype]+ if nested: # the ctypes GEMV only takes double quantized states+ assert _rel_err(out, ref) <= _rel_err(_fast_gemv_ctypes(X, q, s), ref) + 4e-3+++def test_unaligned_weight_view_uses_the_byte_kernel(monkeypatch):+ # The words kernel reads int32; a weight view starting off a 4 byte boundary must not reach it.+ monkeypatch.setattr(nf4_gemv, "_FORCE_KERNEL", "words")+ q, s = _quant(1024, 4096, torch.float16)+ buf = torch.empty(q.numel() + 2, dtype = torch.uint8, device = "cuda")+ buf[2:].copy_(q.view(-1))+ Wv = buf[2:].view(q.shape)+ assert not nf4_gemv._word_aligned(Wv)+ X = torch.randn(1, 1, 4096, dtype = torch.float16, device = "cuda")+ assert _rel_err(gemv_nf4(X, Wv, *_args(q, s)[1:]), _reference(X, q, s)) < TOL[torch.float16]+++def test_vocab_sized_weight():+ q, s = _quant(128256, 4096, torch.bfloat16)+ X = torch.randn(1, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ assert _rel_err(gemv_nf4(X, *_args(q, s)), _reference(X, q, s)) < TOL[torch.bfloat16]+++def test_explicit_out_is_written_and_returned():+ q, s = _quant(1024, 4096, torch.bfloat16)+ X = torch.randn(1, 1, 4096, dtype = torch.bfloat16, device = "cuda")+ out = torch.empty(1, 1, 1024, dtype = torch.bfloat16, device = "cuda")+ assert gemv_nf4(X, *_args(q, s), out = out).data_ptr() == out.data_ptr()+ assert torch.equal(out, gemv_nf4(X, *_args(q, s)))+++def _utils_with_kernels():+ from unsloth.kernels import utils+ if not (utils._USE_NF4_KERNELS and utils._TRITON_GEMV_EAGER):+ pytest.skip("eager decode does not take the Triton GEMV here")+ return utils++[email protected]("nested", [True, False], ids = ["nested", "flat"])+def test_decode_plan_reruns_match_the_kernel(nested):+ """Eager decode repeats the launch planned on each weight's first call. With weights+ interleaved as in a decoder, every rerun equals a fresh gemv_nf4 launch bit for bit."""+ from unsloth.kernels import triton_launch++ utils = _utils_with_kernels()+ weights = [+ _quant(n, k, torch.bfloat16, nested = nested, seed = i)+ for i, (n, k) in enumerate([(512, 256), (256, 1024), (768, 512)])+ ]+ for step in range(3):+ for i, (q, s) in enumerate(weights):+ X = torch.randn(1, 1, s.shape[1], dtype = torch.bfloat16, device = "cuda")+ got = utils.fast_gemv(X, q, s)+ want = gemv_nf4(X, *_args(q, s))+ assert torch.equal(got, want), (step, i)+ if triton_launch._ENABLED:+ assert all(getattr(s, "_unsloth_gemv_plan", None) for _, s in weights)+++def test_decode_plan_is_rebuilt_or_bypassed_when_inputs_change():+ """New absmax (QuantState.to), another activation dtype, or an activation that is not 16 byte+ aligned would not fit the planned launch: each must still give the kernel's own result."""+ import copy+ import pickle++ utils = _utils_with_kernels()+ q, s = _quant(512, 256, torch.bfloat16, nested = False)+ X = torch.randn(1, 1, 256, dtype = torch.bfloat16, device = "cuda")+ for _ in range(2):+ utils.fast_gemv(X, q, s)+ s.absmax = s.absmax * 2+ assert torch.equal(utils.fast_gemv(X, q, s), gemv_nf4(X, *_args(q, s)))+ X16 = X.half()+ assert torch.equal(utils.fast_gemv(X16, q, s), gemv_nf4(X16, *_args(q, s)))+ base = torch.randn(1, 1, 257, dtype = torch.bfloat16, device = "cuda")+ Xm = base[:, :, 1:] # 2 bytes past a 16 byte boundary+ assert torch.equal(utils.fast_gemv(Xm, q, s), gemv_nf4(Xm.contiguous(), *_args(q, s)))+ assert getattr(copy.deepcopy(s), "_unsloth_gemv_plan", None) is None+ assert getattr(pickle.loads(pickle.dumps(s)), "_unsloth_gemv_plan", None) is Nonediff --git a/tests/test_triton_launch.py b/tests/test_triton_launch.pynew file mode 100644index 00000000000..ee678379cb6--- /dev/null+++ b/tests/test_triton_launch.py@@ -0,0 +1,141 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""unsloth/kernels/triton_launch.py launches a cached compiled Triton kernel directly. It must give+the same result as Triton's own launch for every specialization (alignment, integers equal to 1 or+divisible by 16), launch on the current stream, and step aside when a launch hook is registered."""++import pytest+import torch++triton = pytest.importorskip("triton")+import triton.language as tl++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")++from unsloth.kernels import triton_launch+++def _axpy(x_ptr, y_ptr, out_ptr, n, scale, BLOCK: tl.constexpr, ADD: tl.constexpr):+ i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)+ m = i < n+ v = tl.load(x_ptr + i, mask = m) * scale+ if ADD:+ v += tl.load(y_ptr + i, mask = m)+ tl.store(out_ptr + i, v, mask = m)+++def _run(+ x,+ y,+ out,+ n,+ scale,+ add,+ block = 128,+):+ grid = (-(-n // block),)+ triton_launch.launch(+ _axpy,+ grid,+ (x, y, out, n, scale),+ 3,+ dict(BLOCK = block, ADD = add),+ x.device.index,+ num_warps = 4,+ )+ return out++[email protected](autouse = True)+def _fresh_cache(monkeypatch):+ monkeypatch.setattr(triton_launch, "_CACHE", {})+ monkeypatch.setattr(+ triton_launch,+ "_ENABLED",+ triton_launch._raw_stream is not None and torch.version.hip is None,+ )++[email protected]("n", [1, 16, 17, 1000, 4096])[email protected]("offset", [0, 1, 4])[email protected]("add", [False, True])+def test_matches_reference_for_every_specialization(n, offset, add):+ base = torch.randn(n + 8, device = "cuda")+ x = base[offset : offset + n]+ y = torch.randn(n, device = "cuda")+ for scale in (1, 3, 16):+ # Reference: Triton's own launch of the same kernel (x * scale + y may be one FMA).+ ref = torch.empty(n, device = "cuda")+ _axpy[(-(-n // 128),)](x, y, ref, n, scale, BLOCK = 128, ADD = add, num_warps = 4)+ for _ in range(3): # first call compiles, the others take the cached kernel+ out = _run(x, y, torch.empty(n, device = "cuda"), n, scale, add)+ torch.testing.assert_close(out, ref, rtol = 0, atol = 0)+++def test_second_call_uses_the_cache_and_distinct_specializations_get_their_own_entry():+ if not triton_launch._ENABLED:+ pytest.skip("fast launch is CUDA only")+ x = torch.randn(64, device = "cuda")+ compiled = _axpy[(1,)](x, x, torch.empty(64, device = "cuda"), 64, 2, BLOCK = 128, ADD = False)+ if not hasattr(compiled, "packed_metadata"):+ pytest.skip("this Triton returns no packed_metadata, so every launch takes its own path")+ _run(x, x, torch.empty(64, device = "cuda"), 64, 2, False)+ assert len(triton_launch._CACHE) == 1+ _run(x, x, torch.empty(64, device = "cuda"), 64, 2, False)+ assert len(triton_launch._CACHE) == 1+ _run(x[1:], x[1:], torch.empty(63, device = "cuda"), 63, 1, False)+ assert len(triton_launch._CACHE) == 2+++def test_runs_on_the_current_stream():+ x = torch.randn(1 << 20, device = "cuda")+ side = torch.cuda.Stream()+ for _ in range(2):+ out = torch.zeros_like(x)+ with torch.cuda.stream(side):+ torch.cuda._sleep(1_000_000)+ _run(x, x, out, x.numel(), 2, False)+ side.synchronize()+ torch.testing.assert_close(out, x * 2)+++def test_registered_hook_keeps_tritons_own_launch(monkeypatch):+ knobs = getattr(triton, "knobs", None)+ if knobs is None or not hasattr(knobs.runtime.launch_enter_hook, "add"):+ pytest.skip("this Triton has no HookChain launch hooks")+ x = torch.randn(64, device = "cuda")+ _run(x, x, torch.empty(64, device = "cuda"), 64, 2, False)+ seen = []+ hook = lambda metadata: seen.append(1)+ knobs.runtime.launch_enter_hook.add(hook)+ try:+ out = _run(x, x, torch.empty(64, device = "cuda"), 64, 2, False)+ finally:+ knobs.runtime.launch_enter_hook.remove(hook)+ torch.testing.assert_close(out, x * 2)+ assert seen, "a registered launch hook must still be called"+++def test_relaunch_repeats_the_entry_and_yields_to_hooks(monkeypatch):+ x = torch.randn(64, device = "cuda")+ grid = (1,)+ entry = triton_launch.launch(+ _axpy,+ grid,+ (x, x, torch.empty(64, device = "cuda"), 64, 2),+ 3,+ dict(BLOCK = 128, ADD = False),+ x.device.index,+ num_warps = 4,+ )+ if entry is None:+ pytest.skip("no fast launch entry on this build")+ y = torch.randn(64, device = "cuda")+ out = torch.empty(64, device = "cuda")+ assert triton_launch.relaunch(entry, grid, x.device.index, (y, y, out, 64, 2))+ torch.testing.assert_close(out, y * 2)+ # A profiler hook must see every launch, so relaunch hands the call back to Triton's path.+ monkeypatch.setattr(triton_launch, "_hooks_set", lambda: True)+ assert not triton_launch.relaunch(entry, grid, x.device.index, (y, y, out, 64, 2))diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 50e44fc9ca2..36042f4efd0 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -10,9 +10,11 @@# limitations under the License.import torch+from unsloth_zoo.utils import Versionfrom .utils import (_has_multiple_active_adapters,_maybe_fake_quantize_activations,+ addmm_,fast_dequantize,QUANT_STATE,get_lora_parameters,@@ -22,6 +24,19 @@ from .utils import (torch_amp_custom_bwd,)+_is_compiling = torch.compiler.is_compiling++# Inductor before torch 2.11 miscompiles these Functions' traced backward (wrong LoRA gradients).+TRACE_LORA_FUNCTIONS = Version(torch.__version__) >= Version("2.11.0")+++def _apply(function, *args):+ return function.apply(*args)+++if not TRACE_LORA_FUNCTIONS:+ _apply = torch._dynamo.disable(_apply)+class LoRA_MLP(torch.autograd.Function):"""@@ -179,14 +194,15 @@ class LoRA_MLP(torch.autograd.Function):# dX = matmul_lora(df, upW.t(), ...) + matmul_lora(de, gateW.t(), ...), expanded below.upW = fast_dequantize(upW.t(), upW_quant)- dX = torch.matmul(df, upW.t(), out = X if ctx.inplace else None)+ # Eager only: AOT autograd rejects a backward mutating a forward input that requires grad.+ dX = torch.matmul(df, upW.t(), out = X if ctx.inplace and not _is_compiling() else None)del upW- dX.addmm_(up_dB, upA.t(), alpha = upS)+ addmm_(dX, up_dB, upA.t(), alpha = upS)gateW = fast_dequantize(gateW.t(), gateW_quant)- dX.addmm_(de, gateW.t())+ addmm_(dX, de, gateW.t())del gateW- dX.addmm_(gate_dB, gateA.t(), alpha = gateS)+ addmm_(dX, gate_dB, gateA.t(), alpha = gateS)return (dX.view(batch, seq_len, hd),@@ -228,7 +244,8 @@ def apply_lora_mlp_swiglu(gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)- out = LoRA_MLP.apply(+ out = _apply(+ LoRA_MLP,X,gateW,gateW_quant,@@ -269,7 +286,8 @@ def apply_lora_mlp_geglu_exact(gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)- out = LoRA_MLP.apply(+ out = _apply(+ LoRA_MLP,X,gateW,gateW_quant,@@ -306,7 +324,8 @@ def apply_lora_mlp_geglu_approx(self, X):gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)- out = LoRA_MLP.apply(+ out = _apply(+ LoRA_MLP,X,gateW,gateW_quant,@@ -480,19 +499,19 @@ class LoRA_QKV(torch.autograd.Function):# Combine the per-projection derivatives into dX.QW = fast_dequantize(QW.t(), QW_quant)- dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace else None)+ dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace and not _is_compiling() else None)del QW- dX.addmm_(q_dB, QA.t(), alpha = QS)+ addmm_(dX, q_dB, QA.t(), alpha = QS)KW = fast_dequantize(KW.t(), KW_quant)- dX.addmm_(dK, KW.t())+ addmm_(dX, dK, KW.t())del KW- dX.addmm_(k_dB, KA.t(), alpha = KS)+ addmm_(dX, k_dB, KA.t(), alpha = KS)VW = fast_dequantize(VW.t(), VW_quant)- dX.addmm_(dV, VW.t())+ addmm_(dX, dV, VW.t())del VW- dX.addmm_(v_dB, VA.t(), alpha = VS)+ addmm_(dX, v_dB, VA.t(), alpha = VS)return (dX.view(batch, seq_len, hd),@@ -526,7 +545,8 @@ def apply_lora_qkv(QW, QW_quant, QA, QB, QS = get_lora_parameters(self.q_proj)KW, KW_quant, KA, KB, KS = get_lora_parameters(self.k_proj)VW, VW_quant, VA, VB, VS = get_lora_parameters(self.v_proj)- Q, K, V = LoRA_QKV.apply(+ Q, K, V = _apply(+ LoRA_QKV,X,QW,QW_quant,@@ -617,7 +637,7 @@ class LoRA_W(torch.autograd.Function):W = fast_dequantize(W.t(), W_quant)dX = dY @ W.t()del W- dX.addmm_(y_dB, A.t(), alpha = S)+ addmm_(dX, y_dB, A.t(), alpha = S)return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None@@ -627,7 +647,7 @@ def apply_lora_o(self, X):return self.o_proj(X)X = _maybe_fake_quantize_activations(X, self.o_proj)OW, OW_quant, OA, OB, OS = get_lora_parameters(self.o_proj)- O = LoRA_W.apply(X, OW, OW_quant, OA, OB, OS)+ O = _apply(LoRA_W, X, OW, OW_quant, OA, OB, OS)return O@@ -661,7 +681,7 @@ def fast_lora_forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor:lora_B = self.lora_B[active_adapter].weightscaling = self.scaling[active_adapter]W = self.base_layer.weight- return LoRA_W.apply(x, W, QUANT_STATE(W), lora_A, lora_B, scaling)+ return _apply(LoRA_W, x, W, QUANT_STATE(W), lora_A, lora_B, scaling)passpassdiff --git a/unsloth/kernels/nf4.py b/unsloth/kernels/nf4.pynew file mode 100644index 00000000000..b2d75ea1a06--- /dev/null+++ b/unsloth/kernels/nf4.py@@ -0,0 +1,449 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++# NF4 dequantization in one Triton launch on torch's current stream, bit-exact with bitsandbytes+# kDequantizeBlockwise, which needs three launches for a double quantized weight:+# absmax = fp32(code2[absmax_u8[k]] * absmax2[k // blocksize2]) + offset (two roundings)+# out[j] = T(code[nibble(j)] * absmax[j // blocksize]) (high nibble first)++import contextlib+import functools+import math+import os+from typing import List, Optional++import torch+import triton+import triton.language as tl+from triton.language.extra import libdevice+from unsloth_zoo.utils import Version++from .triton_launch import launch, relaunch, tag_compile_cache++__all__ = [+ "dequantize_nf4",+ "dequantize_nf4_planned",+]++# libdevice mul_rn is never contracted into an FMA (Inductor re-emits kernels with fp fusion on) and+# flushes subnormals like bitsandbytes. HIP / TRITON_INTERPRET=1 lack it: fusion off, opaque op.+_HAS_MUL_RN = torch.version.hip is None and os.environ.get("TRITON_INTERPRET", "0") != "1"+++def _mul(a, b, USE_MUL_RN: tl.constexpr):+ if USE_MUL_RN:+ return libdevice.mul_rn(a, b)+ else:+ return a * b+++# Triton's JIT resolves every attribute in a kernel body, even in a dead constexpr branch, so+# tl.gather (absent on Triton 3.2, torch 2.6) lives in a helper defined only where it exists.+if hasattr(tl, "gather"):++ @triton.jit+ def _lut_gather(lut_ptr, n):+ table = tl.load(lut_ptr + tl.arange(0, 16))+ flat = tl.reshape(n, [n.numel])+ return tl.reshape(tl.gather(table, flat, 0), n.shape)++else:++ @triton.jit+ def _lut_gather(lut_ptr, n):+ return tl.load(lut_ptr + n, eviction_policy = "evict_last")+++def _nf4_lut(lut_ptr, n, LUT_MODE: tl.constexpr):+ # n: int32 nibbles. Every mode returns the same fp32 table entry, so the choice is speed only.+ if LUT_MODE == 0:+ # A 64-byte table gather stays in L1.+ return tl.load(lut_ptr + n, eviction_policy = "evict_last")+ elif LUT_MODE == 1:+ # Select tree on the nibble bits over table values held as scalars.+ b0 = (n & 1) != 0+ b1 = (n & 2) != 0+ b2 = (n & 4) != 0+ b3 = (n & 8) != 0+ a0 = tl.where(b0, tl.load(lut_ptr + 1), tl.load(lut_ptr + 0))+ a1 = tl.where(b0, tl.load(lut_ptr + 3), tl.load(lut_ptr + 2))+ a2 = tl.where(b0, tl.load(lut_ptr + 5), tl.load(lut_ptr + 4))+ a3 = tl.where(b0, tl.load(lut_ptr + 7), tl.load(lut_ptr + 6))+ a4 = tl.where(b0, tl.load(lut_ptr + 9), tl.load(lut_ptr + 8))+ a5 = tl.where(b0, tl.load(lut_ptr + 11), tl.load(lut_ptr + 10))+ a6 = tl.where(b0, tl.load(lut_ptr + 13), tl.load(lut_ptr + 12))+ a7 = tl.where(b0, tl.load(lut_ptr + 15), tl.load(lut_ptr + 14))+ c0 = tl.where(b1, a1, a0)+ c1 = tl.where(b1, a3, a2)+ c2 = tl.where(b1, a5, a4)+ c3 = tl.where(b1, a7, a6)+ d0 = tl.where(b2, c1, c0)+ d1 = tl.where(b2, c3, c2)+ return tl.where(b3, d1, d0)+ else:+ # Register-resident table, gathered along its only axis.+ return _lut_gather(lut_ptr, n)+++def _nf4_dequant_kernel(+ W_ptr, # uint8, packed: high nibble is the even element+ absmax_ptr, # uint8 (nested) or fp32, one per absmax block+ code2_ptr, # fp32 [256], nested only+ absmax2_ptr, # fp32, nested only+ offset_ptr, # fp32 [1], nested only+ lut_ptr, # fp32 [16], the NF4 codebook+ out_ptr,+ n_elements,+ n_bytes,+ n_blocks,+ HALF: tl.constexpr, # packed bytes per absmax block (blocksize // 2)+ BLOCKSIZE2_SHIFT: tl.constexpr,+ NESTED: tl.constexpr,+ USE_MUL_RN: tl.constexpr,+ ROWS: tl.constexpr, # absmax blocks per program+ WORDS: tl.constexpr, # load the packed bytes as int32 words (needs n_bytes % 4 == 0)+ EVICT: tl.constexpr, # "evict_first" streams the weight and the output past L2, else ""+ LUT_MODE: tl.constexpr, # see _nf4_lut+):+ # A [ROWS, HALF] tile of packed bytes: each row is one absmax block, so its scale is+ # decoded once per row and broadcast, instead of once per byte.+ # FSDP-QLoRA packs the 4bit weight into float storage; it is bytes either way.+ W_ptr = W_ptr.to(tl.pointer_type(tl.uint8))+ pid = tl.program_id(0).to(tl.int64)+ rows = pid * ROWS + tl.arange(0, ROWS)+ row_mask = rows < n_blocks+ if NESTED:+ a_q = tl.load(absmax_ptr + rows, mask = row_mask, other = 0).to(tl.int32)+ c2 = tl.load(code2_ptr + a_q, mask = row_mask, other = 0.0, eviction_policy = "evict_last")+ s2 = tl.load(+ absmax2_ptr + (rows >> BLOCKSIZE2_SHIFT),+ mask = row_mask,+ other = 0.0,+ eviction_policy = "evict_last",+ )+ scale = _mul(c2, s2, USE_MUL_RN)+ scale = scale + tl.load(offset_ptr)+ else:+ scale = tl.load(absmax_ptr + rows, mask = row_mask, other = 0.0)+ scale = scale[:, None]+ out_ty = out_ptr.dtype.element_ty++ if WORDS:+ # [ROWS, HALF // 4] int32 words; byte j of a word holds elements 2j (high) and 2j + 1.+ NW: tl.constexpr = HALF // 4+ w_offs = rows[:, None] * NW + tl.arange(0, NW)[None, :]+ w = tl.load(+ W_ptr.to(tl.pointer_type(tl.int32)) + w_offs,+ mask = w_offs < (n_bytes // 4),+ other = 0,+ eviction_policy = EVICT,+ )+ i = tl.arange(0, 8)+ shifts = (i // 2) * 8 + (1 - (i % 2)) * 4+ nib = (w[:, :, None] >> shifts[None, None, :]) & 15+ vals = _mul(+ _nf4_lut(lut_ptr, tl.reshape(nib, [ROWS, 2 * HALF]), LUT_MODE), scale, USE_MUL_RN+ )+ vals = vals.to(out_ty)+ else:+ cols = tl.arange(0, HALF)+ byte_offs = rows[:, None] * HALF + cols[None, :]+ q = tl.load(W_ptr + byte_offs, mask = byte_offs < n_bytes, other = 0, eviction_policy = EVICT)+ v_hi = _mul(_nf4_lut(lut_ptr, (q >> 4).to(tl.int32), LUT_MODE), scale, USE_MUL_RN).to(+ out_ty+ )+ v_lo = _mul(_nf4_lut(lut_ptr, (q & 15).to(tl.int32), LUT_MODE), scale, USE_MUL_RN).to(+ out_ty+ )+ vals = tl.interleave(v_hi, v_lo) # [ROWS, 2 * HALF], high nibble first++ out_offs = rows[:, None] * (2 * HALF) + tl.arange(0, 2 * HALF)[None, :]+ tl.store(out_ptr + out_offs, vals, mask = out_offs < n_elements, eviction_policy = EVICT)+++# Per compute capability (prefix match, () default) launch knobs: speed only, all bit-exact.+_CONFIGS = {+ (): (+ # (max n_bytes, target bytes per program, num_warps, words, evict, lut_mode)+ (1 << 16, 256, 2, False, False, 0),+ (1 << 20, 1024, 2, False, False, 0),+ (None, 2048, 4, False, False, 0),+ ),+ # Blackwell datacenter (B200): int32 loads + register table gather.+ (10,): (+ (1 << 16, 256, 2, True, True, 2),+ (None, 1024, 2, True, True, 2),+ ),+}+if Version(triton.__version__) >= Version("3.7.0"):+ # T4: 5% to 7% faster on Triton 3.7, not on 3.6.+ _CONFIGS[(7, 5)] = ((None, 512, 8, True, True, 0),)+# tl.gather on a register table fails to compile on Triton 3.3; older Triton uses the L1 load.+_HAS_TL_GATHER = Version(triton.__version__) >= Version("3.6.0")+# Tests and the sweep script set this to a (target, num_warps, words, evict, lut_mode) tuple.+_CONFIG_OVERRIDE = None++[email protected]_cache(maxsize = None)+def _capability(index: int):+ return torch.cuda.get_device_capability(index)+++def _config_for(n_bytes: int, half: int, device: torch.device):+ if _CONFIG_OVERRIDE is not None:+ target, num_warps, words, evict, lut_mode = _CONFIG_OVERRIDE+ else:+ cap = _capability(device.index if device.index is not None else torch.cuda.current_device())+ table = _CONFIGS.get(cap) or _CONFIGS.get(cap[:1]) or _CONFIGS[()]+ for limit, target, num_warps, words, evict, lut_mode in table:+ if limit is None or n_bytes <= limit:+ break+ if lut_mode == 2 and not _HAS_TL_GATHER:+ lut_mode = 0+ return max(1, target // half), num_warps, words, evict, lut_mode+++def _is_pow2(x: int) -> bool:+ return x > 0 and (x & (x - 1)) == 0+++def _launch(+ kernel,+ W,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ blocksize,+ blocksize2,+ out,+ fp_fusion,+ plan = False,+):+ nested = code2 is not None+ n_elements = out.numel()+ n_bytes = (n_elements + 1) // 2+ half = blocksize // 2+ # Integer ceil division: triton.cdiv is a JIT function on recent Triton and costs microseconds.+ n_blocks = -(-n_elements // blocksize)+ rows, num_warps, words, evict, lut_mode = _config_for(n_bytes, half, W.device)+ # int32 loads need every row to start on a word and no partial trailing word.+ words = (+ words+ and half % 4 == 0+ and n_bytes % 4 == 0+ and (W.storage_offset() * W.element_size()) % 4 == 0+ )+ # The nested-only pointers are never dereferenced for a flat state; absmax fills the slots.+ args = (+ W,+ absmax,+ code2 if nested else absmax,+ absmax2 if nested else absmax,+ offset if nested else absmax,+ code,+ out,+ n_elements,+ n_bytes,+ n_blocks,+ )+ constexprs = dict(+ HALF = half,+ BLOCKSIZE2_SHIFT = (blocksize2.bit_length() - 1) if nested else 0,+ NESTED = nested,+ USE_MUL_RN = _HAS_MUL_RN,+ ROWS = rows,+ WORDS = words,+ EVICT = "evict_first" if evict else "",+ LUT_MODE = lut_mode,+ )+ grid = (-(-n_blocks // rows),)+ options = (+ {"num_warps": num_warps}+ if fp_fusion+ else {"num_warps": num_warps, "enable_fp_fusion": False}+ )+ if kernel is _nf4_dequant_kernel:+ entry = launch(kernel, grid, args, 7, constexprs, W.device.index, **options)+ if plan:+ return _rerun(entry, grid, args, W.device.index)+ else:+ kernel[grid](*args, **constexprs, **options)+ return out+++def _rerun(entry, grid, args, device_index):+ if entry is None:+ return None+ before, after = args[:6], args[7:]++ def rerun(out):+ # out must match the first output in size, dtype and 16 byte alignment.+ return relaunch(entry, grid, device_index, (*before, out, *after))++ return rerun+++if _HAS_MUL_RN:+ # Inductor sees the Triton kernel itself and schedules it with the surrounding graph.+ _register = torch.library.triton_op+ _traced_kernel = lambda: torch.library.wrap_triton(_nf4_dequant_kernel)+ _traced_guard = lambda device: contextlib.nullcontext()+else:+ # HIP: an opaque custom op keeps the eager launch (fusion off) and needs its own device guard.+ _register = torch.library.custom_op+ _traced_kernel = lambda: _nf4_dequant_kernel+ _traced_guard = torch.cuda.device+++@_register("unsloth::dequantize_nf4", mutates_args = ())+def _dequantize_nf4_op(+ W: torch.Tensor,+ absmax: torch.Tensor,+ code2: Optional[torch.Tensor],+ absmax2: Optional[torch.Tensor],+ offset: Optional[torch.Tensor],+ code: torch.Tensor,+ blocksize: int,+ blocksize2: int,+ shape: List[int],+ dtype: torch.dtype,+) -> torch.Tensor:+ out = torch.empty(shape, dtype = dtype, device = W.device)+ with _traced_guard(W.device):+ return _launch(+ _traced_kernel(),+ W,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ blocksize,+ blocksize2,+ out,+ fp_fusion = _HAS_MUL_RN,+ )+++@_register("unsloth::dequantize_nf4_out", mutates_args = ["out"])+def _dequantize_nf4_out_op(+ W: torch.Tensor,+ absmax: torch.Tensor,+ code2: Optional[torch.Tensor],+ absmax2: Optional[torch.Tensor],+ offset: Optional[torch.Tensor],+ code: torch.Tensor,+ blocksize: int,+ blocksize2: int,+ out: torch.Tensor,+) -> None:+ with _traced_guard(W.device):+ _launch(+ _traced_kernel(),+ W,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ blocksize,+ blocksize2,+ out,+ fp_fusion = _HAS_MUL_RN,+ )+++if not _HAS_MUL_RN:++ @_dequantize_nf4_op.register_fake+ def _dequantize_nf4_fake(+ W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape, dtype+ ):+ return W.new_empty(shape, dtype = dtype)++ @_dequantize_nf4_out_op.register_fake+ def _dequantize_nf4_out_fake(+ W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, out+ ):+ return None+++tag_compile_cache(__file__)+++def dequantize_nf4(+ W: torch.Tensor,+ absmax: torch.Tensor,+ code2: Optional[torch.Tensor],+ absmax2: Optional[torch.Tensor],+ offset: Optional[torch.Tensor],+ code: torch.Tensor,+ blocksize: int,+ blocksize2: int,+ shape,+ dtype: torch.dtype,+ out: Optional[torch.Tensor] = None,+) -> torch.Tensor:+ """Dequantize a bitsandbytes NF4 weight into ``out`` (allocated when None).++ ``code`` is the quant state's fp32 [16] NF4 codebook. ``code2``, ``absmax2`` and the device+ tensor ``offset`` are None when the absmax is not double quantized (it is then fp32).+ """+ if dtype not in (torch.float16, torch.bfloat16, torch.float32):+ raise TypeError(f"Unsloth: dequantize_nf4 does not support dtype {dtype}")+ if not _is_pow2(blocksize) or blocksize < 2 or (code2 is not None and not _is_pow2(blocksize2)):+ raise ValueError(f"Unsloth: dequantize_nf4 needs power-of-two blocksizes, got {blocksize}")+ if out is not None and (out.dtype != dtype or out.numel() != math.prod(shape)):+ raise ValueError("Unsloth: dequantize_nf4 out has the wrong dtype or size")+ if torch.compiler.is_compiling():+ if out is None:+ return torch.ops.unsloth.dequantize_nf4(+ W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, list(shape), dtype+ )+ torch.ops.unsloth.dequantize_nf4_out(+ W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, out+ )+ return out+ if out is None:+ out = torch.empty(shape, dtype = dtype, device = W.device)+ # Eager launches skip the op dispatcher, which costs about as much as the kernel.+ same_device = W.device.index == torch.cuda.current_device()+ with contextlib.nullcontext() if same_device else torch.cuda.device(W.device):+ return _launch(+ _nf4_dequant_kernel,+ W,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ blocksize,+ blocksize2,+ out,+ fp_fusion = False,+ )+++def dequantize_nf4_planned(W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, out):+ """Eager dequantize_nf4 into out on the current device, returning a function that repeats+ the launch into another buffer like out for this same weight, skipping the per-call argument+ handling (decode dequantizes every weight once per token). None when that is unavailable."""+ return _launch(+ _nf4_dequant_kernel,+ W,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ blocksize,+ blocksize2,+ out,+ fp_fusion = False,+ plan = True,+ )diff --git a/unsloth/kernels/nf4_gemv.py b/unsloth/kernels/nf4_gemv.pynew file mode 100644index 00000000000..f75694e1f3b--- /dev/null+++ b/unsloth/kernels/nf4_gemv.py@@ -0,0 +1,390 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++# Batch-1 NF4 GEMV for decoding, out[1, 1, N] = X[1, 1, K] @ dequant(W).T, in one Triton launch+# (nested absmax decode, NF4 lookup and dot product fused) on torch's current stream. Registered+# as unsloth::gemv_nf4 so torch.compile traces it and CUDA graphs capture it. Each quantization+# block is summed before scaling, so it matches bitsandbytes to fp32 accumulation noise, not bitwise.+# Every weight row must start a new quantization block (K % blocksize == 0); callers check.++import functools+from typing import Optional++import torch+import triton+import triton.language as tl++from .nf4 import _HAS_MUL_RN, _mul+from .triton_launch import launch, relaunch, tag_compile_cache++__all__ = [+ "gemv_nf4",+ "gemv_nf4_planned",+ "triton_gemv_eager",+]+++def _gemv_nf4_kernel(+ X,+ W,+ ABSMAX,+ CODE2,+ ABSMAX2,+ OFFSET,+ LUT,+ OUT,+ N,+ K,+ BLOCKSIZE: tl.constexpr,+ BLOCKSIZE2: tl.constexpr,+ NESTED: tl.constexpr,+ USE_MUL_RN: tl.constexpr,+ BLOCK_N: tl.constexpr,+ BLOCK_K: tl.constexpr,+):+ # FSDP-QLoRA packs the 4bit weight into float storage; it is bytes either way.+ W = W.to(tl.pointer_type(tl.uint8))+ pid = tl.program_id(0)+ rows = pid * BLOCK_N + tl.arange(0, BLOCK_N)+ rmask = rows < N+ rows64 = rows.to(tl.int64)+ HALF_K: tl.constexpr = BLOCK_K // 2+ NB: tl.constexpr = BLOCK_K // BLOCKSIZE+ half_k = tl.arange(0, HALF_K)+ ks = tl.arange(0, BLOCK_K)+ nbs = tl.arange(0, NB)+ blocks_per_row = K // BLOCKSIZE+ acc = tl.zeros([BLOCK_N], dtype = tl.float32)+ if NESTED:+ offset = tl.load(OFFSET)+ for k0 in range(0, K, BLOCK_K):+ kmask = (k0 + 2 * half_k) < K+ m = rmask[:, None] & kmask[None, :]+ b = tl.load(W + rows64[:, None] * (K // 2) + (k0 // 2 + half_k)[None, :], mask = m, other = 0)+ b = b.to(tl.int32)+ # High nibble is the earlier element, as bitsandbytes packs it.+ hi = tl.load(LUT + (b >> 4))+ lo = tl.load(LUT + (b & 15))+ w = tl.reshape(tl.join(hi, lo), (BLOCK_N, BLOCK_K))+ xk = k0 + ks+ x = tl.load(X + xk, mask = xk < K, other = 0.0).to(tl.float32)+ part = tl.sum(tl.reshape(w * x[None, :], (BLOCK_N, NB, BLOCKSIZE)), axis = 2)+ bmask = rmask[:, None] & ((k0 + nbs * BLOCKSIZE) < K)[None, :]+ blk = rows64[:, None] * blocks_per_row + (k0 // BLOCKSIZE + nbs)[None, :]+ if NESTED:+ q = tl.load(ABSMAX + blk, mask = bmask, other = 0).to(tl.int32)+ c2 = tl.load(CODE2 + q, mask = bmask, other = 0.0)+ s2 = tl.load(ABSMAX2 + blk // BLOCKSIZE2, mask = bmask, other = 0.0)+ # Two separate fp32 roundings as in fast_dequantize, even with fp fusion on.+ a = _mul(c2, s2, USE_MUL_RN)+ a = a + offset+ else:+ a = tl.load(ABSMAX + blk, mask = bmask, other = 0.0)+ acc += tl.sum(part * a, axis = 1)+ tl.store(OUT + rows, acc.to(OUT.dtype.element_ty), mask = rmask)+++def _gemv_nf4_words_kernel(+ X,+ W,+ ABSMAX,+ CODE2,+ ABSMAX2,+ OFFSET,+ LUT,+ OUT,+ N,+ K,+ BLOCKSIZE: tl.constexpr,+ BLOCKSIZE2: tl.constexpr,+ NESTED: tl.constexpr,+ USE_MUL_RN: tl.constexpr,+ BLOCK_N: tl.constexpr,+ BLOCK_K: tl.constexpr,+):+ # _gemv_nf4_kernel reading int32 words of 8 nibbles; faster before Triton 3.7, slower on 3.7.+ W = W.to(tl.pointer_type(tl.int32))+ pid = tl.program_id(0)+ rows = pid * BLOCK_N + tl.arange(0, BLOCK_N)+ rmask = rows < N+ rows64 = rows.to(tl.int64)+ WORDS: tl.constexpr = BLOCK_K // 8+ NB: tl.constexpr = BLOCK_K // BLOCKSIZE+ wk = tl.arange(0, WORDS)+ j = tl.arange(0, 8)+ # Little endian: byte b of a word holds values 2b (high nibble) and 2b + 1 (low nibble).+ shifts = (j // 2) * 8 + (1 - (j % 2)) * 4+ nbs = tl.arange(0, NB)+ words_per_row = K // 8+ blocks_per_row = K // BLOCKSIZE+ acc = tl.zeros([BLOCK_N], dtype = tl.float32)+ if NESTED:+ offset = tl.load(OFFSET)+ for k0 in range(0, K, BLOCK_K):+ kw = k0 // 8 + wk+ m = rmask[:, None] & (kw < words_per_row)[None, :]+ w = tl.load(W + rows64[:, None] * words_per_row + kw[None, :], mask = m, other = 0)+ v = tl.load(LUT + ((w[:, :, None] >> shifts[None, None, :]) & 15))+ xk = k0 + wk[:, None] * 8 + j[None, :]+ x = tl.load(X + xk, mask = xk < K, other = 0.0).to(tl.float32)+ part = tl.sum(tl.reshape(v * x[None, :, :], (BLOCK_N, NB, BLOCKSIZE)), axis = 2)+ bmask = rmask[:, None] & ((k0 + nbs * BLOCKSIZE) < K)[None, :]+ blk = rows64[:, None] * blocks_per_row + (k0 // BLOCKSIZE + nbs)[None, :]+ if NESTED:+ q = tl.load(ABSMAX + blk, mask = bmask, other = 0).to(tl.int32)+ c2 = tl.load(CODE2 + q, mask = bmask, other = 0.0)+ s2 = tl.load(ABSMAX2 + blk // BLOCKSIZE2, mask = bmask, other = 0.0)+ a = _mul(c2, s2, USE_MUL_RN)+ a = a + offset+ else:+ a = tl.load(ABSMAX + blk, mask = bmask, other = 0.0)+ acc += tl.sum(part * a, axis = 1)+ tl.store(OUT + rows, acc.to(OUT.dtype.element_ty), mask = rmask)+++_TRITON_37 = tuple(int(x) for x in triton.__version__.split(".")[:2]) >= (3, 7)+# None picks per GPU and Triton version below; "bytes" or "words" forces one kernel (tests, sweeps).+_FORCE_KERNEL = None+++def triton_gemv_eager(device = None):+ """Whether eager (uncompiled) decode should take the Triton GEMV. On CUDA it does: with the+ direct launch (triton_launch.py) it beat bitsandbytes' GEMV in batch-1 decode on T4, L4, A100,+ RTX PRO 6000 and B200. HIP launches through Triton's own path, which only paid off from+ Triton 3.7."""+ return torch.version.hip is None or _TRITON_37+++def _word_aligned(W):+ # Rows are K / 2 bytes (K % blocksize == 0), so only a view can break 4 byte alignment.+ return (W.storage_offset() * W.element_size()) % 4 == 0+++_NARROW_CAPS = ((7, 5), (8, 9))++[email protected]_cache(maxsize = None)+def _gemv_config(N: int, K: int, blocksize: int, capability, words_ok: bool, force):+ """(use the words kernel, grid, BLOCK_N, BLOCK_K, num_warps).++ Kernel time (CUDA graphs) over Llama 1B to 70B shapes: before Triton 3.7 the words kernel at+ one row per program and one warp beat bitsandbytes on T4, L4, A100, RTX PRO 6000 and B200+ (1.1x to 2.2x, lower error too); Triton 3.7 compiles the byte kernel better, best at 4 rows, BLOCK_K 2048 and+ 4 warps (measured on B200)."""+ block_k = max(blocksize, min(2048, triton.next_power_of_2(K)))+ use_words = words_ok and (force == "words" or (force is None and not _TRITON_37))+ if use_words:+ # One row per program and BLOCK_K 2048 was the fastest config at every Llama 1B / 8B shape on+ # T4, L4 and A100 (interleaved CUDA graph timing; T4 throttles, so sweeps must interleave).+ return True, (N,), 1, block_k, 1+ if force is None:+ if capability in _NARROW_CAPS and N * K > 1 << 25 and K >= 4096:+ # T4 / L4: 3% to 9% faster on 8B MLP shapes; slower on A100 and B200.+ return False, (-(-N // 4),), 4, max(blocksize, 1024), 2+ return False, (-(-N // 4),), 4, block_k, 4+ # Forced byte kernel (tests): the pre-3.7 byte config.+ block_k = max(blocksize, min(1024, triton.next_power_of_2(K)))+ block_n, num_warps = (2, 4) if N <= 2048 else (4, 2)+ return False, (-(-N // block_n),), block_n, block_k, num_warps++[email protected]_cache(maxsize = None)+def _capability(index):+ return torch.cuda.get_device_capability(index)+++def _launch(+ kernels,+ X,+ W,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ out,+ N,+ K,+ blocksize,+ blocksize2,+ plan = False,+):+ nested = code2 is not None+ use_words, grid, block_n, block_k, num_warps = _gemv_config(+ N, K, blocksize, _capability(X.device.index), _word_aligned(W), _FORCE_KERNEL+ )+ kernel = kernels[1] if use_words else kernels[0]+ args = (+ X,+ W,+ absmax,+ code2 if nested else absmax,+ absmax2 if nested else absmax,+ offset if nested else absmax,+ code,+ out,+ N,+ K,+ )+ constexprs = dict(+ BLOCKSIZE = blocksize,+ BLOCKSIZE2 = blocksize2 if nested else 1,+ NESTED = nested,+ USE_MUL_RN = _HAS_MUL_RN,+ BLOCK_N = block_n,+ BLOCK_K = block_k,+ )+ if kernel is _gemv_nf4_kernel or kernel is _gemv_nf4_words_kernel:+ entry = launch(kernel, grid, args, 8, constexprs, X.device.index, num_warps = num_warps)+ if plan:+ return _rerun(entry, grid, args, X.device.index)+ else:+ kernel[grid](*args, **constexprs, num_warps = num_warps)+ return out+++def _rerun(entry, grid, args, device_index):+ if entry is None:+ return None+ weights, sizes = args[1:7], args[8:]++ def rerun(X, out):+ # X and out must match the first launch's in dtype and 16 byte alignment.+ return relaunch(entry, grid, device_index, (X, *weights, out, *sizes))++ return rerun++[email protected]_op("unsloth::gemv_nf4", mutates_args = ())+def _gemv_nf4_op(+ X: torch.Tensor,+ W: torch.Tensor,+ absmax: torch.Tensor,+ code2: Optional[torch.Tensor],+ absmax2: Optional[torch.Tensor],+ offset: Optional[torch.Tensor],+ code: torch.Tensor,+ blocksize: int,+ blocksize2: int,+ out_features: int,+ in_features: int,+) -> torch.Tensor:+ out = torch.empty((1, 1, out_features), dtype = X.dtype, device = X.device)+ return _launch(+ (+ torch.library.wrap_triton(_gemv_nf4_kernel),+ torch.library.wrap_triton(_gemv_nf4_words_kernel),+ ),+ X,+ W,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ out,+ out_features,+ in_features,+ blocksize,+ blocksize2,+ )+++_KERNELS = (_gemv_nf4_kernel, _gemv_nf4_words_kernel)+tag_compile_cache(__file__)+_is_compiling = torch.compiler.is_compiling+_current_device = torch.cuda.current_device+++def gemv_nf4(+ X,+ W_u8,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ blocksize,+ blocksize2,+ shape,+ dtype,+ out = None,+):+ """X @ dequant(W).T for a single token; returns (1, 1, shape[0]) in X's dtype.++ ``code2/absmax2/offset`` are None for a quant state without double quantization, and+ ``offset`` is bitsandbytes' 0-dim device tensor (read in the kernel, so no host sync).+ Eager calls launch the kernel directly: the torch.library dispatcher costs about 20us+ per call, as much as the kernel itself at decode sizes. Compiled code goes through+ the registered op so Dynamo sees it.+ """+ out_features, in_features = int(shape[0]), int(shape[1])+ if not X.is_contiguous():+ X = X.contiguous()+ blocksize2 = int(blocksize2) if code2 is not None else 0+ if _is_compiling():+ result = _gemv_nf4_op(+ X,+ W_u8,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ int(blocksize),+ blocksize2,+ out_features,+ in_features,+ )+ if out is not None:+ out.copy_(result)+ return out+ return result+ if out is None:+ out = torch.empty((1, 1, out_features), dtype = X.dtype, device = X.device)+ args = (+ _KERNELS,+ X,+ W_u8,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ out,+ out_features,+ in_features,+ int(blocksize),+ blocksize2,+ )+ # Triton launches on the current device's current stream, so only switch when X lives elsewhere.+ if X.device.index == _current_device():+ return _launch(*args)+ with torch.cuda.device(X.device):+ return _launch(*args)+++def gemv_nf4_planned(+ X, W_u8, absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape, out+):+ """Eager gemv_nf4 into out on the current device, returning a function (X, out) that repeats+ the launch for this same weight, skipping the per-call argument handling (decode runs every+ weight once per token). None when that is unavailable."""+ return _launch(+ _KERNELS,+ X,+ W_u8,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ out,+ int(shape[0]),+ int(shape[1]),+ int(blocksize),+ int(blocksize2) if code2 is not None else 0,+ plan = True,+ )diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.pyindex cfc721d3618..2078f3266e0 100644--- a/unsloth/kernels/swiglu.py+++ b/unsloth/kernels/swiglu.py@@ -103,7 +103,21 @@ def _DWf_DW_dfg_kernel(DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEtl.store(g + offsets, de_row, mask = mask)+def _DWf_DW_dfg_traced(DW, e, g):+ # Traced: Inductor fuses this in place; the in-place kernel would make it copy e and g first.+ e_float = e.float()+ se = torch.sigmoid(e_float)+ f = (se * e_float).to(DW.dtype)+ h = f * g+ df = DW * f+ dg = DW * g+ de = (dg.float() * se * (1.0 + e_float * (1.0 - se))).to(DW.dtype)+ return h, df, de++def swiglu_DWf_DW_dfg_kernel(DW, e, g):+ if torch.compiler.is_compiling():+ return _DWf_DW_dfg_traced(DW, e, g)batch_seq_len, hd = e.shape # Flattened to 2D, so 1st dim is bsz * seq_lenn_elements = e.numel()grid = lambda meta: (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),)diff --git a/unsloth/kernels/triton_launch.py b/unsloth/kernels/triton_launch.pynew file mode 100644index 00000000000..68225724162--- /dev/null+++ b/unsloth/kernels/triton_launch.py@@ -0,0 +1,129 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++# Low overhead eager launches: the first call per specialization goes through Triton, later ones+# launch the compiled kernel directly, skipping JITFunction's per-call argument binding. The key+# covers what Triton specializes on: dtypes, 16 byte alignment, ints == 1 / % 16 / > int32, options.++import hashlib+import os++import torch+import triton++__all__ = [+ "launch",+ "relaunch",+ "tag_compile_cache",+]++# CPU-only torch builds have no CUDA stream binding.+_raw_stream = getattr(torch._C, "_cuda_getCurrentRawStream", None)+_ENABLED = (+ _raw_stream is not None+ and torch.version.hip is None+ and os.environ.get("UNSLOTH_TRITON_FAST_LAUNCH", "1") != "0"+)+_CACHE = {}+_knobs = getattr(triton, "knobs", None)+++def _active(hook):+ # Triton 3.5+ always installs a HookChain, empty unless something registered a hook.+ return hook is not None and getattr(hook, "calls", True) != []+++def _hooks_set():+ # Profilers (e.g. proton) hook Triton launches; those launches keep Triton's own path.+ if _knobs is not None:+ runtime = _knobs.runtime+ return _active(runtime.launch_enter_hook) or _active(runtime.launch_exit_hook)+ from triton.compiler import CompiledKernel+ return _active(getattr(CompiledKernel, "launch_enter_hook", None)) or _active(+ getattr(CompiledKernel, "launch_exit_hook", None)+ )+++def relaunch(entry, grid, device_index, args):+ """Repeat a launch whose cache entry launch() returned, with args matching the original in+ dtypes, 16 byte alignment and integer specialization, on device_index (the current device).+ False when the caller must use launch() instead."""+ if not _ENABLED or _hooks_set():+ return False+ compiled, tail = entry+ compiled.run(+ grid[0],+ grid[1] if len(grid) > 1 else 1,+ grid[2] if len(grid) > 2 else 1,+ _raw_stream(device_index),+ compiled.function,+ compiled.packed_metadata,+ None,+ None,+ None,+ *args,+ *tail,+ )+ return True+++def launch(kernel, grid, args, n_tensors, constexprs, device_index, **options):+ """kernel[grid](*args, **constexprs, **options) on the current stream of device_index, which+ must be the current device. args are the runtime arguments in signature order, before every+ constexpr: n_tensors tensors, then integers. Returns the cache entry relaunch() takes, or None."""+ global _ENABLED+ if _ENABLED and not _hooks_set():+ key = (+ kernel,+ device_index,+ tuple(constexprs.values()),+ tuple(options.values()),+ tuple([(arg.dtype, arg.data_ptr() & 15) for arg in args[:n_tensors]]),+ tuple([(arg == 1, arg & 15, arg < (1 << 31)) for arg in args[n_tensors:]]),+ )+ entry = _CACHE.get(key)+ if entry is not None:+ compiled, tail = entry+ try:+ compiled.run(+ grid[0],+ grid[1] if len(grid) > 1 else 1,+ grid[2] if len(grid) > 2 else 1,+ _raw_stream(device_index),+ compiled.function,+ compiled.packed_metadata,+ None,+ None,+ None,+ *args,+ *tail,+ )+ return entry+ except Exception:+ # A Triton build whose launcher takes other arguments: use its own path from now.+ _ENABLED = False+ _CACHE.clear()+ compiled = kernel[grid](*args, **constexprs, **options)+ if _ENABLED and hasattr(compiled, "packed_metadata"):+ names = kernel.arg_names[len(args) :]+ if len(names) == len(constexprs):+ compiled.run # loads the module now, so the fast path never does+ entry = _CACHE[key] = (compiled, tuple(constexprs[name] for name in names))+ return entry+ return None+ kernel[grid](*args, **constexprs, **options)+ return None+++def tag_compile_cache(path):+ """Add a hash of a kernel file to torch.compile's cache key. Inductor's FX graph cache keys a+ torch.library.triton_op call without the Triton source behind it, so after an Unsloth upgrade+ a warm cache would keep serving the previous kernel."""+ config = getattr(torch.compiler, "config", None)+ if config is None or not hasattr(config, "cache_key_tag"):+ return+ with open(path, "rb") as file:+ tag = f"unsloth/{os.path.basename(path)}:{hashlib.sha256(file.read()).hexdigest()[:16]}"+ tags = [t for t in config.cache_key_tag.split(",") if t]+ if tag not in tags:+ config.cache_key_tag = ",".join(tags + [tag])diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex ad0b2568a08..5027392d3c7 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -10,6 +10,7 @@# limitations under the License.import importlib+import osimport tritonimport ctypes@@ -211,6 +212,8 @@ def _get_tensor_stream(tensor: torch_Tensor) -> c_void_p:return c_void_p(_gpu_getCurrentRawStream(tensor.device.index))+# Import-time stream snapshots and the old scratch pair. The kernels now read the live stream, but+# these stay defined for external readers (e.g. tests/kaggle/t4_smoke/run_t4_smoke.py).global CUDA_STREAMSglobal XPU_STREAMSglobal WEIGHT_BUFFERS@@ -497,6 +500,117 @@ def _maybe_fake_quantize_activations(X: torch.Tensor, proj: torch.nn.Module) ->return X+def _unpack_quant_state(quant_state):+ """(absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2) from a bnb+ 4bit quant state, class or legacy list form. The nested fields are None (and blocksize2 0)+ when the absmax is not double quantized."""+ if type(quant_state) is not list:+ # New quant_state as a class, per TimDettmers/bitsandbytes#763.+ absmax = quant_state.absmax+ shape = quant_state.shape+ dtype = quant_state.dtype+ blocksize = quant_state.blocksize+ code = quant_state.code+ offset = quant_state.offset+ state2 = quant_state.state2+ else:+ # Old quant_state as a list of lists+ absmax, shape, dtype, blocksize, compressed_stats, _, code = quant_state+ offset, state2 = compressed_stats if compressed_stats is not None else (None, None)+ if state2 is None:+ return absmax, shape, dtype, blocksize, code, None, None, None, 0+ if type(state2) is list:+ absmax2, code2, blocksize2 = state2[0], state2[1], state2[2]+ else:+ absmax2, code2, blocksize2 = state2.absmax, state2.code, state2.blocksize+ if not isinstance(offset, torch_Tensor):+ offset = torch.tensor(offset, dtype = torch_float32, device = absmax.device)+ return absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2+++def _quant_type(quant_state):+ if type(quant_state) is list:+ return quant_state[5]+ return getattr(quant_state, "quant_type", "nf4")+++# Eager use_global_buffer scratch per (kind, device), default stream only (other streams, compiling+# and graph capture get fresh outputs); allocated outside inference mode so training can write it.+_SCRATCH = {}+_DEFAULT_STREAMS = {}+++def _can_use_scratch(device):+ if torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing():+ return False+ default = _DEFAULT_STREAMS.get(device.index)+ if default is None:+ default = _DEFAULT_STREAMS[device.index] = torch.cuda.default_stream(device).cuda_stream+ return _gpu_getCurrentRawStream(device.index) == default+++def _scratch(kind, device, numel, dtype):+ key = (kind, device.index)+ buffer = _SCRATCH.get(key)+ if buffer is None or buffer.dtype != dtype or buffer.numel() < numel:+ with torch.inference_mode(False):+ buffer = torch_empty(numel, dtype = dtype, device = device, requires_grad = False)+ _SCRATCH[key] = buffer+ return buffer[:numel]+++# Fused NF4 kernels, traceable by torch.compile and on the live stream by construction.+# UNSLOTH_BNB_TRITON=0 keeps the bitsandbytes ctypes path.+_USE_NF4_KERNELS = False+_TRITON_GEMV_EAGER = False+_is_compiling = torch.compiler.is_compiling+_current_device = torch.cuda.current_device+if (+ DEVICE_TYPE in ("cuda", "hip")+ and HAS_CUDA_STREAM+ and os.environ.get("UNSLOTH_BNB_TRITON", "1") != "0"+):+ try:+ import triton+ from .nf4 import dequantize_nf4, dequantize_nf4_planned+ from .nf4_gemv import gemv_nf4, gemv_nf4_planned, triton_gemv_eager++ _USE_NF4_KERNELS = True+ # Eager decode uses the Triton GEMV where it is faster; compiled code always does.+ _TRITON_GEMV_EAGER = triton_gemv_eager()+ except Exception:+ pass+++def _no_plan():+ return None+++class _NF4Plan(list):+ # One weight's decode launch (see fast_dequantize). It holds a closure and the scratch buffer,+ # so copies and pickles of the model drop it and rebuild on first use.+ def __reduce__(self):+ return (_no_plan, ())+++def _nf4_kernels_failed(error):+ """A Triton compile failure on an untested GPU switches to the bitsandbytes kernels for good.+ False (re-raise) while tracing, for Dynamo's own exceptions (a graph break is not a kernel+ failure) and for out-of-memory, which the fallback would not fix."""+ global _USE_NF4_KERNELS+ if (+ torch.compiler.is_compiling()+ or isinstance(error, torch.cuda.OutOfMemoryError)+ or type(error).__module__.startswith("torch._dynamo")+ ):+ return False+ _USE_NF4_KERNELS = False+ print(+ f"Unsloth: fused NF4 kernels failed ({type(error).__name__}: {error}), using bitsandbytes."+ )+ return True++if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:@torch.inference_mode@@ -613,84 +727,36 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:- @torch.inference_mode- def fast_dequantize(+ def _fast_dequantize_ctypes(W,- quant_state = None,+ quant_state,out = None,use_global_buffer = False,):- if isinstance(W, Float8Tensor):- return W.dequantize()- if quant_state is None:- return W- if W.dtype == torch.float8_e4m3fn:- return weight_dequant(W, quant_state)- if type(quant_state) is NVFP4QuantState:- return nvfp4_dequantize(- W, quant_state.scale, quant_state.global_scale, quant_state.dtype- )- if type(quant_state) is not list:- # New quant_state as a class, per TimDettmers/bitsandbytes#763.- absmax = quant_state.absmax- shape = quant_state.shape- dtype = quant_state.dtype- blocksize = quant_state.blocksize- offset = quant_state.offset- state2 = quant_state.state2- absmax2 = state2.absmax- code2 = state2.code- blocksize2 = state2.blocksize- else:- # Old quant_state as a list of lists- absmax, shape, dtype, blocksize, compressed_stats, _, _ = quant_state- offset, state2 = compressed_stats- absmax2, code2, blocksize2, _, _, _, _ = state2- pass- global CUDA_STREAMS+ # bitsandbytes ctypes kernels, used when the NF4 kernels are unavailable or disabled.+ absmax, shape, dtype, blocksize, _, code2, absmax2, offset, blocksize2 = (+ _unpack_quant_state(quant_state)+ )+ if code2 is None or dtype not in (torch_float16, torch_bfloat16):+ return bnb_functional.dequantize_4bit(W, quant_state, out = out)device = W.device- device_index = device.index- CUDA_STREAM = CUDA_STREAMS[device_index]+ # Each launch reads the live stream, so out_absmax += offset stays ordered between them.+ CUDA_STREAM = _get_tensor_stream(W)n_elements_absmax = absmax.numel()-- if use_global_buffer:- size = shape[0] * shape[1]- global WEIGHT_BUFFERS- global ABSMAX_BUFFERS- WEIGHT_BUFFER = WEIGHT_BUFFERS[device_index]- ABSMAX_BUFFER = ABSMAX_BUFFERS[device_index]- if WEIGHT_BUFFER is None or WEIGHT_BUFFER.dtype != dtype:- WEIGHT_BUFFERS[device_index] = WEIGHT_BUFFER = torch_empty(- size, dtype = dtype, device = device, requires_grad = False- )- ABSMAX_BUFFERS[device_index] = ABSMAX_BUFFER = torch_empty(- n_elements_absmax,- dtype = torch_float32,- device = device,- requires_grad = False,- )-- if size > WEIGHT_BUFFER.numel():- WEIGHT_BUFFER.resize_(size)- if n_elements_absmax > ABSMAX_BUFFER.numel():- ABSMAX_BUFFER.resize_(n_elements_absmax)-- out = WEIGHT_BUFFER[:size].view(shape)- out_absmax = ABSMAX_BUFFER[:n_elements_absmax]+ if out is not None:+ assert out.shape == shape+ assert out.dtype == dtype+ elif use_global_buffer and _can_use_scratch(device):+ out = _scratch("weight", device, shape[0] * shape[1], dtype).view(shape)+ else:+ out = torch_empty(shape, dtype = dtype, device = device, requires_grad = False)+ if use_global_buffer and _can_use_scratch(device):+ out_absmax = _scratch("absmax", device, n_elements_absmax, torch_float32)else:- if out is None:- out = torch_empty(shape, dtype = dtype, device = device, requires_grad = False)- else:- assert out.shape == shape- assert out.dtype == dtypeout_absmax = torch_empty(- n_elements_absmax,- dtype = torch_float32,- device = device,- requires_grad = False,+ n_elements_absmax, dtype = torch_float32, device = device, requires_grad = False)- passptr_out_absmax = get_ptr(out_absmax)with torch_gpu_device(device):@@ -721,12 +787,114 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:ctypes_c_int(out.numel()),CUDA_STREAM,)- pass# Careful returning transposed data.is_transposed = True if W.shape[0] == 1 else Falsereturn out.t() if is_transposed else out- pass+ def fast_dequantize(+ W,+ quant_state = None,+ out = None,+ use_global_buffer = False,+ ):+ if isinstance(W, Float8Tensor):+ return W.dequantize()+ if quant_state is None:+ return W+ if W.dtype == torch.float8_e4m3fn:+ return weight_dequant(W, quant_state)+ if type(quant_state) is NVFP4QuantState:+ return nvfp4_dequantize(+ W, quant_state.scale, quant_state.global_scale, quant_state.dtype+ )+ if _quant_type(quant_state) != "nf4":+ # The NF4 kernels would decode an FP4 weight with the wrong codebook.+ return bnb_functional.dequantize_4bit(W, quant_state, out = out)+ if not _USE_NF4_KERNELS:+ return _fast_dequantize_ctypes(W, quant_state, out, use_global_buffer)+ device = W.device+ # Decode dequantizes each weight into the scratch buffer once per token: repeat the first+ # launch for this weight directly (see dequantize_nf4_planned).+ scratch = (+ out is None+ and use_global_buffer+ and _can_use_scratch(device)+ and device.index == _current_device()+ )+ if scratch and type(quant_state) is not list:+ plan = getattr(quant_state, "_unsloth_nf4_plan", None)+ if (+ plan is not None+ and plan[0] == W.data_ptr()+ and plan[1] is quant_state.absmax+ and plan[2] is quant_state.dtype+ ):+ if _SCRATCH.get(("weight", device.index)) is plan[4]:+ out = plan[5]+ else:+ shape = quant_state.shape+ out = _scratch("weight", device, shape[0] * shape[1], plan[2]).view(shape)+ plan[4], plan[5] = _SCRATCH[("weight", device.index)], out+ try:+ done = plan[3](out)+ except Exception:+ done = False+ if done:+ return out.t() if W.shape[0] == 1 else out+ quant_state._unsloth_nf4_plan = None+ out = None+ absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2 = (+ _unpack_quant_state(quant_state)+ )+ if out is not None:+ assert out.shape == shape+ assert out.dtype == dtype+ elif use_global_buffer and _can_use_scratch(device):+ out = _scratch("weight", device, shape[0] * shape[1], dtype).view(shape)+ try:+ if (+ scratch+ and dtype in (torch_float16, torch_bfloat16, torch_float32)+ and blocksize & (blocksize - 1) == 0+ and (code2 is None or blocksize2 & (blocksize2 - 1) == 0)+ ):+ # Launches once; the scratch buffer always starts 16 byte aligned, as reruns need.+ rerun = dequantize_nf4_planned(+ W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, out+ )+ if rerun is not None and type(quant_state) is not list:+ quant_state._unsloth_nf4_plan = _NF4Plan(+ [+ W.data_ptr(),+ absmax,+ dtype,+ rerun,+ _SCRATCH.get(("weight", device.index)),+ out,+ ]+ )+ else:+ out = dequantize_nf4(+ W,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ blocksize,+ blocksize2,+ shape,+ dtype,+ out = out,+ )+ except Exception as error:+ if not _nf4_kernels_failed(error):+ raise+ return _fast_dequantize_ctypes(W, quant_state, out, use_global_buffer)+ # Careful returning transposed data.+ is_transposed = True if W.shape[0] == 1 else False+ return out.t() if is_transposed else out+else:@torch.inference_mode@@ -914,37 +1082,22 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:- def fast_gemv(+ def _fast_gemv_ctypes(X,W,quant_state,out = None,):- if quant_state is None:- return torch_matmul(X, W, out = out)# Fast X @ W where seq_len == 1, from bitsandbytes functional.py#L1469 and TimDettmers/bitsandbytes#763._, q_len, hd = X.shape-- if type(quant_state) is not list:- absmax = quant_state.absmax- shape = quant_state.shape- dtype = quant_state.dtype- blocksize = quant_state.blocksize- stats = quant_state.code- offset = quant_state.offset- state2 = quant_state.state2- absmax2 = state2.absmax- code2 = state2.code- blocksize2 = state2.blocksize- else:- absmax, shape, dtype, blocksize, compressed_stats, quant_type, stats = quant_state- offset, state2 = compressed_stats- absmax2, code2, blocksize2, _, _, _, _ = state2- pass- global CUDA_STREAMS+ absmax, shape, dtype, blocksize, stats, code2, absmax2, offset, blocksize2 = (+ _unpack_quant_state(quant_state)+ )+ if shape[1] % blocksize != 0:+ # The gemv kernels assume each weight row starts a new quantization block.+ return torch_matmul(X, fast_dequantize(W, quant_state).t(), out = out)device = W.device- device_index = device.index- CUDA_STREAM = CUDA_STREAMS[device_index]+ CUDA_STREAM = c_void_p(_gpu_getCurrentRawStream(device.index))bout = shape[0]@@ -972,19 +1125,20 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:ldb = ctypes_c_int32(ldb)ldc = ctypes_c_int32(ldc)- df = torch_empty(absmax.shape, dtype = torch_float32, device = device)with torch_gpu_device(device):- cdequantize_blockwise_fp32(- get_ptr(code2),- get_ptr(absmax),- get_ptr(absmax2),- get_ptr(df),- ctypes_c_int(blocksize2),- ctypes_c_int(df.numel()),- CUDA_STREAM,- )- df += offset- absmax = df+ if code2 is not None:+ df = torch_empty(absmax.shape, dtype = torch_float32, device = device)+ cdequantize_blockwise_fp32(+ get_ptr(code2),+ get_ptr(absmax),+ get_ptr(absmax2),+ get_ptr(df),+ ctypes_c_int(blocksize2),+ ctypes_c_int(df.numel()),+ CUDA_STREAM,+ )+ df += offset+ absmax = dffx = (cgemm_4bit_inference_naive_fp16@@ -1008,11 +1162,90 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:blocksize,CUDA_STREAM,)- passreturn out- pass+ def fast_gemv(+ X,+ W,+ quant_state,+ out = None,+ ):+ if quant_state is None:+ return torch_matmul(X, W, out = out)+ if not _USE_NF4_KERNELS or not (_TRITON_GEMV_EAGER or _is_compiling()):+ return _fast_gemv_ctypes(X, W, quant_state, out)+ # Eager decode runs every weight once per token: repeat the first launch for this weight+ # directly (see gemv_nf4_planned).+ eager = (+ not _is_compiling()+ and type(quant_state) is not list+ and X.is_contiguous()+ and X.device.index == _current_device()+ )+ if eager:+ plan = getattr(quant_state, "_unsloth_gemv_plan", None)+ if (+ plan is not None+ and plan[0] == W.data_ptr()+ and plan[1] is quant_state.absmax+ and plan[2] is X.dtype+ and X.data_ptr() & 15 == plan[3]+ and (out is None or (out.dtype is X.dtype and out.data_ptr() & 15 == 0))+ ):+ if out is None:+ out = torch_empty((1, 1, plan[4]), dtype = X.dtype, device = X.device)+ try:+ done = plan[5](X, out)+ except Exception:+ done = False+ if done:+ return out+ quant_state._unsloth_gemv_plan = None+ absmax, shape, dtype, blocksize, code, code2, absmax2, offset, blocksize2 = (+ _unpack_quant_state(quant_state)+ )+ if shape[1] % blocksize != 0:+ # The gemv kernels assume each weight row starts a new quantization block.+ return torch_matmul(X, fast_dequantize(W, quant_state).t(), out = out)+ try:+ if (+ eager+ and X.dim() == 3+ and X.shape[0] == 1+ and X.shape[1] == 1+ and (out is None or (out.dtype is X.dtype and out.data_ptr() & 15 == 0))+ ):+ if out is None:+ out = torch_empty((1, 1, int(shape[0])), dtype = X.dtype, device = X.device)+ # Launches once.+ rerun = gemv_nf4_planned(+ X, W, absmax, code2, absmax2, offset, code, blocksize, blocksize2, shape, out+ )+ if rerun is not None:+ quant_state._unsloth_gemv_plan = _NF4Plan(+ [W.data_ptr(), absmax, X.dtype, X.data_ptr() & 15, int(shape[0]), rerun]+ )+ return out+ return gemv_nf4(+ X,+ W,+ absmax,+ code2,+ absmax2,+ offset,+ code,+ blocksize,+ blocksize2,+ shape,+ dtype,+ out = out,+ )+ except Exception as error:+ if not _nf4_kernels_failed(error):+ raise+ return _fast_gemv_ctypes(X, W, quant_state, out)+else:def fast_gemv(@@ -1210,6 +1443,35 @@ def fast_linear_forward(return out+# Traced, x.addmm_ copies x first; this op stays in place. No derivative: grad off only.+_HAS_ADDMM_OP = hasattr(torch.library, "custom_op")+if _HAS_ADDMM_OP:++ @torch.library.custom_op("unsloth::addmm_", mutates_args = ["out"])+ def _addmm_op(+ out: torch.Tensor, A: torch.Tensor, B: torch.Tensor, alpha: float, beta: float+ ) -> None:+ out.addmm_(A, B, alpha = alpha, beta = beta)+++def addmm_(+ out,+ A,+ B,+ alpha = 1.0,+ beta = 1.0,+):+ if (+ _HAS_ADDMM_OP+ and torch.compiler.is_compiling()+ and not torch.is_grad_enabled()+ and type(alpha) in (int, float)+ ):+ torch.ops.unsloth.addmm_(out, A, B, alpha, beta)+ return out+ return out.addmm_(A, B, alpha = alpha, beta = beta)++def matmul_lora(X,W,@@ -1252,6 +1514,6 @@ def matmul_lora(if A is not None:A, B = A.t(), B.t()XA = torch_matmul(X, A.to(dtype))- out.addmm_(XA, B.to(dtype), alpha = s)+ addmm_(out, XA, B.to(dtype), alpha = s)return out.view(batch, seq_len, -1) if reshape else out