Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
T4: hand batch-4 NF4 decode to the bitsandbytes gemm_4bit where it runs (#15988)
Daniel Hanc97cd18cea9b
diff --git a/tests/test_gemv_rows_cap_table.py b/tests/test_gemv_rows_cap_table.pynew file mode 100644index 00000000000..f63768911c4--- /dev/null+++ b/tests/test_gemv_rows_cap_table.py@@ -0,0 +1,54 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Per-architecture NF4 GEMV row cap and the T4 handoff to bitsandbytes gemm_4bit."""++import pytest+import torch++U = pytest.importorskip("unsloth.kernels.utils")+++def _fresh(monkeypatch, arch):+ monkeypatch.setattr(U, "_GEMV_ROWS_CAP", {})+ monkeypatch.setattr(U, "_GEMV_ROWS_BNB_FROM", {})+ monkeypatch.setattr(U, "_GEMV_MAX_ROWS", 4)+ monkeypatch.setattr(torch.version, "hip", None)+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: arch)+++ "arch, cap, bnb_from",+ [+ ((7, 5), 4, 4),+ ((8, 0), 2, 3),+ ((8, 9), 4, 5),+ ((12, 0), 4, 5),+ ((9, 0), 1, 2),+ ((10, 0), 1, 2),+ ],+)+def test_gemv_rows_cap_per_architecture(arch, cap, bnb_from, monkeypatch):+ _fresh(monkeypatch, arch)+ monkeypatch.delenv("UNSLOTH_GEMV_MAX_ROWS", raising = False)+ assert U._gemv_rows_cap(torch.device("cuda", 0)) == cap+ assert U._GEMV_ROWS_BNB_FROM[0] == bnb_from+++def test_gemv_rows_env_override_keeps_t4_batch_4_on_the_gemv(monkeypatch):+ _fresh(monkeypatch, (7, 5))+ monkeypatch.setenv("UNSLOTH_GEMV_MAX_ROWS", "4")+ assert U._gemv_rows_cap(torch.device("cuda", 0)) == 4+ assert U._GEMV_ROWS_BNB_FROM[0] == 5+++def test_t4_batch_4_steps_aside_when_the_bnb_gemm_applies(monkeypatch):+ _fresh(monkeypatch, (7, 5))+ monkeypatch.delenv("UNSLOTH_GEMV_MAX_ROWS", raising = False)+ asked = []+ monkeypatch.setattr(U, "_bnb_gemm_4bit_ok", lambda X, W, q: asked.append(X.shape[0]) or True)+ monkeypatch.setattr(U, "_GEMV_ROWS_CAP", {None: 4})+ monkeypatch.setattr(U, "_GEMV_ROWS_BNB_FROM", {None: 4})+ X = torch.randn(4, 1, 64, dtype = torch.float16)+ assert U._gemv_rows(X, torch.empty(1, dtype = torch.uint8), object(), None) is None+ assert asked == [4]diff --git a/tests/test_nf4_triton_gemv.py b/tests/test_nf4_triton_gemv.pyindex 865bec544a5..ef356aa9879 100644--- a/tests/test_nf4_triton_gemv.py+++ b/tests/test_nf4_triton_gemv.py@@ -347,6 +347,7 @@ def test_larger_decode_batches_still_dequantize(monkeypatch):def test_rows_or_dequantize_is_fixed_per_architecture(monkeypatch, arch, env, wins):utils = _utils_with_kernels()monkeypatch.setattr(utils, "_GEMV_ROWS_CAP", {})+ monkeypatch.setattr(utils, "_GEMV_ROWS_BNB_FROM", {})if env is None:monkeypatch.delenv("UNSLOTH_GEMV_MAX_ROWS", raising = False)else:@@ -377,6 +378,7 @@ def test_rows_above_the_cap_skip_straight_to_dequantize(monkeypatch):def test_rocm_never_takes_the_rows_gemv_by_default(monkeypatch):utils = _utils_with_kernels()monkeypatch.setattr(utils, "_GEMV_ROWS_CAP", {})+ monkeypatch.setattr(utils, "_GEMV_ROWS_BNB_FROM", {})monkeypatch.delenv("UNSLOTH_GEMV_MAX_ROWS", raising = False)monkeypatch.setattr(torch.version, "hip", "6.4")monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: (12, 0))@@ -423,7 +425,7 @@ def _bnb_gemm_utils():return utils[email protected]("bsz", [5, 8, 16])[email protected]("bsz", [4, 5, 8, 16])def test_decode_rows_above_the_gemv_cap_use_the_bnb_gemm(bsz, monkeypatch):utils = _bnb_gemm_utils()proj = _linear4bit(1024, 4096, dtype = torch.float16)@@ -433,9 +435,13 @@ def test_decode_rows_above_the_gemv_cap_use_the_bnb_gemm(bsz, monkeypatch):@ F.dequantize_4bit(proj.weight.data, proj.weight.quant_state).float().t())monkeypatch.setattr(utils, "_gemv_rows_cap", lambda device: 4)+ monkeypatch.setattr(utils, "_GEMV_ROWS_BNB_FROM", {X.device.index: 4})monkeypatch.setattr(utils, "_BNB_GEMM_4BIT", True)monkeypatch.setattr(utils, "_bnb_gemm_4bit_device_ok", lambda device: True)monkeypatch.setattr(utils, "fast_dequantize", lambda *a, **k: pytest.fail("dequantized"))+ calls = []+ real = utils.bnb.matmul_4bit+ monkeypatch.setattr(utils.bnb, "matmul_4bit", lambda *a, **k: calls.append(1) or real(*a, **k))with torch.no_grad():first = utils.fast_linear_forward(proj, X)assert first.shape == (bsz, 1, 1024)@@ -443,6 +449,7 @@ def test_decode_rows_above_the_gemv_cap_use_the_bnb_gemm(bsz, monkeypatch):out = torch.empty(bsz, 1, 1024, dtype = torch.float16, device = "cuda")assert utils.fast_linear_forward(proj, X, out = out) is outassert torch.equal(out, first)+ assert len(calls) == 2@pytest.mark.parametrize(@@ -455,6 +462,9 @@ def test_decode_rows_above_the_gemv_cap_use_the_bnb_gemm(bsz, monkeypatch):"bf16","weight_elsewhere","unaligned_k",+ "misaligned_x",+ "scalar_offset",+ "list_state2",],)def test_the_bnb_gemm_is_skipped_where_it_does_not_apply(case, monkeypatch):@@ -474,6 +484,13 @@ def test_the_bnb_gemm_is_skipped_where_it_does_not_apply(case, monkeypatch):# K not a multiple of the 64 blocksize.proj = _linear4bit(1024, 4128, dtype = dtype)X = torch.randn(bsz, 1, 4128, dtype = dtype, device = "cuda")+ if case == "misaligned_x":+ X = torch.randn(bsz * 4096 + 1, dtype = dtype, device = "cuda")[1:].view(bsz, 1, 4096)+ if case == "scalar_offset":+ proj.weight.quant_state.offset = proj.weight.quant_state.offset.item()+ if case == "list_state2":+ s2 = proj.weight.quant_state.state2+ proj.weight.quant_state.state2 = [s2.absmax, s2.code, s2.blocksize]if case == "weight_elsewhere":assert not utils._bnb_gemm_4bit_ok(X.view(bsz, -1), proj.weight.data.cpu(), proj.weight.quant_state@@ -487,6 +504,64 @@ def test_the_bnb_gemm_is_skipped_where_it_does_not_apply(case, monkeypatch):assert calls, case+ "case",+ [+ "eligible",+ "switch_off",+ "grad",+ "bf16",+ "compiling",+ "env_override",+ "other_arch",+ "misaligned_x",+ "scalar_offset",+ "list_state2",+ ],+)+def test_t4_batch_4_leaves_the_gemv_only_for_a_bnb_gemm_that_runs(case, monkeypatch):+ utils = _utils_with_kernels()+ _bnb_gemm_utils()+ monkeypatch.setattr(utils, "_GEMV_ROWS_CAP", {})+ monkeypatch.setattr(utils, "_GEMV_ROWS_BNB_FROM", {})+ monkeypatch.setattr(utils, "_GEMV_MAX_ROWS", 4)+ if case == "env_override":+ monkeypatch.setenv("UNSLOTH_GEMV_MAX_ROWS", "4")+ else:+ monkeypatch.delenv("UNSLOTH_GEMV_MAX_ROWS", raising = False)+ # Only the routing cache sees the faked architecture; Triton must still compile for the real GPU.+ with monkeypatch.context() as routing:+ routing.setattr(torch.version, "hip", None)+ arch = (8, 0) if case == "other_arch" else (7, 5)+ routing.setattr(torch.cuda, "get_device_capability", lambda *a: arch)+ utils._gemv_rows_cap(torch.device("cuda", torch.cuda.current_device()))+ monkeypatch.setattr(utils, "_BNB_GEMM_4BIT", case != "switch_off")+ monkeypatch.setattr(utils, "_bnb_gemm_4bit_device_ok", lambda device: True)+ if case == "compiling":+ monkeypatch.setattr(utils, "_is_compiling", lambda: True)+ dtype = torch.bfloat16 if case == "bf16" else torch.float16+ proj = _linear4bit(1024, 4096, dtype = dtype)+ W, W_quant = utils.get_lora_parameters_bias(proj)[:2]+ assert W_quant.nested+ if case == "scalar_offset":+ W_quant.offset = W_quant.offset.item()+ if case == "list_state2":+ s2 = W_quant.state2+ W_quant.state2 = [s2.absmax, s2.code, s2.blocksize]++ def x(bsz):+ if case == "misaligned_x":+ return torch.randn(bsz * 4096 + 1, dtype = dtype, device = "cuda")[1:].view(bsz, 1, 4096)+ return torch.randn(bsz, 1, 4096, dtype = dtype, device = "cuda")++ with torch.set_grad_enabled(case == "grad"):+ rows = {bsz: utils._gemv_rows(x(bsz), W, W_quant, None) is not None for bsz in (2, 3, 4)}+ if case == "other_arch":+ assert rows == {2: True, 3: False, 4: False}+ else:+ assert rows == {2: True, 3: True, 4: case != "eligible"}, case++@pytest.mark.parametrize("env", [None, "0", "1"])def test_the_bnb_gemm_runs_only_on_listed_archs_unless_forced(env, monkeypatch):utils = _bnb_gemm_utils()@@ -557,6 +632,7 @@ def test_rocm_batch_1_decode_takes_the_triton_gemv(monkeypatch, env):monkeypatch.delenv("UNSLOTH_GEMV_MAX_ROWS", raising = False)monkeypatch.setattr(utils, "_TRITON_GEMV_EAGER", nf4_gemv.triton_gemv_eager())monkeypatch.setattr(utils, "_GEMV_ROWS_CAP", {})+ monkeypatch.setattr(utils, "_GEMV_ROWS_BNB_FROM", {})calls = []for name in ("_fast_gemv_ctypes", "gemv_nf4_planned", "gemv_nf4"):real = getattr(utils, name)diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex ad5e7bc3462..27c6e372753 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -1500,6 +1500,10 @@ def _quant_state_dtype(quant_state):# decoding is reproducible; others dequantize. UNSLOTH_GEMV_MAX_ROWS overrides where the Triton GEMV runs._GEMV_ROWS_BY_ARCH = {(7, 5): 4, (8, 0): 2, (8, 9): 4, (12, 0): 4}_GEMV_ROWS_CAP = {}+# Rows from which bnb gemm_4bit replaces the GEMV. Colab T4 eager decode (Qwen3-4B, Llama-3.2-3B): batch 4+# 13-30% faster at equal NLL vs fp32; batch 2 ~40% slower (host bound), batch 3 a tie.+_GEMV_ROWS_BNB_FROM_BY_ARCH = {(7, 5): 4}+_GEMV_ROWS_BNB_FROM = {}def _gemv_rows_cap(device):@@ -1508,6 +1512,7 @@ def _gemv_rows_cap(device):if cap is None:if "UNSLOTH_GEMV_MAX_ROWS" in os.environ:cap = _GEMV_MAX_ROWS+ bnb_from = cap + 1else:try:# ROCm reports gfx versions here (gfx1200 is (12, 0)), so the table is CUDA only.@@ -1515,6 +1520,8 @@ def _gemv_rows_cap(device):except Exception:arch = Nonecap = min(_GEMV_MAX_ROWS, _GEMV_ROWS_BY_ARCH.get(arch, 1))+ bnb_from = _GEMV_ROWS_BNB_FROM_BY_ARCH.get(arch, cap + 1)+ _GEMV_ROWS_BNB_FROM[index] = bnb_from_GEMV_ROWS_CAP[index] = capreturn cap@@ -1574,13 +1581,22 @@ def _bnb_gemm_4bit_ok(X, W, quant_state):and _bnb_gemm_4bit_device_ok(X.device)# bitsandbytes passes raw pointers to the kernel on X's device.and W.device == X.device+ # Its fp16 kernels load 16 bytes at a time from both pointers.+ and not (X.data_ptr() & 15 or W.data_ptr() & 15)and type(quant_state) is not listand quant_state.absmax.device == X.deviceand getattr(quant_state, "quant_type", None) in ("nf4", "fp4")# bitsandbytes dequantizes (and warns) itself when K is not a multiple of the blocksize.and quant_state.shape[-1] % quant_state.blocksize == 0- # matmul_4bit raises for any other nested blocksize.- and (not quant_state.nested or quant_state.state2.blocksize == 256)+ # matmul_4bit raises for any other nested blocksize, a list state2 or a non-tensor offset.+ and (+ not quant_state.nested+ or (+ type(quant_state.state2) is not list+ and quant_state.state2.blocksize == 256+ and isinstance(quant_state.offset, torch_Tensor)+ )+ ))@@ -1590,6 +1606,12 @@ def _gemv_rows(X, W, quant_state, out):# First, so rows above the cap cost the dequantize path nothing.if not 2 <= M <= _gemv_rows_cap(X.device) or W.device != X.device:return None+ if (+ not _is_compiling()+ and M >= _GEMV_ROWS_BNB_FROM.get(X.device.index, M + 1)+ and _bnb_gemm_4bit_ok(X, W, quant_state)+ ):+ return Noneeager = (not _is_compiling()and type(quant_state) is not list