Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Run the RoPE and NF4 Triton kernels on GPUs older than Volta (#13153)
Daniel Han27e088f7e0b4
diff --git a/tests/test_pre_volta_triton_codegen.py b/tests/test_pre_volta_triton_codegen.pynew file mode 100644index 00000000000..6b3566ca0cf--- /dev/null+++ b/tests/test_pre_volta_triton_codegen.py@@ -0,0 +1,99 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""ptxas rejects ld eviction hints below sm_70, so the indexed RoPE kernel crashed on Maxwell+and Pascal ("Modifier '.evict_first' on 'ld' requires .target sm_70"). Compiles for the old+targets with Triton's own ptxas: no old GPU needed."""++import pytest+import torch++if not torch.cuda.is_available() or torch.version.hip is not None:+ pytest.skip("needs an NVIDIA CUDA device", allow_module_level = True)++triton = pytest.importorskip("triton")+from triton.backends.compiler import GPUTarget+from triton.compiler import ASTSource++from unsloth.kernels import utils as utils_mod+from unsloth.kernels import rope_embedding as rope_mod+++def _compile(fn, signature, constexprs, cc):+ signature = {name: signature.get(name, "constexpr") for name in fn.arg_names}+ return triton.compile(+ ASTSource(fn, signature = signature, constexprs = constexprs),+ target = GPUTarget("cuda", cc, 32),+ )+++ "capabilities, supported",+ [+ ([(6, 1)], False),+ ([(5, 2)], False),+ ([(7, 0)], True),+ ([(7, 5)], True),+ ([(10, 0)], True),+ ([(8, 6), (6, 1)], False),+ ([(9, 0), (12, 0)], True),+ ],+)+def test_fused_nf4_kernels_only_from_volta(capabilities, supported, monkeypatch):+ monkeypatch.setattr(utils_mod, "DEVICE_TYPE", "cuda")+ monkeypatch.setattr(utils_mod, "DEVICE_COUNT", len(capabilities))+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: capabilities[index])+ assert utils_mod._nf4_kernels_supported() is supported+++def _rope_qk_constexprs(capability, monkeypatch):+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: capability)+ rope_mod._eviction_hints_ok_at.cache_clear()+ try:+ evict = rope_mod._eviction_hints_ok(torch.device("cuda", 0))+ finally:+ rope_mod._eviction_hints_ok_at.cache_clear()+ return dict(+ Q_seq_stride = 512,+ K_seq_stride = 256,+ cos_row_stride = 64,+ sin_row_stride = 64,+ head_dim = 64,+ n_heads_K = 4,+ BACKWARD_PASS = False,+ HAS_ROPE_INDICES = True,+ LONG_INDEXING = False,+ BLOCK_SIZE = 64,+ EVICT_INDICES = evict,+ )+++_ROPE_SIGNATURE = {+ "Q": "*fp16",+ "Q_batch_stride": "i32",+ "Q_head_stride": "i32",+ "K": "*fp16",+ "K_batch_stride": "i32",+ "K_head_stride": "i32",+ "cos": "*fp16",+ "sin": "*fp16",+ "rope_embedding_indices": "*i32",+ "seqlen": "i32",+}++[email protected]("capability", [(5, 2), (6, 0), (6, 1)])+def test_indexed_rope_assembles_before_volta(capability, monkeypatch):+ constexprs = _rope_qk_constexprs(capability, monkeypatch)+ assert constexprs["EVICT_INDICES"] is False+ cc = capability[0] * 10 + capability[1]+ kernel = rope_mod._rope_embedding_QK.fn+ assert _compile(kernel, _ROPE_SIGNATURE, constexprs, cc).asm["cubin"]++[email protected]("capability", [(7, 0), (7, 5), (8, 0), (9, 0), (10, 0), (12, 0)])+def test_volta_and_newer_keep_the_eviction_hint(capability, monkeypatch):+ constexprs = _rope_qk_constexprs(capability, monkeypatch)+ assert constexprs["EVICT_INDICES"] is True+ ptx = _compile(rope_mod._rope_embedding_QK.fn, _ROPE_SIGNATURE, constexprs, 75).asm["ptx"]+ assert "evict_first" in ptxdiff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex b9fc9ea54f6..8d35ce6cf35 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -31,7 +31,7 @@ from unsloth_zoo.loss_utils import (def _cross_entropy_forward(logits_ptr,- logits_row_stride,+ logits_row_stride: tl.constexpr, # VOCAB_SIZEloss_ptr,logsumexp_ptr,labels_ptr,diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.pyindex 2759b760579..24f1a813943 100644--- a/unsloth/kernels/layernorm.py+++ b/unsloth/kernels/layernorm.py@@ -22,9 +22,9 @@ from unsloth_zoo.patching_utils import (@triton.jitdef layernorm_forward(Y,- Y_row_stride,+ Y_row_stride: tl.constexpr, # n_colsX,- X_row_stride,+ X_row_stride: tl.constexpr, # n_colsW,b,r,@@ -66,9 +66,9 @@ def layernorm_forward(@triton.jitdef layernorm_backward(dY,- dY_row_stride,+ dY_row_stride: tl.constexpr, # n_colsX,- X_row_stride,+ X_row_stride: tl.constexpr, # n_colsW,b,r,diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex 7a42fab70c9..7069254dd62 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -10,6 +10,7 @@# You should have received a copy of the GNU Lesser General Public License# along with this program. If not, see <https://www.gnu.org/licenses/>.+import functoolsimport tritonimport triton.language as tlimport torch@@ -27,17 +28,17 @@ from .rms_layernorm import (def _rope_embedding_QK(Q,- Q_batch_stride,- Q_head_stride,- Q_seq_stride,+ Q_batch_stride, # s*h*d: varies with seq_len+ Q_head_stride, # d fwd, s*d bwd+ Q_seq_stride: tl.constexpr, # h*d fwd, d bwdK,K_batch_stride,K_head_stride,- K_seq_stride,+ K_seq_stride: tl.constexpr, # kv*d fwd, d bwdcos,- cos_row_stride,+ cos_row_stride: tl.constexpr, # dsin,- sin_row_stride,+ sin_row_stride: tl.constexpr, # drope_embedding_indices,seqlen,head_dim: tl.constexpr,@@ -46,6 +47,7 @@ def _rope_embedding_QK(HAS_ROPE_INDICES: tl.constexpr,LONG_INDEXING: tl.constexpr,BLOCK_SIZE: tl.constexpr,+ EVICT_INDICES: tl.constexpr,):row_position = tl.program_id(0)head_position = tl.program_id(1)@@ -54,10 +56,13 @@ def _rope_embedding_QK(mask = col_offsets < half_head_dimif HAS_ROPE_INDICES:- rot_position = tl.load(- rope_embedding_indices + row_position,- eviction_policy = "evict_first",- ).to(tl.int32)+ if EVICT_INDICES:+ rot_position = tl.load(+ rope_embedding_indices + row_position,+ eviction_policy = "evict_first",+ ).to(tl.int32)+ else:+ rot_position = tl.load(rope_embedding_indices + row_position).to(tl.int32)else:rot_position = row_position % seqlen@@ -250,6 +255,19 @@ class Fast_RoPE_Embedding(torch.autograd.Function):)[email protected]_cache(maxsize = None)+def _eviction_hints_ok_at(device_type, index):+ # ptxas rejects ld eviction hints below sm_70 (Maxwell, Pascal).+ if device_type != "cuda" or torch.version.hip is not None:+ return True+ return torch.cuda.get_device_capability(index)[0] >= 7+++def _eviction_hints_ok(device):+ index = device.index if device.index is not None else torch.cuda.current_device()+ return _eviction_hints_ok_at(device.type, index)++def _rope_qk(Q, K, cos, sin, rope_ptr, has_indices, backward, wrap):# Rotates Q [batch, n_heads_Q, seq_len, head_dim] and K in place, at any strides.batch, n_heads_Q, seq_len, head_dim = Q.shape@@ -276,6 +294,7 @@ def _rope_qk(Q, K, cos, sin, rope_ptr, has_indices, backward, wrap):HAS_ROPE_INDICES = has_indices,LONG_INDEXING = long_indexing(Q, K, block = BLOCK_SIZE),BLOCK_SIZE = BLOCK_SIZE,+ EVICT_INDICES = _eviction_hints_ok(Q.device),num_warps = num_warps,)diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 0ad6e33fdaa..7e7e40152a5 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -582,6 +582,13 @@ def _scratch(kind, device, numel, dtype):return buffer[:numel]+def _nf4_kernels_supported():+ # ptxas rejects their ld eviction hints below sm_70 (Maxwell, Pascal).+ if DEVICE_TYPE == "hip":+ return True+ return all(torch.cuda.get_device_capability(i)[0] >= 7 for i in range(DEVICE_COUNT))++# Fused NF4 kernels, traceable by torch.compile and on the live stream by construction.# UNSLOTH_BNB_TRITON=0 keeps the bitsandbytes ctypes path._USE_NF4_KERNELS = False@@ -592,6 +599,7 @@ if (DEVICE_TYPE in ("cuda", "hip")and HAS_CUDA_STREAMand os.environ.get("UNSLOTH_BNB_TRITON", "1") != "0"+ and _nf4_kernels_supported()):try:import triton