Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Run the RoPE and NF4 Triton kernels on GPUs older than Volta (#13153)

Daniel Han27e088f7e0b4
diff --git a/tests/test_pre_volta_triton_codegen.py b/tests/test_pre_volta_triton_codegen.py
new file mode 100644
index 00000000000..6b3566ca0cf
--- /dev/null
+++ b/tests/test_pre_volta_triton_codegen.py
@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""ptxas rejects ld eviction hints below sm_70, so the indexed RoPE kernel crashed on Maxwell
+and Pascal ("Modifier '.evict_first' on 'ld' requires .target sm_70"). Compiles for the old
+targets with Triton's own ptxas: no old GPU needed."""
+
+import pytest
+import torch
+
+if not torch.cuda.is_available() or torch.version.hip is not None:
+ pytest.skip("needs an NVIDIA CUDA device", allow_module_level = True)
+
+triton = pytest.importorskip("triton")
+from triton.backends.compiler import GPUTarget
+from triton.compiler import ASTSource
+
+from unsloth.kernels import utils as utils_mod
+from unsloth.kernels import rope_embedding as rope_mod
+
+
+def _compile(fn, signature, constexprs, cc):
+ signature = {name: signature.get(name, "constexpr") for name in fn.arg_names}
+ return triton.compile(
+ ASTSource(fn, signature = signature, constexprs = constexprs),
+ target = GPUTarget("cuda", cc, 32),
+ )
+
+
+ "capabilities, supported",
+ [
+ ([(6, 1)], False),
+ ([(5, 2)], False),
+ ([(7, 0)], True),
+ ([(7, 5)], True),
+ ([(10, 0)], True),
+ ([(8, 6), (6, 1)], False),
+ ([(9, 0), (12, 0)], True),
+ ],
+)
+def test_fused_nf4_kernels_only_from_volta(capabilities, supported, monkeypatch):
+ monkeypatch.setattr(utils_mod, "DEVICE_TYPE", "cuda")
+ monkeypatch.setattr(utils_mod, "DEVICE_COUNT", len(capabilities))
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: capabilities[index])
+ assert utils_mod._nf4_kernels_supported() is supported
+
+
+def _rope_qk_constexprs(capability, monkeypatch):
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = None: capability)
+ rope_mod._eviction_hints_ok_at.cache_clear()
+ try:
+ evict = rope_mod._eviction_hints_ok(torch.device("cuda", 0))
+ finally:
+ rope_mod._eviction_hints_ok_at.cache_clear()
+ return dict(
+ Q_seq_stride = 512,
+ K_seq_stride = 256,
+ cos_row_stride = 64,
+ sin_row_stride = 64,
+ head_dim = 64,
+ n_heads_K = 4,
+ BACKWARD_PASS = False,
+ HAS_ROPE_INDICES = True,
+ LONG_INDEXING = False,
+ BLOCK_SIZE = 64,
+ EVICT_INDICES = evict,
+ )
+
+
+_ROPE_SIGNATURE = {
+ "Q": "*fp16",
+ "Q_batch_stride": "i32",
+ "Q_head_stride": "i32",
+ "K": "*fp16",
+ "K_batch_stride": "i32",
+ "K_head_stride": "i32",
+ "cos": "*fp16",
+ "sin": "*fp16",
+ "rope_embedding_indices": "*i32",
+ "seqlen": "i32",
+}
+
+
[email protected]("capability", [(5, 2), (6, 0), (6, 1)])
+def test_indexed_rope_assembles_before_volta(capability, monkeypatch):
+ constexprs = _rope_qk_constexprs(capability, monkeypatch)
+ assert constexprs["EVICT_INDICES"] is False
+ cc = capability[0] * 10 + capability[1]
+ kernel = rope_mod._rope_embedding_QK.fn
+ assert _compile(kernel, _ROPE_SIGNATURE, constexprs, cc).asm["cubin"]
+
+
[email protected]("capability", [(7, 0), (7, 5), (8, 0), (9, 0), (10, 0), (12, 0)])
+def test_volta_and_newer_keep_the_eviction_hint(capability, monkeypatch):
+ constexprs = _rope_qk_constexprs(capability, monkeypatch)
+ assert constexprs["EVICT_INDICES"] is True
+ ptx = _compile(rope_mod._rope_embedding_QK.fn, _ROPE_SIGNATURE, constexprs, 75).asm["ptx"]
+ assert "evict_first" in ptx
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index b9fc9ea54f6..8d35ce6cf35 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -31,7 +31,7 @@ from unsloth_zoo.loss_utils import (
def _cross_entropy_forward(
logits_ptr,
- logits_row_stride,
+ logits_row_stride: tl.constexpr, # VOCAB_SIZE
loss_ptr,
logsumexp_ptr,
labels_ptr,
diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py
index 2759b760579..24f1a813943 100644
--- a/unsloth/kernels/layernorm.py
+++ b/unsloth/kernels/layernorm.py
@@ -22,9 +22,9 @@ from unsloth_zoo.patching_utils import (
@triton.jit
def layernorm_forward(
Y,
- Y_row_stride,
+ Y_row_stride: tl.constexpr, # n_cols
X,
- X_row_stride,
+ X_row_stride: tl.constexpr, # n_cols
W,
b,
r,
@@ -66,9 +66,9 @@ def layernorm_forward(
@triton.jit
def layernorm_backward(
dY,
- dY_row_stride,
+ dY_row_stride: tl.constexpr, # n_cols
X,
- X_row_stride,
+ X_row_stride: tl.constexpr, # n_cols
W,
b,
r,
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index 7a42fab70c9..7069254dd62 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -10,6 +10,7 @@
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
+import functools
import triton
import triton.language as tl
import torch
@@ -27,17 +28,17 @@ from .rms_layernorm import (
def _rope_embedding_QK(
Q,
- Q_batch_stride,
- Q_head_stride,
- Q_seq_stride,
+ Q_batch_stride, # s*h*d: varies with seq_len
+ Q_head_stride, # d fwd, s*d bwd
+ Q_seq_stride: tl.constexpr, # h*d fwd, d bwd
K,
K_batch_stride,
K_head_stride,
- K_seq_stride,
+ K_seq_stride: tl.constexpr, # kv*d fwd, d bwd
cos,
- cos_row_stride,
+ cos_row_stride: tl.constexpr, # d
sin,
- sin_row_stride,
+ sin_row_stride: tl.constexpr, # d
rope_embedding_indices,
seqlen,
head_dim: tl.constexpr,
@@ -46,6 +47,7 @@ def _rope_embedding_QK(
HAS_ROPE_INDICES: tl.constexpr,
LONG_INDEXING: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
+ EVICT_INDICES: tl.constexpr,
):
row_position = tl.program_id(0)
head_position = tl.program_id(1)
@@ -54,10 +56,13 @@ def _rope_embedding_QK(
mask = col_offsets < half_head_dim
if HAS_ROPE_INDICES:
- rot_position = tl.load(
- rope_embedding_indices + row_position,
- eviction_policy = "evict_first",
- ).to(tl.int32)
+ if EVICT_INDICES:
+ rot_position = tl.load(
+ rope_embedding_indices + row_position,
+ eviction_policy = "evict_first",
+ ).to(tl.int32)
+ else:
+ rot_position = tl.load(rope_embedding_indices + row_position).to(tl.int32)
else:
rot_position = row_position % seqlen
@@ -250,6 +255,19 @@ class Fast_RoPE_Embedding(torch.autograd.Function):
)
[email protected]_cache(maxsize = None)
+def _eviction_hints_ok_at(device_type, index):
+ # ptxas rejects ld eviction hints below sm_70 (Maxwell, Pascal).
+ if device_type != "cuda" or torch.version.hip is not None:
+ return True
+ return torch.cuda.get_device_capability(index)[0] >= 7
+
+
+def _eviction_hints_ok(device):
+ index = device.index if device.index is not None else torch.cuda.current_device()
+ return _eviction_hints_ok_at(device.type, index)
+
+
def _rope_qk(Q, K, cos, sin, rope_ptr, has_indices, backward, wrap):
# Rotates Q [batch, n_heads_Q, seq_len, head_dim] and K in place, at any strides.
batch, n_heads_Q, seq_len, head_dim = Q.shape
@@ -276,6 +294,7 @@ def _rope_qk(Q, K, cos, sin, rope_ptr, has_indices, backward, wrap):
HAS_ROPE_INDICES = has_indices,
LONG_INDEXING = long_indexing(Q, K, block = BLOCK_SIZE),
BLOCK_SIZE = BLOCK_SIZE,
+ EVICT_INDICES = _eviction_hints_ok(Q.device),
num_warps = num_warps,
)
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 0ad6e33fdaa..7e7e40152a5 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -582,6 +582,13 @@ def _scratch(kind, device, numel, dtype):
return buffer[:numel]
+def _nf4_kernels_supported():
+ # ptxas rejects their ld eviction hints below sm_70 (Maxwell, Pascal).
+ if DEVICE_TYPE == "hip":
+ return True
+ return all(torch.cuda.get_device_capability(i)[0] >= 7 for i in range(DEVICE_COUNT))
+
+
# Fused NF4 kernels, traceable by torch.compile and on the live stream by construction.
# UNSLOTH_BNB_TRITON=0 keeps the bitsandbytes ctypes path.
_USE_NF4_KERNELS = False
@@ -592,6 +599,7 @@ if (
DEVICE_TYPE in ("cuda", "hip")
and HAS_CUDA_STREAM
and os.environ.get("UNSLOTH_BNB_TRITON", "1") != "0"
+ and _nf4_kernels_supported()
):
try:
import triton