Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

RoPE: multi-head tiles, Q and K in one launch, no clones (2-2.7x per call) (#14417)

Daniel Han20e8179bfb0e
diff --git a/tests/test_rope_kernel_layouts.py b/tests/test_rope_kernel_layouts.py
new file mode 100644
index 00000000000..ee883d195fe
--- /dev/null
+++ b/tests/test_rope_kernel_layouts.py
@@ -0,0 +1,234 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""fast_rope_embedding against an fp64 oracle over head dims, head counts, GQA, input layouts,
+packed position ids, dtypes and gradient strides, plus the in-place contract callers see."""
+
+import pytest
+import torch
+
+pytestmark = [
+ pytest.mark.gpu,
+ pytest.mark.skipif(not torch.cuda.is_available(), reason = "Triton RoPE kernel needs CUDA"),
+]
+
+BF16 = torch.bfloat16
+# (batch, seq, heads, kv heads, head_dim): pow2 and not (80, 96), 1 to 40 heads, GQA, decode.
+SHAPES = [
+ (2, 37, 8, 8, 64),
+ (1, 64, 32, 8, 128),
+ (3, 20, 4, 1, 64),
+ (2, 33, 40, 8, 128),
+ (1, 17, 6, 2, 80),
+ (2, 5, 12, 12, 96),
+ (2, 9, 16, 4, 256),
+ (1, 1, 32, 8, 128),
+ (4, 3, 1, 1, 32),
+]
+
+
+def _cos_sin(rows, head_dim, dtype):
+ inv = 1.0 / (10000 ** (torch.arange(0, head_dim, 2, device = "cuda").float() / head_dim))
+ freqs = torch.outer(torch.arange(rows, device = "cuda").float(), inv)
+ emb = torch.cat((freqs, freqs), -1)
+ return emb.cos().to(dtype), emb.sin().to(dtype)
+
+
+def _packed_ids(batch, seq):
+ # Packed sequences restart at 0; rows differ per batch.
+ lens = [max(1, seq // 3), max(1, seq // 3)]
+ if seq - sum(lens) > 0:
+ lens.append(seq - sum(lens))
+ row = torch.cat([torch.arange(n, dtype = torch.int32) for n in lens])[:seq]
+ return torch.stack([row + b for b in range(batch)]).cuda()
+
+
+def _oracle(
+ x,
+ cos,
+ sin,
+ ids,
+ backward = False,
+):
+ # x [b, h, s, d]; fp64 rotation of the exact low-precision inputs.
+ x, cos, sin = x.double(), cos.double(), sin.double()
+ seq = x.shape[2]
+ if ids is None:
+ c, s = cos[:seq], sin[:seq]
+ else:
+ c, s = cos[ids.long()].unsqueeze(1), sin[ids.long()].unsqueeze(1)
+ half = x.shape[-1] // 2
+ x0, x1 = x[..., :half], x[..., half:]
+ c, s = c[..., :half], s[..., :half]
+ if backward:
+ s = -s
+ out = torch.cat((x0 * c - x1 * s, x1 * c + x0 * s), -1)
+ # Magnitude of the terms, for a rounding bound that survives cancellation.
+ scale = torch.cat(((x0 * c).abs() + (x1 * s).abs(), (x1 * c).abs() + (x0 * s).abs()), -1)
+ return out, scale
+
+
+def _assert_rounding_close(got, ref, scale, dtype):
+ # fma(a, b, -round(c * d)) then a store: at most ~1 ulp of the terms involved.
+ eps = torch.finfo(dtype).eps
+ err = (got.double() - ref).abs()
+ bound = eps * (scale + ref.abs()) + torch.finfo(dtype).tiny
+ worst = (err / bound).max().item()
+ assert worst <= 1.0, f"error {worst:.3f}x the rounding bound"
+
+
+def _inputs(shape, dtype, layout, g):
+ b, s, h, kv, d = shape
+ if layout == "projection":
+ # As the attention modules build them: [b, h, s, d] views of [b, s, h * d] projections.
+ q_lin = torch.randn(b, s, h * d, device = "cuda", generator = g).to(dtype)
+ k_lin = torch.randn(b, s, kv * d, device = "cuda", generator = g).to(dtype)
+ q_lin.requires_grad_(True)
+ k_lin.requires_grad_(True)
+ Q = (q_lin * 1).view(b, s, h, d).transpose(1, 2)
+ K = (k_lin * 1).view(b, s, kv, d).transpose(1, 2)
+ to_bhsd = lambda t, n: t.view(b, s, n, d).transpose(1, 2)
+ else:
+ q_lin = torch.randn(b, h, s, d, device = "cuda", generator = g).to(dtype).requires_grad_(True)
+ k_lin = torch.randn(b, kv, s, d, device = "cuda", generator = g).to(dtype).requires_grad_(True)
+ Q, K = q_lin * 1, k_lin * 1
+ to_bhsd = lambda t, n: t
+ return q_lin, k_lin, Q, K, to_bhsd
+
+
[email protected]("grad_kind", ["dense", "transposed", "expanded"])
[email protected]("layout", ["projection", "contiguous"])
[email protected]("with_ids", [False, True], ids = ["positions", "position_ids"])
[email protected]("dtype", [torch.float16, BF16, torch.float32], ids = str)
[email protected]("shape", SHAPES, ids = lambda s: "b{}_s{}_h{}_kv{}_d{}".format(*s))
+def test_rope_matches_fp64_oracle(shape, dtype, with_ids, layout, grad_kind):
+ from unsloth.kernels.rope_embedding import fast_rope_embedding
+
+ b, s, h, kv, d = shape
+ g = torch.Generator(device = "cuda").manual_seed(sum(shape))
+ cos, sin = _cos_sin(s + 11, d, dtype)
+ ids = _packed_ids(b, s) if with_ids else None
+ q_lin, k_lin, Q, K, to_bhsd = _inputs(shape, dtype, layout, g)
+ q_in, k_in = Q.detach().clone(), K.detach().clone()
+
+ q, k = fast_rope_embedding(Q, K, cos, sin, ids)
+ assert q.shape == (b, h, s, d) and k.shape == (b, kv, s, d)
+ for got, x in ((q, q_in), (k, k_in)):
+ ref, scale = _oracle(x, cos, sin, ids)
+ _assert_rounding_close(got, ref, scale, dtype)
+
+ if grad_kind == "dense":
+ dq = torch.randn(q.shape, device = "cuda", generator = g).to(dtype)
+ dk = torch.randn(k.shape, device = "cuda", generator = g).to(dtype)
+ elif grad_kind == "transposed":
+ dq = torch.randn(b, s, h, d, device = "cuda", generator = g).to(dtype).transpose(1, 2)
+ dk = torch.randn(b, s, kv, d, device = "cuda", generator = g).to(dtype).transpose(1, 2)
+ else:
+ dq = torch.randn(1, 1, 1, d, device = "cuda", generator = g).to(dtype).expand(q.shape)
+ dk = torch.randn(1, 1, 1, d, device = "cuda", generator = g).to(dtype).expand(k.shape)
+ dq_before, dk_before = dq.clone(), dk.clone()
+ torch.autograd.backward((q, k), (dq, dk))
+ assert torch.equal(dq, dq_before) and torch.equal(dk, dk_before), "upstream grad was mutated"
+ for grad, n, up in ((q_lin.grad, h, dq), (k_lin.grad, kv, dk)):
+ ref, scale = _oracle(up, cos, sin, ids, backward = True)
+ _assert_rounding_close(to_bhsd(grad, n), ref, scale, dtype)
+
+
+def test_rope_float32_tables_with_half_inputs():
+ """Gemma-style fp32 cos / sin with bf16 Q / K computes in fp32 and stores bf16."""
+ from unsloth.kernels.rope_embedding import fast_rope_embedding
+
+ shape = (2, 19, 8, 2, 128)
+ g = torch.Generator(device = "cuda").manual_seed(3)
+ cos, sin = _cos_sin(32, 128, torch.float32)
+ for ids in (None, _packed_ids(2, 19)):
+ _, _, Q, K, _ = _inputs(shape, BF16, "projection", g)
+ q_in, k_in = Q.detach().clone(), K.detach().clone()
+ q, k = fast_rope_embedding(Q, K, cos, sin, ids)
+ assert q.dtype == BF16 and k.dtype == BF16
+ for got, x in ((q, q_in), (k, k_in)):
+ ref, scale = _oracle(x, cos, sin, ids)
+ _assert_rounding_close(got, ref, scale, BF16)
+
+
[email protected]("with_ids", [False, True], ids = ["positions", "position_ids"])
+def test_rope_output_layout_and_aliasing(with_ids):
+ """Layouts and aliasing the attention modules see: outputs keep the projection layout, a
+ strided position-id input is not mutated, gradients come back contiguous."""
+ from unsloth.kernels.rope_embedding import fast_rope_embedding
+
+ b, s, h, kv, d = 2, 24, 8, 2, 64
+ g = torch.Generator(device = "cuda").manual_seed(5)
+ cos, sin = _cos_sin(32, d, torch.float16)
+ ids = _packed_ids(b, s) if with_ids else None
+ q_lin, k_lin, Q, K, _ = _inputs((b, s, h, kv, d), torch.float16, "projection", g)
+ Q_before = Q.detach().clone()
+ q, k = fast_rope_embedding(Q, K, cos, sin, ids)
+ assert q.stride() == Q.stride() and k.stride() == K.stride()
+ if with_ids:
+ assert torch.equal(Q.detach(), Q_before), "a strided input was rotated in place"
+ else:
+ # The position path rotates the [b, s, h, d] buffer in place, as it always has.
+ assert q.data_ptr() == Q.data_ptr()
+ torch.autograd.backward((q, k), (torch.ones_like(q), torch.ones_like(k)))
+ assert q_lin.grad.is_contiguous() and k_lin.grad.is_contiguous()
+
+
[email protected]("with_ids", [False, True], ids = ["positions", "position_ids"])
+def test_rope_contiguous_qk_input_rotates_in_place(with_ids):
+ """[b, h, s, d]-contiguous Q / K on the position-id path are rotated in place (no copy)."""
+ from unsloth.kernels.rope_embedding import Fast_RoPE_Embedding_QK
+
+ b, s, h, kv, d = 1, 8, 4, 2, 64
+ cos, sin = _cos_sin(16, d, torch.float32)
+ Q = torch.randn(b, h, s, d, device = "cuda")
+ K = torch.randn(b, kv, s, d, device = "cuda")
+ ids = _packed_ids(b, s)
+ with torch.no_grad():
+ q, k = Fast_RoPE_Embedding_QK.apply(Q, K, cos, sin, ids)
+ assert q.data_ptr() == Q.data_ptr() and k.data_ptr() == K.data_ptr()
+
+
+def test_rope_heads_per_program_bounds():
+ from unsloth.kernels.rope_embedding import ROPE_TILE_ELEMENTS, _heads_per_program
+ for n_heads in (1, 2, 3, 7, 8, 32, 40, 64, 128):
+ for block in (16, 32, 64, 128, 256, 1024, 2048):
+ heads = _heads_per_program(n_heads, block)
+ assert heads >= 1 and heads & (heads - 1) == 0
+ assert heads <= max(1, ROPE_TILE_ELEMENTS // block)
+ assert heads <= triton_next_pow2(n_heads)
+
+
+def triton_next_pow2(n):
+ return 1 << (n - 1).bit_length()
+
+
[email protected]("layout", ["projection", "contiguous"])
+def test_rope_positions_pair_matches_two_single_calls(layout):
+ """The fused Q + K position path keeps the bytes, strides, aliasing and gradient layout of
+ Fast_RoPE_Embedding applied to Q and K separately."""
+ from unsloth.kernels.rope_embedding import Fast_RoPE_Embedding, fast_rope_embedding
+
+ shape = (2, 21, 8, 2, 64)
+ cos, sin = _cos_sin(32, 64, BF16)
+
+ def run(fn):
+ g = torch.Generator(device = "cuda").manual_seed(11)
+ q_lin, k_lin, Q, K, _ = _inputs(shape, BF16, layout, g)
+ q, k = fn(Q, K)
+ aliased = (q.data_ptr() == Q.data_ptr(), k.data_ptr() == K.data_ptr())
+ dq = torch.randn(q.shape, device = "cuda", generator = g).to(BF16)
+ dk = torch.randn(k.shape, device = "cuda", generator = g).to(BF16)
+ torch.autograd.backward((q, k), (dq, dk))
+ return q, k, q_lin.grad, k_lin.grad, aliased
+
+ def single(Q, K):
+ q = Fast_RoPE_Embedding.apply(Q.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)
+ k = Fast_RoPE_Embedding.apply(K.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)
+ return q, k
+
+ ref = run(single)
+ got = run(lambda Q, K: fast_rope_embedding(Q, K, cos, sin, None))
+ assert ref[4] == got[4]
+ for r, x in zip(ref[:4], got[:4]):
+ assert r.stride() == x.stride()
+ assert torch.equal(r.view(torch.int16), x.view(torch.int16))
diff --git a/tests/test_triton_fp_fusion.py b/tests/test_triton_fp_fusion.py
index 4413790175c..53b84be344a 100644
--- a/tests/test_triton_fp_fusion.py
+++ b/tests/test_triton_fp_fusion.py
@@ -263,7 +263,6 @@ def test_compiled_kernels_keep_fp_fusion(tmp_path, default):
expected = {
"rms_layernorm._rms_layernorm_forward",
"rms_layernorm._rms_layernorm_backward",
- "rope_embedding._rope_embedding",
"rope_embedding._rope_embedding_QK",
"cross_entropy_loss._cross_entropy_forward",
"cross_entropy_loss._chunked_cross_entropy_forward",
@@ -293,7 +292,7 @@ def test_compiled_kernels_keep_fp_fusion(tmp_path, default):
if not result["hip"]:
# Contraction really happened: fma.rn in PTX.
for name in (
- "rope_embedding._rope_embedding",
+ "rope_embedding._rope_embedding_QK",
"geglu._exact_forward_kernel",
"swiglu._DWf_DW_dfg_kernel",
):
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index 181367a6d3c..f7363566aba 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -17,7 +17,7 @@ import triton.language as tl
import torch
from ..device_type import DEVICE_COUNT, DEVICE_TYPE_TORCH
from typing import Tuple
-from .utils import calculate_settings, long_indexing, torch_gpu_device, torch_device_stream
+from .utils import long_indexing, next_power_of_2, torch_gpu_device, torch_device_stream
from .rms_layernorm import (
_BF16_TRACEABLE,
_TRACEABLE,
@@ -27,33 +27,120 @@ from .rms_layernorm import (
)
+def _rope_heads_tile(
+ X,
+ OUT,
+ x_batch_stride,
+ x_seq_stride,
+ x_head_stride,
+ x_dim_stride,
+ out_batch_stride,
+ out_seq_stride,
+ out_head_stride,
+ out_dim_stride,
+ batch_id,
+ seq_index,
+ head_start,
+ n_heads,
+ cos1,
+ sin1,
+ half_head_dim: tl.constexpr,
+ HEADS: tl.constexpr,
+ BLOCK_SIZE: tl.constexpr,
+ CAST_TO_SIN: tl.constexpr,
+ LONG_INDEXING: tl.constexpr,
+ BACKWARD_PASS: tl.constexpr,
+):
+ # Rotates HEADS heads of one (batch, seq) row from X into OUT (OUT may be X: each element is
+ # read before it is written by the same program).
+ heads = head_start + tl.arange(0, HEADS)
+ cols = tl.arange(0, BLOCK_SIZE)
+ mask = (heads[:, None] < n_heads) & (cols[None, :] < half_head_dim)
+ if LONG_INDEXING:
+ heads = heads.to(tl.int64)
+ x_ptr = (
+ X
+ + batch_id * x_batch_stride
+ + seq_index * x_seq_stride
+ + heads[:, None] * x_head_stride
+ + cols[None, :] * x_dim_stride
+ )
+ x0 = tl.load(x_ptr, mask = mask, other = 0)
+ x1 = tl.load(x_ptr + half_head_dim * x_dim_stride, mask = mask, other = 0)
+ if CAST_TO_SIN:
+ # Gemma sometimes needs RoPE done in float32 rather than bfloat16.
+ x0 = x0.to(sin1.dtype)
+ x1 = x1.to(sin1.dtype)
+ out_ptr = (
+ OUT
+ + batch_id * out_batch_stride
+ + seq_index * out_seq_stride
+ + heads[:, None] * out_head_stride
+ + cols[None, :] * out_dim_stride
+ )
+ c = cos1[None, :]
+ s = sin1[None, :]
+ # x0 * c - x1 * s and x1 * c + x0 * s with the contraction pinned: left to the backend it
+ # changes with the tile shape. These are the fma forms the one-head-per-program kernels got.
+ out0 = tl.fma(x0, c, -(x1 * s))
+ if BACKWARD_PASS:
+ out1 = tl.fma(x1, c, x0 * s)
+ else:
+ out1 = tl.fma(x0, s, x1 * c)
+ tl.store(out_ptr, out0, mask = mask)
+ tl.store(out_ptr + half_head_dim * out_dim_stride, out1, mask = mask)
+
+
def _rope_embedding_QK(
Q,
- Q_batch_stride, # s*h*d: varies with seq_len
- Q_head_stride, # d fwd, s*d bwd
- Q_seq_stride: tl.constexpr, # h*d fwd, d bwd
+ Q_out,
K,
- K_batch_stride,
- K_head_stride,
- K_seq_stride: tl.constexpr, # kv*d fwd, d bwd
+ K_out,
+ q_batch_stride,
+ q_seq_stride,
+ q_head_stride,
+ q_dim_stride,
+ qo_batch_stride,
+ qo_seq_stride,
+ qo_head_stride,
+ qo_dim_stride,
+ k_batch_stride,
+ k_seq_stride,
+ k_head_stride,
+ k_dim_stride,
+ ko_batch_stride,
+ ko_seq_stride,
+ ko_head_stride,
+ ko_dim_stride,
cos,
- cos_row_stride: tl.constexpr, # d
+ cos_row_stride,
sin,
- sin_row_stride: tl.constexpr, # d
+ sin_row_stride,
rope_embedding_indices,
seqlen,
- head_dim: tl.constexpr,
- n_heads_K: tl.constexpr,
+ n_heads_Q,
+ n_heads_K,
+ n_groups_Q,
+ half_head_dim: tl.constexpr,
BACKWARD_PASS: tl.constexpr,
HAS_ROPE_INDICES: tl.constexpr,
LONG_INDEXING: tl.constexpr,
+ CAST_TO_SIN: tl.constexpr,
+ HEADS_Q: tl.constexpr,
+ HEADS_K: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
EVICT_INDICES: tl.constexpr,
):
+ """
+ RoPE is Q * cos + rotate_half(Q) * sin; the backward is the same with -sin.
+ One program = one (batch, seq) row x a group of HEADS_Q query heads or HEADS_K key heads,
+ BLOCK_SIZE lanes over half of head_dim, read from any strides and written out of place
+ (or in place when Q_out is Q).
+ """
row_position = tl.program_id(0)
- head_position = tl.program_id(1)
+ group = tl.program_id(1)
col_offsets = tl.arange(0, BLOCK_SIZE)
- half_head_dim = head_dim // 2
mask = col_offsets < half_head_dim
if HAS_ROPE_INDICES:
@@ -67,18 +154,8 @@ def _rope_embedding_QK(
else:
rot_position = row_position % seqlen
- cos_ptr = cos + rot_position * cos_row_stride
- sin_ptr = sin + rot_position * sin_row_stride
- sin1 = tl.load(
- sin_ptr + col_offsets,
- mask = mask,
- other = 0,
- )
- cos1 = tl.load(
- cos_ptr + col_offsets,
- mask = mask,
- other = 0,
- )
+ sin1 = tl.load(sin + rot_position * sin_row_stride + col_offsets, mask = mask, other = 0)
+ cos1 = tl.load(cos + rot_position * cos_row_stride + col_offsets, mask = mask, other = 0)
if BACKWARD_PASS:
sin1 = -sin1
@@ -88,170 +165,234 @@ def _rope_embedding_QK(
# Widen after the int32 div/mod: 64-bit div/mod is slow at normal sizes.
batch_id = batch_id.to(tl.int64)
seq_index = seq_index.to(tl.int64)
- head_position = head_position.to(tl.int64)
-
- q_ptr = Q + batch_id * Q_batch_stride + head_position * Q_head_stride + seq_index * Q_seq_stride
- q0 = tl.load(q_ptr + col_offsets, mask = mask, other = 0)
- q1 = tl.load(q_ptr + half_head_dim + col_offsets, mask = mask, other = 0)
- tl.store(q_ptr + col_offsets, q0 * cos1 - q1 * sin1, mask = mask)
- tl.store(q_ptr + half_head_dim + col_offsets, q1 * cos1 + q0 * sin1, mask = mask)
- if head_position < n_heads_K:
- k_ptr = (
- K + batch_id * K_batch_stride + head_position * K_head_stride + seq_index * K_seq_stride
+ if group < n_groups_Q:
+ _rope_heads_tile(
+ Q,
+ Q_out,
+ q_batch_stride,
+ q_seq_stride,
+ q_head_stride,
+ q_dim_stride,
+ qo_batch_stride,
+ qo_seq_stride,
+ qo_head_stride,
+ qo_dim_stride,
+ batch_id,
+ seq_index,
+ group * HEADS_Q,
+ n_heads_Q,
+ cos1,
+ sin1,
+ half_head_dim,
+ HEADS_Q,
+ BLOCK_SIZE,
+ CAST_TO_SIN,
+ LONG_INDEXING,
+ BACKWARD_PASS,
+ )
+ else:
+ _rope_heads_tile(
+ K,
+ K_out,
+ k_batch_stride,
+ k_seq_stride,
+ k_head_stride,
+ k_dim_stride,
+ ko_batch_stride,
+ ko_seq_stride,
+ ko_head_stride,
+ ko_dim_stride,
+ batch_id,
+ seq_index,
+ (group - n_groups_Q) * HEADS_K,
+ n_heads_K,
+ cos1,
+ sin1,
+ half_head_dim,
+ HEADS_K,
+ BLOCK_SIZE,
+ CAST_TO_SIN,
+ LONG_INDEXING,
+ BACKWARD_PASS,
)
- k0 = tl.load(k_ptr + col_offsets, mask = mask, other = 0)
- k1 = tl.load(k_ptr + half_head_dim + col_offsets, mask = mask, other = 0)
- tl.store(k_ptr + col_offsets, k0 * cos1 - k1 * sin1, mask = mask)
- tl.store(k_ptr + half_head_dim + col_offsets, k1 * cos1 + k0 * sin1, mask = mask)
+# No triton.heuristics wrapper: the launcher passes real bools, and the wrapper's per-call cost
+# showed up at small (decode / short-sequence) shapes.
_rope_embedding_QK = triton.jit(_rope_embedding_QK)
-_rope_embedding_QK = triton.heuristics(
- {
- "BACKWARD_PASS": lambda args: bool(args["BACKWARD_PASS"]),
- "HAS_ROPE_INDICES": lambda args: bool(args["HAS_ROPE_INDICES"]),
- "LONG_INDEXING": lambda args: bool(args["LONG_INDEXING"]),
- }
-)(_rope_embedding_QK)
+# Elements of one half-head tile a program rotates: several heads per program instead of one
+# program per (row, head) with half its lanes masked off.
+ROPE_TILE_ELEMENTS: int = 1024
+# None picks per GPU: 8 warps on pre-Ampere CUDA (T4 measured ~5% faster at Llama-8B shapes), else 4
+# (A100 / RTX PRO 6000 are flat between 2 and 8, a little slower at 8).
+ROPE_NUM_WARPS = None
-ROPE_GROUP_SIZE: int = 4
[email protected]_cache(maxsize = None)
+def _rope_num_warps_at(device_type, index):
+ if device_type == "cuda" and torch.version.hip is None:
+ if torch.cuda.get_device_capability(index)[0] < 8:
+ return 8
+ return 4
-def _rope_embedding(
- Q,
- Q_row_stride: tl.constexpr,
- cos,
- cos_row_stride: tl.constexpr,
- sin,
- sin_row_stride: tl.constexpr,
- seqlen,
- head_dim: tl.constexpr,
- n_heads: tl.constexpr,
- BACKWARD_PASS: tl.constexpr,
- LONG_INDEXING: tl.constexpr,
- BLOCK_SIZE: tl.constexpr,
-):
- """
- Calculates the RoPE Embedding quickly
- RoPE is Q * cos + rotate_half(Q) * sin
- See our blog post for more info
- """
- ROPE_GROUP_SIZE = 4
- row_position = tl.program_id(0)
- group_head_position = tl.program_id(1)
- col_offsets = tl.arange(0, BLOCK_SIZE)
- half_head_dim = head_dim // 2
- mask = col_offsets < half_head_dim
- sin1 = tl.load(
- sin + (row_position % seqlen) * sin_row_stride + half_head_dim * 0 + col_offsets,
- mask = mask,
- other = 0,
- )
- cos1 = tl.load(
- cos + (row_position % seqlen) * cos_row_stride + half_head_dim * 0 + col_offsets,
- mask = mask,
- other = 0,
- )
+def _rope_num_warps(device):
+ if ROPE_NUM_WARPS is not None:
+ return ROPE_NUM_WARPS
+ index = device.index
+ if index is None and device.type == "cuda":
+ index = torch.cuda.current_device()
+ return _rope_num_warps_at(device.type, index)
- if BACKWARD_PASS:
- # See the Unsloth blog post for more info.
- sin1 = -sin1
- # [TODO] Autotune ROPE_GROUP_SIZE to be 1, 2, 4, 8
- head_start = group_head_position * ROPE_GROUP_SIZE
- head_end = min((head_start + ROPE_GROUP_SIZE), n_heads)
+def _heads_per_program(n_heads, block_size):
+ return max(1, min(next_power_of_2(n_heads), ROPE_TILE_ELEMENTS // block_size))
- # 10% faster kernel from HuyNguyen-hust, unslothai/unsloth#238.
- if LONG_INDEXING:
- Q += row_position.to(tl.int64) * Q_row_stride
- else:
- Q += row_position * Q_row_stride
- for k in range(head_start, head_end):
- offs_q1 = k * head_dim + col_offsets
- offs_q2 = k * head_dim + col_offsets + half_head_dim
- # Gemma sometimes needs RoPE done in float32 rather than bfloat16.
- Q1 = tl.load(Q + offs_q1, mask = mask, other = 0).to(sin1.dtype)
- Q2 = tl.load(Q + offs_q2, mask = mask, other = 0).to(sin1.dtype)
-
- tl.store(Q + offs_q1, Q1 * cos1 - Q2 * sin1, mask = mask)
- tl.store(Q + offs_q2, Q2 * cos1 + Q1 * sin1, mask = mask)
-
-
-_rope_embedding = triton.jit(_rope_embedding)
-_rope_embedding = triton.heuristics(
- {
- "BACKWARD_PASS": lambda args: bool(args["BACKWARD_PASS"]),
- "LONG_INDEXING": lambda args: bool(args["LONG_INDEXING"]),
- }
-)(_rope_embedding)
-
-
-def _rope_rows(Q, cos, sin, seq_len, n_heads, head_dim, backward, wrap):
- # Rotates Q [batch * seq_len, n_heads * head_dim] in place; BLOCK_SIZE = head_dim//2 was racy.
- BLOCK_SIZE, num_warps = calculate_settings(head_dim // 2)
- n_groups = (
- n_heads + ROPE_GROUP_SIZE - 1
- ) // ROPE_GROUP_SIZE # also a SymInt under dynamic=True
- wrap(_rope_embedding)[
- (
- Q.shape[0],
- n_groups,
- )
- ](
+def _rope_launch(Q, Q_out, K, K_out, cos, sin, rope_ptr, has_indices, backward, cast_to_sin, wrap):
+ # Q, K, Q_out, K_out: [batch, n_heads, seq_len, head_dim] at any strides (K may be None).
+ # Q_out may be Q itself (in place).
+ batch, n_heads_Q, seq_len, head_dim = Q.shape
+ half_head_dim = head_dim // 2
+ BLOCK_SIZE = next_power_of_2(half_head_dim)
+ HEADS_Q = _heads_per_program(n_heads_Q, BLOCK_SIZE)
+ n_groups_Q = (n_heads_Q + HEADS_Q - 1) // HEADS_Q # also a SymInt under dynamic=True
+ if K is None:
+ # No key heads: a dummy, never read or written. Not Q / Q_out, since a traced kernel would
+ # see Q_out mutated through two arguments.
+ dummy = Q.new_empty((1, 1, 1, 1))
+ K, K_out, n_heads_K, HEADS_K, n_groups_K = dummy, dummy, 0, 1, 0
+ else:
+ n_heads_K = K.shape[1]
+ HEADS_K = _heads_per_program(n_heads_K, BLOCK_SIZE)
+ n_groups_K = (n_heads_K + HEADS_K - 1) // HEADS_K
+ if rope_ptr is None:
+ rope_ptr = cos # never read without HAS_ROPE_INDICES
+ tensors = (Q, Q_out, K, K_out)
+ wrap(_rope_embedding_QK)[(batch * seq_len, n_groups_Q + n_groups_K)](
Q,
+ Q_out,
+ K,
+ K_out,
Q.stride(0),
+ Q.stride(2),
+ Q.stride(1),
+ Q.stride(3),
+ Q_out.stride(0),
+ Q_out.stride(2),
+ Q_out.stride(1),
+ Q_out.stride(3),
+ K.stride(0),
+ K.stride(2),
+ K.stride(1),
+ K.stride(3),
+ K_out.stride(0),
+ K_out.stride(2),
+ K_out.stride(1),
+ K_out.stride(3),
cos,
cos.stride(0),
sin,
sin.stride(0),
+ rope_ptr,
seq_len,
- head_dim,
- n_heads,
- BACKWARD_PASS = backward,
- LONG_INDEXING = long_indexing(Q, block = BLOCK_SIZE),
+ n_heads_Q,
+ n_heads_K,
+ n_groups_Q,
+ half_head_dim = half_head_dim,
+ BACKWARD_PASS = bool(backward),
+ HAS_ROPE_INDICES = bool(has_indices),
+ LONG_INDEXING = long_indexing(*tensors, block = 2 * BLOCK_SIZE * max(HEADS_Q, HEADS_K)),
+ CAST_TO_SIN = bool(cast_to_sin),
+ HEADS_Q = HEADS_Q,
+ HEADS_K = HEADS_K,
BLOCK_SIZE = BLOCK_SIZE,
- num_warps = num_warps,
+ EVICT_INDICES = _eviction_hints_ok(Q.device) if has_indices else False,
+ num_warps = _rope_num_warps(Q.device),
enable_fp_fusion = True,
)
+def _rope_rows(Q, Q_out, cos, sin, backward, wrap):
+ # Q, Q_out: [batch, seq_len, n_heads, head_dim] at any strides; Q_out may be Q (in place).
+ # Rows cast Q to sin's dtype (the QK path does not), kept for bitwise parity with before.
+ _rope_launch(
+ Q.transpose(1, 2),
+ Q_out.transpose(1, 2),
+ None,
+ None,
+ cos,
+ sin,
+ None,
+ False,
+ backward,
+ True,
+ wrap,
+ )
+
+
+def _bshd_buffer(X):
+ # A fresh [batch, n_heads, seq_len, head_dim] view of a contiguous [batch, seq_len, n_heads,
+ # head_dim] buffer: the layout Q.transpose(1, 2).contiguous().transpose(1, 2) produced.
+ batch, n_heads, seq_len, head_dim = X.shape
+ return torch.empty(
+ (batch, seq_len, n_heads, head_dim), dtype = X.dtype, device = X.device
+ ).transpose(1, 2)
+
+
+class Fast_RoPE_Embedding_QK_Positions(torch.autograd.Function):
+ """Fast_RoPE_Embedding on Q and K ([batch, n_heads, seq_len, head_dim]) in one launch each way.
+
+ Same bytes, layouts and aliasing as Fast_RoPE_Embedding.apply(X.transpose(1, 2).contiguous(),
+ cos, sin).transpose(1, 2) for X in (Q, K): in place where X.transpose(1, 2) is contiguous,
+ else into a fresh [batch, seq_len, n_heads, head_dim] buffer; gradients in that layout too."""
+
+ @staticmethod
+ def forward(ctx, Q, K, cos, sin):
+ cos, sin = cos.squeeze(), sin.squeeze()
+ assert Q.shape[2] <= cos.shape[0]
+ Q_out = Q if Q.transpose(1, 2).is_contiguous() else _bshd_buffer(Q)
+ K_out = K if K.transpose(1, 2).is_contiguous() else _bshd_buffer(K)
+ with torch_gpu_device(Q.device):
+ _rope_launch(Q, Q_out, K, K_out, cos, sin, None, False, False, True, _eager_kernel)
+ ctx.cos = cos
+ ctx.sin = sin
+ return Q_out.view(Q_out.shape), K_out.view(K_out.shape)
+
+ @staticmethod
+ def backward(ctx, dQ, dK):
+ dQ_out, dK_out = _bshd_buffer(dQ), _bshd_buffer(dK)
+ with torch_gpu_device(dQ.device):
+ _rope_launch(
+ dQ, dQ_out, dK, dK_out, ctx.cos, ctx.sin, None, False, True, True, _eager_kernel
+ )
+ return dQ_out, dK_out, None, None
+
+
class Fast_RoPE_Embedding(torch.autograd.Function):
@staticmethod
def forward(ctx, Q, cos, sin):
cos, sin = cos.squeeze(), sin.squeeze()
- batch: int
- seq_len: int
- n_heads: int
- head_dim: int
- batch, seq_len, n_heads, head_dim = Q.shape
- Q = Q.reshape(batch * seq_len, n_heads * head_dim)
- assert seq_len <= cos.shape[0]
+ assert Q.shape[1] <= cos.shape[0]
+ # In place on contiguous Q (callers pass Q.transpose(1, 2).contiguous()), as before;
+ # a strided Q is written to a fresh contiguous tensor instead of being copied first.
+ Q_out = Q if Q.is_contiguous() else torch.empty(Q.shape, dtype = Q.dtype, device = Q.device)
with torch_gpu_device(Q.device):
- _rope_rows(Q, cos, sin, seq_len, n_heads, head_dim, False, _eager_kernel)
+ _rope_rows(Q, Q_out, cos, sin, False, _eager_kernel)
ctx.cos = cos
ctx.sin = sin
- return Q.reshape(batch, seq_len, n_heads, head_dim)
+ return Q_out.view(Q_out.shape)
@staticmethod
def backward(ctx, dY):
- batch: int
- seq_len: int
- n_heads: int
- head_dim: int
- batch, seq_len, n_heads, head_dim = dY.shape
- # The in-place kernel needs a private contiguous buffer, even for already-contiguous grads.
- dY = dY.clone(memory_format = torch.contiguous_format).view(
- batch * seq_len, n_heads * head_dim
- )
+ # Out of place into a fresh contiguous buffer: dY may be shared or stride-0 (expanded).
+ dQ = torch.empty(dY.shape, dtype = dY.dtype, device = dY.device)
with torch_gpu_device(dY.device):
- _rope_rows(dY, ctx.cos, ctx.sin, seq_len, n_heads, head_dim, True, _eager_kernel)
- dY = dY.reshape(batch, seq_len, n_heads, head_dim)
+ _rope_rows(dY, dQ, ctx.cos, ctx.sin, True, _eager_kernel)
return (
- dY,
+ dQ,
None,
None,
)
@@ -270,36 +411,9 @@ def _eviction_hints_ok(device):
return _eviction_hints_ok_at(device.type, index)
-def _rope_qk(Q, K, cos, sin, rope_ptr, has_indices, backward, wrap):
- # Rotates Q [batch, n_heads_Q, seq_len, head_dim] and K in place, at any strides.
- batch, n_heads_Q, seq_len, head_dim = Q.shape
- n_heads_K = K.shape[1]
- BLOCK_SIZE, num_warps = calculate_settings(head_dim)
- wrap(_rope_embedding_QK)[(batch * seq_len, n_heads_Q)](
- Q,
- Q.stride(0),
- Q.stride(1),
- Q.stride(2),
- K,
- K.stride(0),
- K.stride(1),
- K.stride(2),
- cos,
- cos.stride(0),
- sin,
- sin.stride(0),
- rope_ptr,
- seq_len,
- head_dim = head_dim,
- n_heads_K = n_heads_K,
- BACKWARD_PASS = backward,
- HAS_ROPE_INDICES = has_indices,
- LONG_INDEXING = long_indexing(Q, K, block = BLOCK_SIZE),
- BLOCK_SIZE = BLOCK_SIZE,
- EVICT_INDICES = _eviction_hints_ok(Q.device),
- num_warps = num_warps,
- enable_fp_fusion = True,
- )
+def _rope_qk(Q, K, Q_out, K_out, cos, sin, rope_ptr, has_indices, backward, wrap):
+ # Rotates Q [batch, n_heads_Q, seq_len, head_dim] and K into Q_out / K_out, at any strides.
+ _rope_launch(Q, Q_out, K, K_out, cos, sin, rope_ptr, has_indices, backward, False, wrap)
class Fast_RoPE_Embedding_QK(torch.autograd.Function):
@@ -308,18 +422,19 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
has_indices = rope_indices is not None
cos, sin = cos.squeeze(), sin.squeeze()
- # Inplace rotary embedding is generally fine.
- Q_out = Q.clone() if not Q.is_contiguous() else Q
- K_out = K.clone() if not K.is_contiguous() else K
+ # Contiguous inputs rotate in place; strided ones (transposed views of the projections)
+ # are read where they are and written to a fresh tensor of the same layout.
+ Q_out = Q if Q.is_contiguous() else torch.empty_like(Q)
+ K_out = K if K.is_contiguous() else torch.empty_like(K)
if has_indices:
# TRL's rotary indices are always int32, so the cast is only for safety.
rope_ptr = rope_indices.reshape(-1).to(dtype = torch.int32, device = Q.device)
else:
- rope_ptr = cos.new_empty(1, dtype = torch.int32)
+ rope_ptr = None
with torch_gpu_device(Q.device):
- _rope_qk(Q_out, K_out, cos, sin, rope_ptr, has_indices, False, _eager_kernel)
+ _rope_qk(Q, K, Q_out, K_out, cos, sin, rope_ptr, has_indices, False, _eager_kernel)
ctx.has_indices = has_indices
ctx.cos = cos
@@ -333,21 +448,31 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
@staticmethod
def backward(ctx, dQ, dK):
- rope_ptr = ctx.rope_indices if ctx.has_indices else ctx.cos.new_empty(1, dtype = torch.int32)
+ rope_ptr = ctx.rope_indices
- # Contiguous gradients can still share storage with each other or another branch.
- dQ_out = dQ.clone(memory_format = torch.contiguous_format)
- dK_out = dK.clone(memory_format = torch.contiguous_format)
+ # Out of place: gradients can share storage with each other or another branch.
+ dQ_out = torch.empty(dQ.shape, dtype = dQ.dtype, device = dQ.device)
+ dK_out = torch.empty(dK.shape, dtype = dK.dtype, device = dK.device)
with torch_gpu_device(dQ.device):
_rope_qk(
- dQ_out, dK_out, ctx.cos, ctx.sin, rope_ptr, ctx.has_indices, True, _eager_kernel
+ dQ,
+ dK,
+ dQ_out,
+ dK_out,
+ ctx.cos,
+ ctx.sin,
+ rope_ptr,
+ ctx.has_indices,
+ True,
+ _eager_kernel,
)
return (dQ_out, dK_out, None, None, None)
-# Compiled graphs take these ops: the same kernels on fresh copies (the Functions rotate in place).
+# Compiled graphs take these ops: the same kernel, written out of place (the Functions may rotate
+# in place).
if _TRACEABLE:
@torch.library.triton_op("unsloth::rope_embedding", mutates_args = ())
@@ -355,10 +480,8 @@ if _TRACEABLE:
Q: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, backward: bool
) -> torch.Tensor:
# Q: [batch, seq_len, n_heads, head_dim] at any strides.
- batch, seq_len, n_heads, head_dim = Q.shape
- out = Q.clone(memory_format = torch.contiguous_format)
- rows = out.view(batch * seq_len, n_heads * head_dim)
- _rope_rows(rows, cos, sin, seq_len, n_heads, head_dim, backward, _traced_kernel)
+ out = torch.empty(Q.shape, dtype = Q.dtype, device = Q.device)
+ _rope_rows(Q, out, cos, sin, backward, _traced_kernel)
return out
def _rope_setup_context(ctx, inputs, output):
@@ -381,8 +504,8 @@ if _TRACEABLE:
rope_indices: torch.Tensor,
backward: bool,
) -> Tuple[torch.Tensor, torch.Tensor]:
- Q_out, K_out = Q.clone(), K.clone()
- _rope_qk(Q_out, K_out, cos, sin, rope_indices, True, backward, _traced_kernel)
+ Q_out, K_out = torch.empty_like(Q), torch.empty_like(K)
+ _rope_qk(Q, K, Q_out, K_out, cos, sin, rope_indices, True, backward, _traced_kernel)
return Q_out, K_out
def _rope_qk_setup_context(ctx, inputs, output):
@@ -503,8 +626,8 @@ def _fast_rope_embedding_eager(Q, K, cos, sin, rope_embedding_indices):
if rope_embedding_indices is not None:
Q_out, K_out = Fast_RoPE_Embedding_QK.apply(Q, K, cos, sin, rope_embedding_indices)
else:
- Q_out = Fast_RoPE_Embedding.apply(Q.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)
- K_out = Fast_RoPE_Embedding.apply(K.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)
+ # Fast_RoPE_Embedding on Q and K, fused into one launch (forward and backward).
+ Q_out, K_out = Fast_RoPE_Embedding_QK_Positions.apply(Q, K, cos, sin)
_sync_if_split(Q)
return Q_out, K_out