Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
RoPE: multi-head tiles, Q and K in one launch, no clones (2-2.7x per call) (#14417)
Daniel Han20e8179bfb0e
diff --git a/tests/test_rope_kernel_layouts.py b/tests/test_rope_kernel_layouts.pynew file mode 100644index 00000000000..ee883d195fe--- /dev/null+++ b/tests/test_rope_kernel_layouts.py@@ -0,0 +1,234 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""fast_rope_embedding against an fp64 oracle over head dims, head counts, GQA, input layouts,+packed position ids, dtypes and gradient strides, plus the in-place contract callers see."""++import pytest+import torch++pytestmark = [+ pytest.mark.gpu,+ pytest.mark.skipif(not torch.cuda.is_available(), reason = "Triton RoPE kernel needs CUDA"),+]++BF16 = torch.bfloat16+# (batch, seq, heads, kv heads, head_dim): pow2 and not (80, 96), 1 to 40 heads, GQA, decode.+SHAPES = [+ (2, 37, 8, 8, 64),+ (1, 64, 32, 8, 128),+ (3, 20, 4, 1, 64),+ (2, 33, 40, 8, 128),+ (1, 17, 6, 2, 80),+ (2, 5, 12, 12, 96),+ (2, 9, 16, 4, 256),+ (1, 1, 32, 8, 128),+ (4, 3, 1, 1, 32),+]+++def _cos_sin(rows, head_dim, dtype):+ inv = 1.0 / (10000 ** (torch.arange(0, head_dim, 2, device = "cuda").float() / head_dim))+ freqs = torch.outer(torch.arange(rows, device = "cuda").float(), inv)+ emb = torch.cat((freqs, freqs), -1)+ return emb.cos().to(dtype), emb.sin().to(dtype)+++def _packed_ids(batch, seq):+ # Packed sequences restart at 0; rows differ per batch.+ lens = [max(1, seq // 3), max(1, seq // 3)]+ if seq - sum(lens) > 0:+ lens.append(seq - sum(lens))+ row = torch.cat([torch.arange(n, dtype = torch.int32) for n in lens])[:seq]+ return torch.stack([row + b for b in range(batch)]).cuda()+++def _oracle(+ x,+ cos,+ sin,+ ids,+ backward = False,+):+ # x [b, h, s, d]; fp64 rotation of the exact low-precision inputs.+ x, cos, sin = x.double(), cos.double(), sin.double()+ seq = x.shape[2]+ if ids is None:+ c, s = cos[:seq], sin[:seq]+ else:+ c, s = cos[ids.long()].unsqueeze(1), sin[ids.long()].unsqueeze(1)+ half = x.shape[-1] // 2+ x0, x1 = x[..., :half], x[..., half:]+ c, s = c[..., :half], s[..., :half]+ if backward:+ s = -s+ out = torch.cat((x0 * c - x1 * s, x1 * c + x0 * s), -1)+ # Magnitude of the terms, for a rounding bound that survives cancellation.+ scale = torch.cat(((x0 * c).abs() + (x1 * s).abs(), (x1 * c).abs() + (x0 * s).abs()), -1)+ return out, scale+++def _assert_rounding_close(got, ref, scale, dtype):+ # fma(a, b, -round(c * d)) then a store: at most ~1 ulp of the terms involved.+ eps = torch.finfo(dtype).eps+ err = (got.double() - ref).abs()+ bound = eps * (scale + ref.abs()) + torch.finfo(dtype).tiny+ worst = (err / bound).max().item()+ assert worst <= 1.0, f"error {worst:.3f}x the rounding bound"+++def _inputs(shape, dtype, layout, g):+ b, s, h, kv, d = shape+ if layout == "projection":+ # As the attention modules build them: [b, h, s, d] views of [b, s, h * d] projections.+ q_lin = torch.randn(b, s, h * d, device = "cuda", generator = g).to(dtype)+ k_lin = torch.randn(b, s, kv * d, device = "cuda", generator = g).to(dtype)+ q_lin.requires_grad_(True)+ k_lin.requires_grad_(True)+ Q = (q_lin * 1).view(b, s, h, d).transpose(1, 2)+ K = (k_lin * 1).view(b, s, kv, d).transpose(1, 2)+ to_bhsd = lambda t, n: t.view(b, s, n, d).transpose(1, 2)+ else:+ q_lin = torch.randn(b, h, s, d, device = "cuda", generator = g).to(dtype).requires_grad_(True)+ k_lin = torch.randn(b, kv, s, d, device = "cuda", generator = g).to(dtype).requires_grad_(True)+ Q, K = q_lin * 1, k_lin * 1+ to_bhsd = lambda t, n: t+ return q_lin, k_lin, Q, K, to_bhsd++[email protected]("grad_kind", ["dense", "transposed", "expanded"])[email protected]("layout", ["projection", "contiguous"])[email protected]("with_ids", [False, True], ids = ["positions", "position_ids"])[email protected]("dtype", [torch.float16, BF16, torch.float32], ids = str)[email protected]("shape", SHAPES, ids = lambda s: "b{}_s{}_h{}_kv{}_d{}".format(*s))+def test_rope_matches_fp64_oracle(shape, dtype, with_ids, layout, grad_kind):+ from unsloth.kernels.rope_embedding import fast_rope_embedding++ b, s, h, kv, d = shape+ g = torch.Generator(device = "cuda").manual_seed(sum(shape))+ cos, sin = _cos_sin(s + 11, d, dtype)+ ids = _packed_ids(b, s) if with_ids else None+ q_lin, k_lin, Q, K, to_bhsd = _inputs(shape, dtype, layout, g)+ q_in, k_in = Q.detach().clone(), K.detach().clone()++ q, k = fast_rope_embedding(Q, K, cos, sin, ids)+ assert q.shape == (b, h, s, d) and k.shape == (b, kv, s, d)+ for got, x in ((q, q_in), (k, k_in)):+ ref, scale = _oracle(x, cos, sin, ids)+ _assert_rounding_close(got, ref, scale, dtype)++ if grad_kind == "dense":+ dq = torch.randn(q.shape, device = "cuda", generator = g).to(dtype)+ dk = torch.randn(k.shape, device = "cuda", generator = g).to(dtype)+ elif grad_kind == "transposed":+ dq = torch.randn(b, s, h, d, device = "cuda", generator = g).to(dtype).transpose(1, 2)+ dk = torch.randn(b, s, kv, d, device = "cuda", generator = g).to(dtype).transpose(1, 2)+ else:+ dq = torch.randn(1, 1, 1, d, device = "cuda", generator = g).to(dtype).expand(q.shape)+ dk = torch.randn(1, 1, 1, d, device = "cuda", generator = g).to(dtype).expand(k.shape)+ dq_before, dk_before = dq.clone(), dk.clone()+ torch.autograd.backward((q, k), (dq, dk))+ assert torch.equal(dq, dq_before) and torch.equal(dk, dk_before), "upstream grad was mutated"+ for grad, n, up in ((q_lin.grad, h, dq), (k_lin.grad, kv, dk)):+ ref, scale = _oracle(up, cos, sin, ids, backward = True)+ _assert_rounding_close(to_bhsd(grad, n), ref, scale, dtype)+++def test_rope_float32_tables_with_half_inputs():+ """Gemma-style fp32 cos / sin with bf16 Q / K computes in fp32 and stores bf16."""+ from unsloth.kernels.rope_embedding import fast_rope_embedding++ shape = (2, 19, 8, 2, 128)+ g = torch.Generator(device = "cuda").manual_seed(3)+ cos, sin = _cos_sin(32, 128, torch.float32)+ for ids in (None, _packed_ids(2, 19)):+ _, _, Q, K, _ = _inputs(shape, BF16, "projection", g)+ q_in, k_in = Q.detach().clone(), K.detach().clone()+ q, k = fast_rope_embedding(Q, K, cos, sin, ids)+ assert q.dtype == BF16 and k.dtype == BF16+ for got, x in ((q, q_in), (k, k_in)):+ ref, scale = _oracle(x, cos, sin, ids)+ _assert_rounding_close(got, ref, scale, BF16)++[email protected]("with_ids", [False, True], ids = ["positions", "position_ids"])+def test_rope_output_layout_and_aliasing(with_ids):+ """Layouts and aliasing the attention modules see: outputs keep the projection layout, a+ strided position-id input is not mutated, gradients come back contiguous."""+ from unsloth.kernels.rope_embedding import fast_rope_embedding++ b, s, h, kv, d = 2, 24, 8, 2, 64+ g = torch.Generator(device = "cuda").manual_seed(5)+ cos, sin = _cos_sin(32, d, torch.float16)+ ids = _packed_ids(b, s) if with_ids else None+ q_lin, k_lin, Q, K, _ = _inputs((b, s, h, kv, d), torch.float16, "projection", g)+ Q_before = Q.detach().clone()+ q, k = fast_rope_embedding(Q, K, cos, sin, ids)+ assert q.stride() == Q.stride() and k.stride() == K.stride()+ if with_ids:+ assert torch.equal(Q.detach(), Q_before), "a strided input was rotated in place"+ else:+ # The position path rotates the [b, s, h, d] buffer in place, as it always has.+ assert q.data_ptr() == Q.data_ptr()+ torch.autograd.backward((q, k), (torch.ones_like(q), torch.ones_like(k)))+ assert q_lin.grad.is_contiguous() and k_lin.grad.is_contiguous()++[email protected]("with_ids", [False, True], ids = ["positions", "position_ids"])+def test_rope_contiguous_qk_input_rotates_in_place(with_ids):+ """[b, h, s, d]-contiguous Q / K on the position-id path are rotated in place (no copy)."""+ from unsloth.kernels.rope_embedding import Fast_RoPE_Embedding_QK++ b, s, h, kv, d = 1, 8, 4, 2, 64+ cos, sin = _cos_sin(16, d, torch.float32)+ Q = torch.randn(b, h, s, d, device = "cuda")+ K = torch.randn(b, kv, s, d, device = "cuda")+ ids = _packed_ids(b, s)+ with torch.no_grad():+ q, k = Fast_RoPE_Embedding_QK.apply(Q, K, cos, sin, ids)+ assert q.data_ptr() == Q.data_ptr() and k.data_ptr() == K.data_ptr()+++def test_rope_heads_per_program_bounds():+ from unsloth.kernels.rope_embedding import ROPE_TILE_ELEMENTS, _heads_per_program+ for n_heads in (1, 2, 3, 7, 8, 32, 40, 64, 128):+ for block in (16, 32, 64, 128, 256, 1024, 2048):+ heads = _heads_per_program(n_heads, block)+ assert heads >= 1 and heads & (heads - 1) == 0+ assert heads <= max(1, ROPE_TILE_ELEMENTS // block)+ assert heads <= triton_next_pow2(n_heads)+++def triton_next_pow2(n):+ return 1 << (n - 1).bit_length()++[email protected]("layout", ["projection", "contiguous"])+def test_rope_positions_pair_matches_two_single_calls(layout):+ """The fused Q + K position path keeps the bytes, strides, aliasing and gradient layout of+ Fast_RoPE_Embedding applied to Q and K separately."""+ from unsloth.kernels.rope_embedding import Fast_RoPE_Embedding, fast_rope_embedding++ shape = (2, 21, 8, 2, 64)+ cos, sin = _cos_sin(32, 64, BF16)++ def run(fn):+ g = torch.Generator(device = "cuda").manual_seed(11)+ q_lin, k_lin, Q, K, _ = _inputs(shape, BF16, layout, g)+ q, k = fn(Q, K)+ aliased = (q.data_ptr() == Q.data_ptr(), k.data_ptr() == K.data_ptr())+ dq = torch.randn(q.shape, device = "cuda", generator = g).to(BF16)+ dk = torch.randn(k.shape, device = "cuda", generator = g).to(BF16)+ torch.autograd.backward((q, k), (dq, dk))+ return q, k, q_lin.grad, k_lin.grad, aliased++ def single(Q, K):+ q = Fast_RoPE_Embedding.apply(Q.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)+ k = Fast_RoPE_Embedding.apply(K.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)+ return q, k++ ref = run(single)+ got = run(lambda Q, K: fast_rope_embedding(Q, K, cos, sin, None))+ assert ref[4] == got[4]+ for r, x in zip(ref[:4], got[:4]):+ assert r.stride() == x.stride()+ assert torch.equal(r.view(torch.int16), x.view(torch.int16))diff --git a/tests/test_triton_fp_fusion.py b/tests/test_triton_fp_fusion.pyindex 4413790175c..53b84be344a 100644--- a/tests/test_triton_fp_fusion.py+++ b/tests/test_triton_fp_fusion.py@@ -263,7 +263,6 @@ def test_compiled_kernels_keep_fp_fusion(tmp_path, default):expected = {"rms_layernorm._rms_layernorm_forward","rms_layernorm._rms_layernorm_backward",- "rope_embedding._rope_embedding","rope_embedding._rope_embedding_QK","cross_entropy_loss._cross_entropy_forward","cross_entropy_loss._chunked_cross_entropy_forward",@@ -293,7 +292,7 @@ def test_compiled_kernels_keep_fp_fusion(tmp_path, default):if not result["hip"]:# Contraction really happened: fma.rn in PTX.for name in (- "rope_embedding._rope_embedding",+ "rope_embedding._rope_embedding_QK","geglu._exact_forward_kernel","swiglu._DWf_DW_dfg_kernel",):diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex 181367a6d3c..f7363566aba 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -17,7 +17,7 @@ import triton.language as tlimport torchfrom ..device_type import DEVICE_COUNT, DEVICE_TYPE_TORCHfrom typing import Tuple-from .utils import calculate_settings, long_indexing, torch_gpu_device, torch_device_stream+from .utils import long_indexing, next_power_of_2, torch_gpu_device, torch_device_streamfrom .rms_layernorm import (_BF16_TRACEABLE,_TRACEABLE,@@ -27,33 +27,120 @@ from .rms_layernorm import ()+def _rope_heads_tile(+ X,+ OUT,+ x_batch_stride,+ x_seq_stride,+ x_head_stride,+ x_dim_stride,+ out_batch_stride,+ out_seq_stride,+ out_head_stride,+ out_dim_stride,+ batch_id,+ seq_index,+ head_start,+ n_heads,+ cos1,+ sin1,+ half_head_dim: tl.constexpr,+ HEADS: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+ CAST_TO_SIN: tl.constexpr,+ LONG_INDEXING: tl.constexpr,+ BACKWARD_PASS: tl.constexpr,+):+ # Rotates HEADS heads of one (batch, seq) row from X into OUT (OUT may be X: each element is+ # read before it is written by the same program).+ heads = head_start + tl.arange(0, HEADS)+ cols = tl.arange(0, BLOCK_SIZE)+ mask = (heads[:, None] < n_heads) & (cols[None, :] < half_head_dim)+ if LONG_INDEXING:+ heads = heads.to(tl.int64)+ x_ptr = (+ X+ + batch_id * x_batch_stride+ + seq_index * x_seq_stride+ + heads[:, None] * x_head_stride+ + cols[None, :] * x_dim_stride+ )+ x0 = tl.load(x_ptr, mask = mask, other = 0)+ x1 = tl.load(x_ptr + half_head_dim * x_dim_stride, mask = mask, other = 0)+ if CAST_TO_SIN:+ # Gemma sometimes needs RoPE done in float32 rather than bfloat16.+ x0 = x0.to(sin1.dtype)+ x1 = x1.to(sin1.dtype)+ out_ptr = (+ OUT+ + batch_id * out_batch_stride+ + seq_index * out_seq_stride+ + heads[:, None] * out_head_stride+ + cols[None, :] * out_dim_stride+ )+ c = cos1[None, :]+ s = sin1[None, :]+ # x0 * c - x1 * s and x1 * c + x0 * s with the contraction pinned: left to the backend it+ # changes with the tile shape. These are the fma forms the one-head-per-program kernels got.+ out0 = tl.fma(x0, c, -(x1 * s))+ if BACKWARD_PASS:+ out1 = tl.fma(x1, c, x0 * s)+ else:+ out1 = tl.fma(x0, s, x1 * c)+ tl.store(out_ptr, out0, mask = mask)+ tl.store(out_ptr + half_head_dim * out_dim_stride, out1, mask = mask)++def _rope_embedding_QK(Q,- Q_batch_stride, # s*h*d: varies with seq_len- Q_head_stride, # d fwd, s*d bwd- Q_seq_stride: tl.constexpr, # h*d fwd, d bwd+ Q_out,K,- K_batch_stride,- K_head_stride,- K_seq_stride: tl.constexpr, # kv*d fwd, d bwd+ K_out,+ q_batch_stride,+ q_seq_stride,+ q_head_stride,+ q_dim_stride,+ qo_batch_stride,+ qo_seq_stride,+ qo_head_stride,+ qo_dim_stride,+ k_batch_stride,+ k_seq_stride,+ k_head_stride,+ k_dim_stride,+ ko_batch_stride,+ ko_seq_stride,+ ko_head_stride,+ ko_dim_stride,cos,- cos_row_stride: tl.constexpr, # d+ cos_row_stride,sin,- sin_row_stride: tl.constexpr, # d+ sin_row_stride,rope_embedding_indices,seqlen,- head_dim: tl.constexpr,- n_heads_K: tl.constexpr,+ n_heads_Q,+ n_heads_K,+ n_groups_Q,+ half_head_dim: tl.constexpr,BACKWARD_PASS: tl.constexpr,HAS_ROPE_INDICES: tl.constexpr,LONG_INDEXING: tl.constexpr,+ CAST_TO_SIN: tl.constexpr,+ HEADS_Q: tl.constexpr,+ HEADS_K: tl.constexpr,BLOCK_SIZE: tl.constexpr,EVICT_INDICES: tl.constexpr,):+ """+ RoPE is Q * cos + rotate_half(Q) * sin; the backward is the same with -sin.+ One program = one (batch, seq) row x a group of HEADS_Q query heads or HEADS_K key heads,+ BLOCK_SIZE lanes over half of head_dim, read from any strides and written out of place+ (or in place when Q_out is Q).+ """row_position = tl.program_id(0)- head_position = tl.program_id(1)+ group = tl.program_id(1)col_offsets = tl.arange(0, BLOCK_SIZE)- half_head_dim = head_dim // 2mask = col_offsets < half_head_dimif HAS_ROPE_INDICES:@@ -67,18 +154,8 @@ def _rope_embedding_QK(else:rot_position = row_position % seqlen- cos_ptr = cos + rot_position * cos_row_stride- sin_ptr = sin + rot_position * sin_row_stride- sin1 = tl.load(- sin_ptr + col_offsets,- mask = mask,- other = 0,- )- cos1 = tl.load(- cos_ptr + col_offsets,- mask = mask,- other = 0,- )+ sin1 = tl.load(sin + rot_position * sin_row_stride + col_offsets, mask = mask, other = 0)+ cos1 = tl.load(cos + rot_position * cos_row_stride + col_offsets, mask = mask, other = 0)if BACKWARD_PASS:sin1 = -sin1@@ -88,170 +165,234 @@ def _rope_embedding_QK(# Widen after the int32 div/mod: 64-bit div/mod is slow at normal sizes.batch_id = batch_id.to(tl.int64)seq_index = seq_index.to(tl.int64)- head_position = head_position.to(tl.int64)-- q_ptr = Q + batch_id * Q_batch_stride + head_position * Q_head_stride + seq_index * Q_seq_stride- q0 = tl.load(q_ptr + col_offsets, mask = mask, other = 0)- q1 = tl.load(q_ptr + half_head_dim + col_offsets, mask = mask, other = 0)- tl.store(q_ptr + col_offsets, q0 * cos1 - q1 * sin1, mask = mask)- tl.store(q_ptr + half_head_dim + col_offsets, q1 * cos1 + q0 * sin1, mask = mask)- if head_position < n_heads_K:- k_ptr = (- K + batch_id * K_batch_stride + head_position * K_head_stride + seq_index * K_seq_stride+ if group < n_groups_Q:+ _rope_heads_tile(+ Q,+ Q_out,+ q_batch_stride,+ q_seq_stride,+ q_head_stride,+ q_dim_stride,+ qo_batch_stride,+ qo_seq_stride,+ qo_head_stride,+ qo_dim_stride,+ batch_id,+ seq_index,+ group * HEADS_Q,+ n_heads_Q,+ cos1,+ sin1,+ half_head_dim,+ HEADS_Q,+ BLOCK_SIZE,+ CAST_TO_SIN,+ LONG_INDEXING,+ BACKWARD_PASS,+ )+ else:+ _rope_heads_tile(+ K,+ K_out,+ k_batch_stride,+ k_seq_stride,+ k_head_stride,+ k_dim_stride,+ ko_batch_stride,+ ko_seq_stride,+ ko_head_stride,+ ko_dim_stride,+ batch_id,+ seq_index,+ (group - n_groups_Q) * HEADS_K,+ n_heads_K,+ cos1,+ sin1,+ half_head_dim,+ HEADS_K,+ BLOCK_SIZE,+ CAST_TO_SIN,+ LONG_INDEXING,+ BACKWARD_PASS,)- k0 = tl.load(k_ptr + col_offsets, mask = mask, other = 0)- k1 = tl.load(k_ptr + half_head_dim + col_offsets, mask = mask, other = 0)- tl.store(k_ptr + col_offsets, k0 * cos1 - k1 * sin1, mask = mask)- tl.store(k_ptr + half_head_dim + col_offsets, k1 * cos1 + k0 * sin1, mask = mask)+# No triton.heuristics wrapper: the launcher passes real bools, and the wrapper's per-call cost+# showed up at small (decode / short-sequence) shapes._rope_embedding_QK = triton.jit(_rope_embedding_QK)-_rope_embedding_QK = triton.heuristics(- {- "BACKWARD_PASS": lambda args: bool(args["BACKWARD_PASS"]),- "HAS_ROPE_INDICES": lambda args: bool(args["HAS_ROPE_INDICES"]),- "LONG_INDEXING": lambda args: bool(args["LONG_INDEXING"]),- }-)(_rope_embedding_QK)+# Elements of one half-head tile a program rotates: several heads per program instead of one+# program per (row, head) with half its lanes masked off.+ROPE_TILE_ELEMENTS: int = 1024+# None picks per GPU: 8 warps on pre-Ampere CUDA (T4 measured ~5% faster at Llama-8B shapes), else 4+# (A100 / RTX PRO 6000 are flat between 2 and 8, a little slower at 8).+ROPE_NUM_WARPS = None-ROPE_GROUP_SIZE: int = 4[email protected]_cache(maxsize = None)+def _rope_num_warps_at(device_type, index):+ if device_type == "cuda" and torch.version.hip is None:+ if torch.cuda.get_device_capability(index)[0] < 8:+ return 8+ return 4-def _rope_embedding(- Q,- Q_row_stride: tl.constexpr,- cos,- cos_row_stride: tl.constexpr,- sin,- sin_row_stride: tl.constexpr,- seqlen,- head_dim: tl.constexpr,- n_heads: tl.constexpr,- BACKWARD_PASS: tl.constexpr,- LONG_INDEXING: tl.constexpr,- BLOCK_SIZE: tl.constexpr,-):- """- Calculates the RoPE Embedding quickly- RoPE is Q * cos + rotate_half(Q) * sin- See our blog post for more info- """- ROPE_GROUP_SIZE = 4- row_position = tl.program_id(0)- group_head_position = tl.program_id(1)- col_offsets = tl.arange(0, BLOCK_SIZE)- half_head_dim = head_dim // 2- mask = col_offsets < half_head_dim- sin1 = tl.load(- sin + (row_position % seqlen) * sin_row_stride + half_head_dim * 0 + col_offsets,- mask = mask,- other = 0,- )- cos1 = tl.load(- cos + (row_position % seqlen) * cos_row_stride + half_head_dim * 0 + col_offsets,- mask = mask,- other = 0,- )+def _rope_num_warps(device):+ if ROPE_NUM_WARPS is not None:+ return ROPE_NUM_WARPS+ index = device.index+ if index is None and device.type == "cuda":+ index = torch.cuda.current_device()+ return _rope_num_warps_at(device.type, index)- if BACKWARD_PASS:- # See the Unsloth blog post for more info.- sin1 = -sin1- # [TODO] Autotune ROPE_GROUP_SIZE to be 1, 2, 4, 8- head_start = group_head_position * ROPE_GROUP_SIZE- head_end = min((head_start + ROPE_GROUP_SIZE), n_heads)+def _heads_per_program(n_heads, block_size):+ return max(1, min(next_power_of_2(n_heads), ROPE_TILE_ELEMENTS // block_size))- # 10% faster kernel from HuyNguyen-hust, unslothai/unsloth#238.- if LONG_INDEXING:- Q += row_position.to(tl.int64) * Q_row_stride- else:- Q += row_position * Q_row_stride- for k in range(head_start, head_end):- offs_q1 = k * head_dim + col_offsets- offs_q2 = k * head_dim + col_offsets + half_head_dim- # Gemma sometimes needs RoPE done in float32 rather than bfloat16.- Q1 = tl.load(Q + offs_q1, mask = mask, other = 0).to(sin1.dtype)- Q2 = tl.load(Q + offs_q2, mask = mask, other = 0).to(sin1.dtype)-- tl.store(Q + offs_q1, Q1 * cos1 - Q2 * sin1, mask = mask)- tl.store(Q + offs_q2, Q2 * cos1 + Q1 * sin1, mask = mask)---_rope_embedding = triton.jit(_rope_embedding)-_rope_embedding = triton.heuristics(- {- "BACKWARD_PASS": lambda args: bool(args["BACKWARD_PASS"]),- "LONG_INDEXING": lambda args: bool(args["LONG_INDEXING"]),- }-)(_rope_embedding)---def _rope_rows(Q, cos, sin, seq_len, n_heads, head_dim, backward, wrap):- # Rotates Q [batch * seq_len, n_heads * head_dim] in place; BLOCK_SIZE = head_dim//2 was racy.- BLOCK_SIZE, num_warps = calculate_settings(head_dim // 2)- n_groups = (- n_heads + ROPE_GROUP_SIZE - 1- ) // ROPE_GROUP_SIZE # also a SymInt under dynamic=True- wrap(_rope_embedding)[- (- Q.shape[0],- n_groups,- )- ](+def _rope_launch(Q, Q_out, K, K_out, cos, sin, rope_ptr, has_indices, backward, cast_to_sin, wrap):+ # Q, K, Q_out, K_out: [batch, n_heads, seq_len, head_dim] at any strides (K may be None).+ # Q_out may be Q itself (in place).+ batch, n_heads_Q, seq_len, head_dim = Q.shape+ half_head_dim = head_dim // 2+ BLOCK_SIZE = next_power_of_2(half_head_dim)+ HEADS_Q = _heads_per_program(n_heads_Q, BLOCK_SIZE)+ n_groups_Q = (n_heads_Q + HEADS_Q - 1) // HEADS_Q # also a SymInt under dynamic=True+ if K is None:+ # No key heads: a dummy, never read or written. Not Q / Q_out, since a traced kernel would+ # see Q_out mutated through two arguments.+ dummy = Q.new_empty((1, 1, 1, 1))+ K, K_out, n_heads_K, HEADS_K, n_groups_K = dummy, dummy, 0, 1, 0+ else:+ n_heads_K = K.shape[1]+ HEADS_K = _heads_per_program(n_heads_K, BLOCK_SIZE)+ n_groups_K = (n_heads_K + HEADS_K - 1) // HEADS_K+ if rope_ptr is None:+ rope_ptr = cos # never read without HAS_ROPE_INDICES+ tensors = (Q, Q_out, K, K_out)+ wrap(_rope_embedding_QK)[(batch * seq_len, n_groups_Q + n_groups_K)](Q,+ Q_out,+ K,+ K_out,Q.stride(0),+ Q.stride(2),+ Q.stride(1),+ Q.stride(3),+ Q_out.stride(0),+ Q_out.stride(2),+ Q_out.stride(1),+ Q_out.stride(3),+ K.stride(0),+ K.stride(2),+ K.stride(1),+ K.stride(3),+ K_out.stride(0),+ K_out.stride(2),+ K_out.stride(1),+ K_out.stride(3),cos,cos.stride(0),sin,sin.stride(0),+ rope_ptr,seq_len,- head_dim,- n_heads,- BACKWARD_PASS = backward,- LONG_INDEXING = long_indexing(Q, block = BLOCK_SIZE),+ n_heads_Q,+ n_heads_K,+ n_groups_Q,+ half_head_dim = half_head_dim,+ BACKWARD_PASS = bool(backward),+ HAS_ROPE_INDICES = bool(has_indices),+ LONG_INDEXING = long_indexing(*tensors, block = 2 * BLOCK_SIZE * max(HEADS_Q, HEADS_K)),+ CAST_TO_SIN = bool(cast_to_sin),+ HEADS_Q = HEADS_Q,+ HEADS_K = HEADS_K,BLOCK_SIZE = BLOCK_SIZE,- num_warps = num_warps,+ EVICT_INDICES = _eviction_hints_ok(Q.device) if has_indices else False,+ num_warps = _rope_num_warps(Q.device),enable_fp_fusion = True,)+def _rope_rows(Q, Q_out, cos, sin, backward, wrap):+ # Q, Q_out: [batch, seq_len, n_heads, head_dim] at any strides; Q_out may be Q (in place).+ # Rows cast Q to sin's dtype (the QK path does not), kept for bitwise parity with before.+ _rope_launch(+ Q.transpose(1, 2),+ Q_out.transpose(1, 2),+ None,+ None,+ cos,+ sin,+ None,+ False,+ backward,+ True,+ wrap,+ )+++def _bshd_buffer(X):+ # A fresh [batch, n_heads, seq_len, head_dim] view of a contiguous [batch, seq_len, n_heads,+ # head_dim] buffer: the layout Q.transpose(1, 2).contiguous().transpose(1, 2) produced.+ batch, n_heads, seq_len, head_dim = X.shape+ return torch.empty(+ (batch, seq_len, n_heads, head_dim), dtype = X.dtype, device = X.device+ ).transpose(1, 2)+++class Fast_RoPE_Embedding_QK_Positions(torch.autograd.Function):+ """Fast_RoPE_Embedding on Q and K ([batch, n_heads, seq_len, head_dim]) in one launch each way.++ Same bytes, layouts and aliasing as Fast_RoPE_Embedding.apply(X.transpose(1, 2).contiguous(),+ cos, sin).transpose(1, 2) for X in (Q, K): in place where X.transpose(1, 2) is contiguous,+ else into a fresh [batch, seq_len, n_heads, head_dim] buffer; gradients in that layout too."""++ @staticmethod+ def forward(ctx, Q, K, cos, sin):+ cos, sin = cos.squeeze(), sin.squeeze()+ assert Q.shape[2] <= cos.shape[0]+ Q_out = Q if Q.transpose(1, 2).is_contiguous() else _bshd_buffer(Q)+ K_out = K if K.transpose(1, 2).is_contiguous() else _bshd_buffer(K)+ with torch_gpu_device(Q.device):+ _rope_launch(Q, Q_out, K, K_out, cos, sin, None, False, False, True, _eager_kernel)+ ctx.cos = cos+ ctx.sin = sin+ return Q_out.view(Q_out.shape), K_out.view(K_out.shape)++ @staticmethod+ def backward(ctx, dQ, dK):+ dQ_out, dK_out = _bshd_buffer(dQ), _bshd_buffer(dK)+ with torch_gpu_device(dQ.device):+ _rope_launch(+ dQ, dQ_out, dK, dK_out, ctx.cos, ctx.sin, None, False, True, True, _eager_kernel+ )+ return dQ_out, dK_out, None, None++class Fast_RoPE_Embedding(torch.autograd.Function):@staticmethoddef forward(ctx, Q, cos, sin):cos, sin = cos.squeeze(), sin.squeeze()- batch: int- seq_len: int- n_heads: int- head_dim: int- batch, seq_len, n_heads, head_dim = Q.shape- Q = Q.reshape(batch * seq_len, n_heads * head_dim)- assert seq_len <= cos.shape[0]+ assert Q.shape[1] <= cos.shape[0]+ # In place on contiguous Q (callers pass Q.transpose(1, 2).contiguous()), as before;+ # a strided Q is written to a fresh contiguous tensor instead of being copied first.+ Q_out = Q if Q.is_contiguous() else torch.empty(Q.shape, dtype = Q.dtype, device = Q.device)with torch_gpu_device(Q.device):- _rope_rows(Q, cos, sin, seq_len, n_heads, head_dim, False, _eager_kernel)+ _rope_rows(Q, Q_out, cos, sin, False, _eager_kernel)ctx.cos = cosctx.sin = sin- return Q.reshape(batch, seq_len, n_heads, head_dim)+ return Q_out.view(Q_out.shape)@staticmethoddef backward(ctx, dY):- batch: int- seq_len: int- n_heads: int- head_dim: int- batch, seq_len, n_heads, head_dim = dY.shape- # The in-place kernel needs a private contiguous buffer, even for already-contiguous grads.- dY = dY.clone(memory_format = torch.contiguous_format).view(- batch * seq_len, n_heads * head_dim- )+ # Out of place into a fresh contiguous buffer: dY may be shared or stride-0 (expanded).+ dQ = torch.empty(dY.shape, dtype = dY.dtype, device = dY.device)with torch_gpu_device(dY.device):- _rope_rows(dY, ctx.cos, ctx.sin, seq_len, n_heads, head_dim, True, _eager_kernel)- dY = dY.reshape(batch, seq_len, n_heads, head_dim)+ _rope_rows(dY, dQ, ctx.cos, ctx.sin, True, _eager_kernel)return (- dY,+ dQ,None,None,)@@ -270,36 +411,9 @@ def _eviction_hints_ok(device):return _eviction_hints_ok_at(device.type, index)-def _rope_qk(Q, K, cos, sin, rope_ptr, has_indices, backward, wrap):- # Rotates Q [batch, n_heads_Q, seq_len, head_dim] and K in place, at any strides.- batch, n_heads_Q, seq_len, head_dim = Q.shape- n_heads_K = K.shape[1]- BLOCK_SIZE, num_warps = calculate_settings(head_dim)- wrap(_rope_embedding_QK)[(batch * seq_len, n_heads_Q)](- Q,- Q.stride(0),- Q.stride(1),- Q.stride(2),- K,- K.stride(0),- K.stride(1),- K.stride(2),- cos,- cos.stride(0),- sin,- sin.stride(0),- rope_ptr,- seq_len,- head_dim = head_dim,- n_heads_K = n_heads_K,- BACKWARD_PASS = backward,- HAS_ROPE_INDICES = has_indices,- LONG_INDEXING = long_indexing(Q, K, block = BLOCK_SIZE),- BLOCK_SIZE = BLOCK_SIZE,- EVICT_INDICES = _eviction_hints_ok(Q.device),- num_warps = num_warps,- enable_fp_fusion = True,- )+def _rope_qk(Q, K, Q_out, K_out, cos, sin, rope_ptr, has_indices, backward, wrap):+ # Rotates Q [batch, n_heads_Q, seq_len, head_dim] and K into Q_out / K_out, at any strides.+ _rope_launch(Q, Q_out, K, K_out, cos, sin, rope_ptr, has_indices, backward, False, wrap)class Fast_RoPE_Embedding_QK(torch.autograd.Function):@@ -308,18 +422,19 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):has_indices = rope_indices is not Nonecos, sin = cos.squeeze(), sin.squeeze()- # Inplace rotary embedding is generally fine.- Q_out = Q.clone() if not Q.is_contiguous() else Q- K_out = K.clone() if not K.is_contiguous() else K+ # Contiguous inputs rotate in place; strided ones (transposed views of the projections)+ # are read where they are and written to a fresh tensor of the same layout.+ Q_out = Q if Q.is_contiguous() else torch.empty_like(Q)+ K_out = K if K.is_contiguous() else torch.empty_like(K)if has_indices:# TRL's rotary indices are always int32, so the cast is only for safety.rope_ptr = rope_indices.reshape(-1).to(dtype = torch.int32, device = Q.device)else:- rope_ptr = cos.new_empty(1, dtype = torch.int32)+ rope_ptr = Nonewith torch_gpu_device(Q.device):- _rope_qk(Q_out, K_out, cos, sin, rope_ptr, has_indices, False, _eager_kernel)+ _rope_qk(Q, K, Q_out, K_out, cos, sin, rope_ptr, has_indices, False, _eager_kernel)ctx.has_indices = has_indicesctx.cos = cos@@ -333,21 +448,31 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):@staticmethoddef backward(ctx, dQ, dK):- rope_ptr = ctx.rope_indices if ctx.has_indices else ctx.cos.new_empty(1, dtype = torch.int32)+ rope_ptr = ctx.rope_indices- # Contiguous gradients can still share storage with each other or another branch.- dQ_out = dQ.clone(memory_format = torch.contiguous_format)- dK_out = dK.clone(memory_format = torch.contiguous_format)+ # Out of place: gradients can share storage with each other or another branch.+ dQ_out = torch.empty(dQ.shape, dtype = dQ.dtype, device = dQ.device)+ dK_out = torch.empty(dK.shape, dtype = dK.dtype, device = dK.device)with torch_gpu_device(dQ.device):_rope_qk(- dQ_out, dK_out, ctx.cos, ctx.sin, rope_ptr, ctx.has_indices, True, _eager_kernel+ dQ,+ dK,+ dQ_out,+ dK_out,+ ctx.cos,+ ctx.sin,+ rope_ptr,+ ctx.has_indices,+ True,+ _eager_kernel,)return (dQ_out, dK_out, None, None, None)-# Compiled graphs take these ops: the same kernels on fresh copies (the Functions rotate in place).+# Compiled graphs take these ops: the same kernel, written out of place (the Functions may rotate+# in place).if _TRACEABLE:@torch.library.triton_op("unsloth::rope_embedding", mutates_args = ())@@ -355,10 +480,8 @@ if _TRACEABLE:Q: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, backward: bool) -> torch.Tensor:# Q: [batch, seq_len, n_heads, head_dim] at any strides.- batch, seq_len, n_heads, head_dim = Q.shape- out = Q.clone(memory_format = torch.contiguous_format)- rows = out.view(batch * seq_len, n_heads * head_dim)- _rope_rows(rows, cos, sin, seq_len, n_heads, head_dim, backward, _traced_kernel)+ out = torch.empty(Q.shape, dtype = Q.dtype, device = Q.device)+ _rope_rows(Q, out, cos, sin, backward, _traced_kernel)return outdef _rope_setup_context(ctx, inputs, output):@@ -381,8 +504,8 @@ if _TRACEABLE:rope_indices: torch.Tensor,backward: bool,) -> Tuple[torch.Tensor, torch.Tensor]:- Q_out, K_out = Q.clone(), K.clone()- _rope_qk(Q_out, K_out, cos, sin, rope_indices, True, backward, _traced_kernel)+ Q_out, K_out = torch.empty_like(Q), torch.empty_like(K)+ _rope_qk(Q, K, Q_out, K_out, cos, sin, rope_indices, True, backward, _traced_kernel)return Q_out, K_outdef _rope_qk_setup_context(ctx, inputs, output):@@ -503,8 +626,8 @@ def _fast_rope_embedding_eager(Q, K, cos, sin, rope_embedding_indices):if rope_embedding_indices is not None:Q_out, K_out = Fast_RoPE_Embedding_QK.apply(Q, K, cos, sin, rope_embedding_indices)else:- Q_out = Fast_RoPE_Embedding.apply(Q.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)- K_out = Fast_RoPE_Embedding.apply(K.transpose(1, 2).contiguous(), cos, sin).transpose(1, 2)+ # Fast_RoPE_Embedding on Q and K, fused into one launch (forward and backward).+ Q_out, K_out = Fast_RoPE_Embedding_QK_Positions.apply(Q, K, cos, sin)_sync_if_split(Q)return Q_out, K_out