Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

RoPE: constexpr the strides and head counts fixed per model (#15243)

Daniel Han4db3eaeb5590
diff --git a/tests/test_rope_constexpr_recompiles.py b/tests/test_rope_constexpr_recompiles.py
new file mode 100644
index 00000000000..10e52f0d421
--- /dev/null
+++ b/tests/test_rope_constexpr_recompiles.py
@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The RoPE kernel's constexpr strides and head counts are fixed per model: a compiled forward and
+backward over changing batch sizes and sequence lengths stays one graph and matches eager."""
+
+import pytest
+import torch
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+
+if not has_real_cuda():
+ pytest.skip(reason = "needs a CUDA device", allow_module_level = True)
+
+pytest.importorskip("triton")
+
+import unsloth # noqa: F401
+from unsloth.kernels import rope_embedding as RE
+
+# Batch never 1 or equal to a head count, seq never head_dim: torch's own 0/1 and duck-size
+# specializations would recompile on any kernel.
+N_HEADS, N_KV, HEAD_DIM = 8, 4, 64
+
+
+def _qk(batch, seq):
+ torch.manual_seed(batch * 1000 + seq)
+ width = (N_HEADS + 2 * N_KV) * HEAD_DIM
+ qkv = torch.randn(batch, seq, width, device = "cuda", dtype = torch.float16, requires_grad = True)
+ Q = qkv[..., : N_HEADS * HEAD_DIM].view(batch, seq, N_HEADS, HEAD_DIM).transpose(1, 2)
+ K = qkv[..., N_HEADS * HEAD_DIM : (N_HEADS + N_KV) * HEAD_DIM]
+ K = K.view(batch, seq, N_KV, HEAD_DIM).transpose(1, 2)
+ return qkv, Q, K
+
+
[email protected]("with_indices", [False, True])
+def test_compiled_rope_keeps_one_graph_across_batch_and_seq(with_indices):
+ if not RE._TRACEABLE:
+ pytest.skip(reason = "RoPE ops are not traceable on this torch / Triton")
+ from torch._dynamo.utils import counters
+
+ freqs = torch.outer(
+ torch.arange(4096.0, device = "cuda"), torch.rand(HEAD_DIM // 2, device = "cuda")
+ )
+ emb = torch.cat((freqs, freqs), -1)
+ cos, sin = emb.cos().half(), emb.sin().half()
+
+ def rope(Q, K, idx):
+ q, k = RE.fast_rope_embedding(Q, K, cos, sin, idx)
+ return q.float().square().sum() + k.float().sum()
+
+ torch._dynamo.reset()
+ counters.clear()
+ compiled = torch.compile(rope, dynamic = True, fullgraph = True)
+ for batch, seq in [(3, 37), (5, 96), (6, 129), (7, 13), (3, 300)]:
+ idx = torch.randint(0, 4096, (batch, seq), device = "cuda") if with_indices else None
+ qkv, Q, K = _qk(batch, seq)
+ compiled(Q, K, idx).backward()
+ grad = qkv.grad
+ qkv, Q, K = _qk(batch, seq)
+ rope(Q, K, idx).backward()
+ torch.testing.assert_close(grad, qkv.grad, rtol = 1e-3, atol = 1e-3)
+ assert counters["stats"]["unique_graphs"] == 1
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index 5a859766146..59fe0bdd76e 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -99,30 +99,30 @@ def _rope_embedding_QK(
K,
K_out,
q_batch_stride,
- q_seq_stride,
+ q_seq_stride: tl.constexpr, # 4096 n_heads * head_dim (6144 fused QKV, 128 contiguous)
q_head_stride,
- q_dim_stride,
+ q_dim_stride: tl.constexpr, # 1
qo_batch_stride,
- qo_seq_stride,
+ qo_seq_stride: tl.constexpr, # 4096 or 128
qo_head_stride,
- qo_dim_stride,
+ qo_dim_stride: tl.constexpr, # 1
k_batch_stride,
- k_seq_stride,
+ k_seq_stride: tl.constexpr, # 1024 n_kv_heads * head_dim
k_head_stride,
- k_dim_stride,
+ k_dim_stride: tl.constexpr, # 1
ko_batch_stride,
- ko_seq_stride,
+ ko_seq_stride: tl.constexpr, # 1024 or 128
ko_head_stride,
- ko_dim_stride,
+ ko_dim_stride: tl.constexpr, # 1
cos,
- cos_row_stride,
+ cos_row_stride, # runtime: a SymInt in compiled graphs
sin,
sin_row_stride,
rope_embedding_indices,
seqlen,
- n_heads_Q,
- n_heads_K,
- n_groups_Q,
+ n_heads_Q: tl.constexpr, # 32
+ n_heads_K: tl.constexpr, # 8 (0 without K)
+ n_groups_Q: tl.constexpr, # 1
half_head_dim: tl.constexpr,
BACKWARD_PASS: tl.constexpr,
HAS_ROPE_INDICES: tl.constexpr,