Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
RoPE: constexpr the strides and head counts fixed per model (#15243)
Daniel Han4db3eaeb5590
diff --git a/tests/test_rope_constexpr_recompiles.py b/tests/test_rope_constexpr_recompiles.pynew file mode 100644index 00000000000..10e52f0d421--- /dev/null+++ b/tests/test_rope_constexpr_recompiles.py@@ -0,0 +1,62 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The RoPE kernel's constexpr strides and head counts are fixed per model: a compiled forward and+backward over changing batch sizes and sequence lengths stays one graph and matches eager."""++import pytest+import torch+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py++if not has_real_cuda():+ pytest.skip(reason = "needs a CUDA device", allow_module_level = True)++pytest.importorskip("triton")++import unsloth # noqa: F401+from unsloth.kernels import rope_embedding as RE++# Batch never 1 or equal to a head count, seq never head_dim: torch's own 0/1 and duck-size+# specializations would recompile on any kernel.+N_HEADS, N_KV, HEAD_DIM = 8, 4, 64+++def _qk(batch, seq):+ torch.manual_seed(batch * 1000 + seq)+ width = (N_HEADS + 2 * N_KV) * HEAD_DIM+ qkv = torch.randn(batch, seq, width, device = "cuda", dtype = torch.float16, requires_grad = True)+ Q = qkv[..., : N_HEADS * HEAD_DIM].view(batch, seq, N_HEADS, HEAD_DIM).transpose(1, 2)+ K = qkv[..., N_HEADS * HEAD_DIM : (N_HEADS + N_KV) * HEAD_DIM]+ K = K.view(batch, seq, N_KV, HEAD_DIM).transpose(1, 2)+ return qkv, Q, K++[email protected]("with_indices", [False, True])+def test_compiled_rope_keeps_one_graph_across_batch_and_seq(with_indices):+ if not RE._TRACEABLE:+ pytest.skip(reason = "RoPE ops are not traceable on this torch / Triton")+ from torch._dynamo.utils import counters++ freqs = torch.outer(+ torch.arange(4096.0, device = "cuda"), torch.rand(HEAD_DIM // 2, device = "cuda")+ )+ emb = torch.cat((freqs, freqs), -1)+ cos, sin = emb.cos().half(), emb.sin().half()++ def rope(Q, K, idx):+ q, k = RE.fast_rope_embedding(Q, K, cos, sin, idx)+ return q.float().square().sum() + k.float().sum()++ torch._dynamo.reset()+ counters.clear()+ compiled = torch.compile(rope, dynamic = True, fullgraph = True)+ for batch, seq in [(3, 37), (5, 96), (6, 129), (7, 13), (3, 300)]:+ idx = torch.randint(0, 4096, (batch, seq), device = "cuda") if with_indices else None+ qkv, Q, K = _qk(batch, seq)+ compiled(Q, K, idx).backward()+ grad = qkv.grad+ qkv, Q, K = _qk(batch, seq)+ rope(Q, K, idx).backward()+ torch.testing.assert_close(grad, qkv.grad, rtol = 1e-3, atol = 1e-3)+ assert counters["stats"]["unique_graphs"] == 1diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex 5a859766146..59fe0bdd76e 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -99,30 +99,30 @@ def _rope_embedding_QK(K,K_out,q_batch_stride,- q_seq_stride,+ q_seq_stride: tl.constexpr, # 4096 n_heads * head_dim (6144 fused QKV, 128 contiguous)q_head_stride,- q_dim_stride,+ q_dim_stride: tl.constexpr, # 1qo_batch_stride,- qo_seq_stride,+ qo_seq_stride: tl.constexpr, # 4096 or 128qo_head_stride,- qo_dim_stride,+ qo_dim_stride: tl.constexpr, # 1k_batch_stride,- k_seq_stride,+ k_seq_stride: tl.constexpr, # 1024 n_kv_heads * head_dimk_head_stride,- k_dim_stride,+ k_dim_stride: tl.constexpr, # 1ko_batch_stride,- ko_seq_stride,+ ko_seq_stride: tl.constexpr, # 1024 or 128ko_head_stride,- ko_dim_stride,+ ko_dim_stride: tl.constexpr, # 1cos,- cos_row_stride,+ cos_row_stride, # runtime: a SymInt in compiled graphssin,sin_row_stride,rope_embedding_indices,seqlen,- n_heads_Q,- n_heads_K,- n_groups_Q,+ n_heads_Q: tl.constexpr, # 32+ n_heads_K: tl.constexpr, # 8 (0 without K)+ n_groups_Q: tl.constexpr, # 1half_head_dim: tl.constexpr,BACKWARD_PASS: tl.constexpr,HAS_ROPE_INDICES: tl.constexpr,