Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

RoPE: 64-bit cos / sin row offsets for tables past 2**31 elements (#14714)

Daniel Han37eec0a3d0bc
diff --git a/tests/test_decode_fused_rope.py b/tests/test_decode_fused_rope.py
index 26cee4576ad..b6957c6f924 100644
--- a/tests/test_decode_fused_rope.py
+++ b/tests/test_decode_fused_rope.py
@@ -254,3 +254,53 @@ def test_decode_rope_on_another_device_orders_with_the_copy():
ulp = torch.finfo(q.dtype).eps * max(qu.abs().max().item(), ku.abs().max().item())
assert (q0.cpu().float() - qu.cpu().float()).abs().max().item() <= 2 * ulp
assert (k0.cpu().float() - ku.cpu().float()).abs().max().item() <= 2 * ulp
+
+
+def test_a_cos_table_past_int32_offsets_takes_64_bit_indexing():
+ # Meta tensors and a launch stub: only the constexprs the launcher picks are read.
+ def launched(rows):
+ seen = {}
+
+ class _Grid:
+ def __getitem__(self, grid):
+ return lambda *args, **kwargs: seen.update(kwargs)
+
+ cos = torch.empty(rows, 128, device = "meta", dtype = torch.float16)
+ q = torch.empty(2, 8, 1, 128, device = "meta", dtype = torch.float16)
+ k = torch.empty(2, 2, 1, 128, device = "meta", dtype = torch.float16)
+ RE._rope_launch(q, q, k, k, cos, cos, None, False, False, False, lambda fn: _Grid())
+ return seen["LONG_INDEXING"]
+
+ assert launched(2**31 // 128 + 4)
+ assert not launched(4096)
+
+
+def test_decode_rope_past_int32_cache_offsets_matches_the_reference():
+ head_dim = 128
+ pos = 2**31 // head_dim + 3
+ need = 2 * (pos + 1) * head_dim * 2 + (1 << 30)
+ if torch.cuda.mem_get_info()[0] < need:
+ pytest.skip("needs about 9 GiB of free GPU memory for two 4 GiB tables")
+ cos = torch.empty(pos + 1, head_dim, device = "cuda", dtype = torch.float16)
+ sin = torch.empty_like(cos)
+ half = head_dim // 2
+ cos[pos, :half].uniform_(-1, 1)
+ sin[pos, :half].uniform_(-1, 1)
+ cos[pos, half:] = cos[pos, :half]
+ sin[pos, half:] = sin[pos, :half]
+ torch.manual_seed(0)
+ Q = torch.randn(2, 8, 1, head_dim, device = "cuda", dtype = torch.float16)
+ K = torch.randn(2, 2, 1, head_dim, device = "cuda", dtype = torch.float16)
+
+ def reference(x):
+ x = x.double()
+ rot = torch.cat([-x[..., half:], x[..., :half]], -1)
+ return x * cos[pos].double() + rot * sin[pos].double()
+
+ expected_q, expected_k = reference(Q), reference(K)
+ position_ids = torch.full((2, 1), pos, device = "cuda")
+ assert RE.decode_rope_inplace(Q, K, cos, sin, position_ids)
+ torch.cuda.synchronize()
+ torch.testing.assert_close(Q.double(), expected_q, rtol = 2e-3, atol = 2e-3)
+ torch.testing.assert_close(K.double(), expected_k, rtol = 2e-3, atol = 2e-3)
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index 2dea85edbd5..5a859766146 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -154,6 +154,9 @@ def _rope_embedding_QK(
rot_position = tl.load(rope_embedding_indices + row_position).to(tl.int32)
else:
rot_position = row_position % seqlen
+ if LONG_INDEXING:
+ # cos / sin tables past 2**31 elements need a 64-bit row offset.
+ rot_position = rot_position.to(tl.int64)
sin1 = tl.load(sin + rot_position * sin_row_stride + col_offsets, mask = mask, other = 0)
cos1 = tl.load(cos + rot_position * cos_row_stride + col_offsets, mask = mask, other = 0)
@@ -271,7 +274,7 @@ def _rope_launch(Q, Q_out, K, K_out, cos, sin, rope_ptr, has_indices, backward,
n_groups_K = (n_heads_K + HEADS_K - 1) // HEADS_K
if rope_ptr is None:
rope_ptr = cos # never read without HAS_ROPE_INDICES
- tensors = (Q, Q_out, K, K_out)
+ tensors = (Q, Q_out, K, K_out, cos, sin)
wrap(_rope_embedding_QK)[(batch * seq_len, n_groups_Q + n_groups_K)](
Q,
Q_out,