Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix non-contiguous position indices in fused and unfused RoPE (#15235)

cherry77-cloud6c3dcdd4b078
diff --git a/unsloth/kernels/qk_norm_rope.py b/unsloth/kernels/qk_norm_rope.py
index e64601ee5fd..0b9dc9bc65c 100644
--- a/unsloth/kernels/qk_norm_rope.py
+++ b/unsloth/kernels/qk_norm_rope.py
@@ -454,7 +454,7 @@ def fast_qk_norm_rope(
position_ids = position_ids.reshape(1, seq_len).expand(batch, seq_len)
if position_ids.numel() != batch * seq_len:
return _unfused(q_norm, k_norm, Q, K, cos, sin, position_ids)
- rope_indices = position_ids.reshape(-1).to(dtype = torch.int32, device = Q.device)
+ rope_indices = position_ids.reshape(-1).to(dtype = torch.int32, device = Q.device).contiguous()
else:
if seq_len > cos2.shape[0]:
return _unfused(q_norm, k_norm, Q, K, cos, sin, position_ids)
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index 59fe0bdd76e..dc5f3962688 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -433,7 +433,7 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
if has_indices:
# TRL's rotary indices are always int32, so the cast is only for safety.
- rope_ptr = rope_indices.reshape(-1).to(dtype = torch.int32, device = Q.device)
+ rope_ptr = rope_indices.reshape(-1).to(dtype = torch.int32, device = Q.device).contiguous()
else:
rope_ptr = None
@@ -704,7 +704,9 @@ def fast_rope_embedding(
return _fast_rope_embedding_untraced(Q, K, cos, sin, rope_embedding_indices)
cos, sin = cos.squeeze(), sin.squeeze()
if rope_embedding_indices is not None:
- rope_ptr = rope_embedding_indices.reshape(-1).to(dtype = torch.int32, device = Q.device)
+ rope_ptr = (
+ rope_embedding_indices.reshape(-1).to(dtype = torch.int32, device = Q.device).contiguous()
+ )
return torch.ops.unsloth.rope_embedding_qk(Q, K, cos, sin, rope_ptr, False)
Q_out = torch.ops.unsloth.rope_embedding(Q.transpose(1, 2), cos, sin, False).transpose(1, 2)
K_out = torch.ops.unsloth.rope_embedding(K.transpose(1, 2), cos, sin, False).transpose(1, 2)