Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix non-contiguous position indices in fused and unfused RoPE (#15235)
cherry77-cloud6c3dcdd4b078
diff --git a/unsloth/kernels/qk_norm_rope.py b/unsloth/kernels/qk_norm_rope.pyindex e64601ee5fd..0b9dc9bc65c 100644--- a/unsloth/kernels/qk_norm_rope.py+++ b/unsloth/kernels/qk_norm_rope.py@@ -454,7 +454,7 @@ def fast_qk_norm_rope(position_ids = position_ids.reshape(1, seq_len).expand(batch, seq_len)if position_ids.numel() != batch * seq_len:return _unfused(q_norm, k_norm, Q, K, cos, sin, position_ids)- rope_indices = position_ids.reshape(-1).to(dtype = torch.int32, device = Q.device)+ rope_indices = position_ids.reshape(-1).to(dtype = torch.int32, device = Q.device).contiguous()else:if seq_len > cos2.shape[0]:return _unfused(q_norm, k_norm, Q, K, cos, sin, position_ids)diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex 59fe0bdd76e..dc5f3962688 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -433,7 +433,7 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):if has_indices:# TRL's rotary indices are always int32, so the cast is only for safety.- rope_ptr = rope_indices.reshape(-1).to(dtype = torch.int32, device = Q.device)+ rope_ptr = rope_indices.reshape(-1).to(dtype = torch.int32, device = Q.device).contiguous()else:rope_ptr = None@@ -704,7 +704,9 @@ def fast_rope_embedding(return _fast_rope_embedding_untraced(Q, K, cos, sin, rope_embedding_indices)cos, sin = cos.squeeze(), sin.squeeze()if rope_embedding_indices is not None:- rope_ptr = rope_embedding_indices.reshape(-1).to(dtype = torch.int32, device = Q.device)+ rope_ptr = (+ rope_embedding_indices.reshape(-1).to(dtype = torch.int32, device = Q.device).contiguous()+ )return torch.ops.unsloth.rope_embedding_qk(Q, K, cos, sin, rope_ptr, False)Q_out = torch.ops.unsloth.rope_embedding(Q.transpose(1, 2), cos, sin, False).transpose(1, 2)K_out = torch.ops.unsloth.rope_embedding(K.transpose(1, 2), cos, sin, False).transpose(1, 2)