Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix: preserve shared upstream gradients in RoPE backward (#12528)

cherry77-cloudd81de9f3aab8
diff --git a/tests/test_rope_zero_stride_grad.py b/tests/test_rope_zero_stride_grad.py
index 83a391b6f50..41164cbc392 100644
--- a/tests/test_rope_zero_stride_grad.py
+++ b/tests/test_rope_zero_stride_grad.py
@@ -47,3 +47,36 @@ def test_rope_backward_matches_reference(use_indices, grad_kind):
torch.testing.assert_close(q, rq, atol = 1e-5, rtol = 1e-5)
torch.testing.assert_close(x.grad, x_ref.grad, atol = 1e-4, rtol = 1e-4)
+
+
[email protected]("implementation", ["single", "indexed_qk", "slow"])
+def test_rope_backward_preserves_shared_dense_gradient(implementation):
+ from unsloth.kernels.rope_embedding import (
+ Fast_RoPE_Embedding,
+ Fast_RoPE_Embedding_QK,
+ Slow_RoPE_Embedding,
+ )
+
+ torch.manual_seed(1)
+ b, h, s, d = 2, 4, 16, 64
+ inv = 1.0 / (10000 ** (torch.arange(0, d, 2, device = "cuda").float() / d))
+ freqs = torch.outer(torch.arange(s, device = "cuda").float(), inv)
+ emb = torch.cat((freqs, freqs), -1)
+ cos, sin = emb.cos(), emb.sin()
+ x = torch.randn(b, h, s, d, device = "cuda", requires_grad = True)
+ z = torch.randn_like(x, requires_grad = True)
+
+ if implementation == "single":
+ q = Fast_RoPE_Embedding.apply(x.transpose(1, 2).contiguous(), cos, sin)
+ k = Fast_RoPE_Embedding.apply(z.transpose(1, 2).contiguous(), cos, sin)
+ elif implementation == "indexed_qk":
+ indices = torch.arange(s, device = "cuda").expand(b, s).contiguous()
+ q, k = Fast_RoPE_Embedding_QK.apply(x, z, cos, sin, indices)
+ else:
+ q = Slow_RoPE_Embedding.apply(x, cos[None, None], sin[None, None], None)
+ k = Slow_RoPE_Embedding.apply(z, cos[None, None], sin[None, None], None)
+
+ grad = torch.randn_like(q)
+ grad_before = grad.clone()
+ (q + k).backward(grad)
+ assert torch.equal(grad, grad_before)
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index 45e31fa1e46..1bdb3b139fc 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -222,8 +222,10 @@ class Fast_RoPE_Embedding(torch.autograd.Function):
n_heads: int
head_dim: int
batch, seq_len, n_heads, head_dim = dY.shape
- # contiguous: an expanded gradient (e.g. from Q.sum()) would be rotated through stride 0.
- dY = dY.contiguous().reshape(batch * seq_len, n_heads * head_dim)
+ # The in-place kernel needs a private contiguous buffer, even for already-contiguous grads.
+ dY = dY.clone(memory_format = torch.contiguous_format).view(
+ batch * seq_len, n_heads * head_dim
+ )
with torch_gpu_device(dY.device):
_rope_rows(dY, ctx.cos, ctx.sin, seq_len, n_heads, head_dim, True, _eager_kernel)
dY = dY.reshape(batch, seq_len, n_heads, head_dim)
@@ -296,9 +298,9 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
def backward(ctx, dQ, dK):
rope_ptr = ctx.rope_indices if ctx.has_indices else ctx.cos.new_empty(1, dtype = torch.int32)
- # Inplace rotary embedding is generally fine.
- dQ_out = dQ.clone() if not dQ.is_contiguous() else dQ
- dK_out = dK.clone() if not dK.is_contiguous() else dK
+ # Contiguous gradients can still share storage with each other or another branch.
+ dQ_out = dQ.clone(memory_format = torch.contiguous_format)
+ dK_out = dK.clone(memory_format = torch.contiguous_format)
with torch_gpu_device(dQ.device):
_rope_qk(
@@ -422,6 +424,7 @@ class Slow_RoPE_Embedding(torch.autograd.Function):
# Q * cos + rotate_half.T(Q) * sin
half = dY.shape[-1] // 2
RH_dY = torch.cat((dY[..., half:], -dY[..., :half]), dim = -1)
+ dY = dY.clone()
dY *= cos
dY.addcmul_(RH_dY, sin)
return dY, None, None, None