Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix: preserve shared upstream gradients in RoPE backward (#12528)
cherry77-cloudd81de9f3aab8
diff --git a/tests/test_rope_zero_stride_grad.py b/tests/test_rope_zero_stride_grad.pyindex 83a391b6f50..41164cbc392 100644--- a/tests/test_rope_zero_stride_grad.py+++ b/tests/test_rope_zero_stride_grad.py@@ -47,3 +47,36 @@ def test_rope_backward_matches_reference(use_indices, grad_kind):torch.testing.assert_close(q, rq, atol = 1e-5, rtol = 1e-5)torch.testing.assert_close(x.grad, x_ref.grad, atol = 1e-4, rtol = 1e-4)++[email protected]("implementation", ["single", "indexed_qk", "slow"])+def test_rope_backward_preserves_shared_dense_gradient(implementation):+ from unsloth.kernels.rope_embedding import (+ Fast_RoPE_Embedding,+ Fast_RoPE_Embedding_QK,+ Slow_RoPE_Embedding,+ )++ torch.manual_seed(1)+ b, h, s, d = 2, 4, 16, 64+ inv = 1.0 / (10000 ** (torch.arange(0, d, 2, device = "cuda").float() / d))+ freqs = torch.outer(torch.arange(s, device = "cuda").float(), inv)+ emb = torch.cat((freqs, freqs), -1)+ cos, sin = emb.cos(), emb.sin()+ x = torch.randn(b, h, s, d, device = "cuda", requires_grad = True)+ z = torch.randn_like(x, requires_grad = True)++ if implementation == "single":+ q = Fast_RoPE_Embedding.apply(x.transpose(1, 2).contiguous(), cos, sin)+ k = Fast_RoPE_Embedding.apply(z.transpose(1, 2).contiguous(), cos, sin)+ elif implementation == "indexed_qk":+ indices = torch.arange(s, device = "cuda").expand(b, s).contiguous()+ q, k = Fast_RoPE_Embedding_QK.apply(x, z, cos, sin, indices)+ else:+ q = Slow_RoPE_Embedding.apply(x, cos[None, None], sin[None, None], None)+ k = Slow_RoPE_Embedding.apply(z, cos[None, None], sin[None, None], None)++ grad = torch.randn_like(q)+ grad_before = grad.clone()+ (q + k).backward(grad)+ assert torch.equal(grad, grad_before)diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex 45e31fa1e46..1bdb3b139fc 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -222,8 +222,10 @@ class Fast_RoPE_Embedding(torch.autograd.Function):n_heads: inthead_dim: intbatch, seq_len, n_heads, head_dim = dY.shape- # contiguous: an expanded gradient (e.g. from Q.sum()) would be rotated through stride 0.- dY = dY.contiguous().reshape(batch * seq_len, n_heads * head_dim)+ # The in-place kernel needs a private contiguous buffer, even for already-contiguous grads.+ dY = dY.clone(memory_format = torch.contiguous_format).view(+ batch * seq_len, n_heads * head_dim+ )with torch_gpu_device(dY.device):_rope_rows(dY, ctx.cos, ctx.sin, seq_len, n_heads, head_dim, True, _eager_kernel)dY = dY.reshape(batch, seq_len, n_heads, head_dim)@@ -296,9 +298,9 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):def backward(ctx, dQ, dK):rope_ptr = ctx.rope_indices if ctx.has_indices else ctx.cos.new_empty(1, dtype = torch.int32)- # Inplace rotary embedding is generally fine.- dQ_out = dQ.clone() if not dQ.is_contiguous() else dQ- dK_out = dK.clone() if not dK.is_contiguous() else dK+ # Contiguous gradients can still share storage with each other or another branch.+ dQ_out = dQ.clone(memory_format = torch.contiguous_format)+ dK_out = dK.clone(memory_format = torch.contiguous_format)with torch_gpu_device(dQ.device):_rope_qk(@@ -422,6 +424,7 @@ class Slow_RoPE_Embedding(torch.autograd.Function):# Q * cos + rotate_half.T(Q) * sinhalf = dY.shape[-1] // 2RH_dY = torch.cat((dY[..., half:], -dY[..., :half]), dim = -1)+ dY = dY.clone()dY *= cosdY.addcmul_(RH_dY, sin)return dY, None, None, None