Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Compiled RoPE: keep per-tensor launches when only Q or K needs a gradient (#17205)

Daniel Han2179383acfd2
diff --git a/tests/test_rms_rope_compile.py b/tests/test_rms_rope_compile.py
index e0a8759f009..81f36457d5f 100644
--- a/tests/test_rms_rope_compile.py
+++ b/tests/test_rms_rope_compile.py
@@ -234,6 +234,38 @@ def test_rope_compiled_positions_one_launch_each_way(monkeypatch, one_launch):
assert _graph_breaks() == 0
[email protected]("grad", ["q", "k"])
+def test_rope_compiled_positions_one_sided_grad_stays_per_tensor(grad):
+ if not TRACEABLE:
+ pytest.skip("this torch has no torch.library.triton_op")
+ assert rope_embedding.COMPILED_ROPE_QK_ONE_LAUNCH
+ bsz, seq, n_heads, n_kv, head_dim = 2, 37, 8, 2, 64
+ g = torch.Generator(device = "cuda").manual_seed(0)
+ Q = torch.randn(bsz, n_heads, seq, head_dim, device = "cuda", generator = g, dtype = CDTYPE)
+ K = torch.randn(bsz, n_kv, seq, head_dim, device = "cuda", generator = g, dtype = CDTYPE)
+ (Q if grad == "q" else K).requires_grad_(True)
+ cos, sin = _cos_sin(seq + 16, head_dim, CDTYPE)
+ compiled = _compile(lambda q, k, c, s: fast_rope_embedding(q, k, c, s))
+
+ def step():
+ Q.grad = K.grad = None
+ Q_out, K_out = compiled(Q, K, cos, sin)
+ (Q_out.float().sum() + K_out.float().sum()).backward()
+ return Q_out, K_out
+
+ step()
+ (Q_out, K_out), launches = _rope_kernel_launches(step)
+ assert Q_out.requires_grad == (grad == "q")
+ assert K_out.requires_grad == (grad == "k")
+ # Forward: one launch per tensor; backward: one for the tensor that needs the gradient.
+ assert launches == 3, launches
+ eager_Q, eager_K = fast_rope_embedding(Q.detach(), K.detach(), cos, sin)
+ assert _bytes_equal(Q_out.detach(), eager_Q) and _bytes_equal(K_out.detach(), eager_K)
+ leaf = Q if grad == "q" else K
+ assert leaf.grad is not None and (K if grad == "q" else Q).grad is None
+ assert _graph_breaks() == 0
+
+
@pytest.mark.parametrize("with_indices", [False, True], ids = ["positions", "rope_indices"])
def test_rope_eager_is_the_autograd_function(with_indices):
"""Outside torch.compile the public function is still the Fast_RoPE autograd Functions."""
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index 1f02efb8f57..83418fbed1a 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -754,6 +754,8 @@ def fast_rope_embedding(
and K.shape[0] == Q.shape[0]
and K.shape[2] == Q.shape[2]
and K.shape[3] == Q.shape[3]
+ # The paired backward rotates both gradients: only Q or K needing one stays per tensor.
+ and Q.requires_grad == K.requires_grad
):
Q_out, K_out = torch.ops.unsloth.rope_embedding_rows_qk(
Q.transpose(1, 2), K.transpose(1, 2), cos, sin, False