Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Compiled RoPE: keep per-tensor launches when only Q or K needs a gradient (#17205)
Daniel Han2179383acfd2
diff --git a/tests/test_rms_rope_compile.py b/tests/test_rms_rope_compile.pyindex e0a8759f009..81f36457d5f 100644--- a/tests/test_rms_rope_compile.py+++ b/tests/test_rms_rope_compile.py@@ -234,6 +234,38 @@ def test_rope_compiled_positions_one_launch_each_way(monkeypatch, one_launch):assert _graph_breaks() == 0[email protected]("grad", ["q", "k"])+def test_rope_compiled_positions_one_sided_grad_stays_per_tensor(grad):+ if not TRACEABLE:+ pytest.skip("this torch has no torch.library.triton_op")+ assert rope_embedding.COMPILED_ROPE_QK_ONE_LAUNCH+ bsz, seq, n_heads, n_kv, head_dim = 2, 37, 8, 2, 64+ g = torch.Generator(device = "cuda").manual_seed(0)+ Q = torch.randn(bsz, n_heads, seq, head_dim, device = "cuda", generator = g, dtype = CDTYPE)+ K = torch.randn(bsz, n_kv, seq, head_dim, device = "cuda", generator = g, dtype = CDTYPE)+ (Q if grad == "q" else K).requires_grad_(True)+ cos, sin = _cos_sin(seq + 16, head_dim, CDTYPE)+ compiled = _compile(lambda q, k, c, s: fast_rope_embedding(q, k, c, s))++ def step():+ Q.grad = K.grad = None+ Q_out, K_out = compiled(Q, K, cos, sin)+ (Q_out.float().sum() + K_out.float().sum()).backward()+ return Q_out, K_out++ step()+ (Q_out, K_out), launches = _rope_kernel_launches(step)+ assert Q_out.requires_grad == (grad == "q")+ assert K_out.requires_grad == (grad == "k")+ # Forward: one launch per tensor; backward: one for the tensor that needs the gradient.+ assert launches == 3, launches+ eager_Q, eager_K = fast_rope_embedding(Q.detach(), K.detach(), cos, sin)+ assert _bytes_equal(Q_out.detach(), eager_Q) and _bytes_equal(K_out.detach(), eager_K)+ leaf = Q if grad == "q" else K+ assert leaf.grad is not None and (K if grad == "q" else Q).grad is None+ assert _graph_breaks() == 0++@pytest.mark.parametrize("with_indices", [False, True], ids = ["positions", "rope_indices"])def test_rope_eager_is_the_autograd_function(with_indices):"""Outside torch.compile the public function is still the Fast_RoPE autograd Functions."""diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex 1f02efb8f57..83418fbed1a 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -754,6 +754,8 @@ def fast_rope_embedding(and K.shape[0] == Q.shape[0]and K.shape[2] == Q.shape[2]and K.shape[3] == Q.shape[3]+ # The paired backward rotates both gradients: only Q or K needing one stays per tensor.+ and Q.requires_grad == K.requires_grad):Q_out, K_out = torch.ops.unsloth.rope_embedding_rows_qk(Q.transpose(1, 2), K.transpose(1, 2), cos, sin, False