Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
RoPE: rotate Q and K in one launch each way under torch.compile (#17109)
Daniel Han315fd8907dc1
diff --git a/tests/test_rms_rope_compile.py b/tests/test_rms_rope_compile.pyindex 5e6e38e02b0..e0a8759f009 100644--- a/tests/test_rms_rope_compile.py+++ b/tests/test_rms_rope_compile.py@@ -199,6 +199,41 @@ def test_rope_compiled_matches_eager(shape, with_indices, dtype):assert _graph_breaks() == 0+def _rope_kernel_launches(fn, *args):+ from torch.profiler import ProfilerActivity, profile++ torch.cuda.synchronize()+ with profile(activities = [ProfilerActivity.CUDA]) as prof:+ out = fn(*args)+ torch.cuda.synchronize()+ names = [e.name for e in prof.events() if e.device_type == torch.autograd.DeviceType.CUDA]+ return out, sum("_rope_embedding_QK" in name for name in names)++[email protected]("one_launch", [True, False], ids = ["one_launch", "per_tensor"])+def test_rope_compiled_positions_one_launch_each_way(monkeypatch, one_launch):+ """Compiled positions RoPE: one launch each way (per tensor with the flag off), eager's bytes."""+ if not TRACEABLE:+ pytest.skip("this torch has no torch.library.triton_op")+ monkeypatch.setattr(rope_embedding, "COMPILED_ROPE_QK_ONE_LAUNCH", one_launch)+ compiled = _compile(lambda q, k, c, s, i: fast_rope_embedding(q, k, c, s, i))+ shape = (2, 37, 8, 2, 64)+ _rope_case(compiled, *shape, CDTYPE, False, pure = True)+ got, launches = _rope_kernel_launches(+ _rope_case,+ compiled,+ *shape,+ CDTYPE,+ False,+ True,+ )+ assert launches == (2 if one_launch else 4), launches+ eager = _rope_case(fast_rope_embedding, *shape, CDTYPE, False)+ for e, c, name in zip(eager, got, ("Q", "K", "dQ", "dK")):+ assert _bytes_equal(e, c), name+ assert _graph_breaks() == 0++@pytest.mark.parametrize("with_indices", [False, True], ids = ["positions", "rope_indices"])def test_rope_eager_is_the_autograd_function(with_indices):"""Outside torch.compile the public function is still the Fast_RoPE autograd Functions."""diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex 1b2fd377910..1f02efb8f57 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -499,6 +499,40 @@ if _TRACEABLE:_rope_embedding_op_backward, setup_context = _rope_setup_context)+ @torch.library.triton_op("unsloth::rope_embedding_rows_qk", mutates_args = ())+ def _rope_embedding_rows_qk_op(+ Q: torch.Tensor, K: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, backward: bool+ ) -> Tuple[torch.Tensor, torch.Tensor]:+ # unsloth::rope_embedding on Q and K in one launch (same kernel and flags: bitwise equal).+ Q_out = torch.empty(Q.shape, dtype = Q.dtype, device = Q.device)+ K_out = torch.empty(K.shape, dtype = K.dtype, device = K.device)+ _rope_launch(+ Q.transpose(1, 2),+ Q_out.transpose(1, 2),+ K.transpose(1, 2),+ K_out.transpose(1, 2),+ cos,+ sin,+ None,+ False,+ backward,+ True,+ _traced_kernel,+ )+ return Q_out, K_out++ def _rope_rows_qk_setup_context(ctx, inputs, output):+ ctx.save_for_backward(inputs[2], inputs[3])++ def _rope_embedding_rows_qk_op_backward(ctx, dQ, dK):+ cos, sin = ctx.saved_tensors+ dQ, dK = torch.ops.unsloth.rope_embedding_rows_qk(dQ, dK, cos, sin, True)+ return dQ, dK, None, None, None++ _rope_embedding_rows_qk_op.register_autograd(+ _rope_embedding_rows_qk_op_backward, setup_context = _rope_rows_qk_setup_context+ )+@torch.library.triton_op("unsloth::rope_embedding_qk", mutates_args = ())def _rope_embedding_qk_op(Q: torch.Tensor,@@ -638,6 +672,11 @@ def _fast_rope_embedding_eager(Q, K, cos, sin, rope_embedding_indices):_fast_rope_embedding_untraced = torch.compiler.disable(_fast_rope_embedding_eager)+# UNSLOTH_COMPILED_ROPE_QK_ONE_LAUNCH=0: compiled RoPE launches once per tensor instead of once for Q and K.+COMPILED_ROPE_QK_ONE_LAUNCH = os.environ.get(+ "UNSLOTH_COMPILED_ROPE_QK_ONE_LAUNCH", "1"+).strip().lower() not in ("0", "false", "no", "off")+# Decode RoPE in one launch instead of ~12 per layer per token; UNSLOTH_DECODE_FUSED_ROPE=0 disables it.DECODE_FUSED_ROPE = os.environ.get("UNSLOTH_DECODE_FUSED_ROPE", "1").strip().lower() not in (@@ -708,6 +747,18 @@ def fast_rope_embedding(rope_embedding_indices.reshape(-1).to(dtype = torch.int32, device = Q.device).contiguous())return torch.ops.unsloth.rope_embedding_qk(Q, K, cos, sin, rope_ptr, False)+ if (+ COMPILED_ROPE_QK_ONE_LAUNCH+ and K.dtype == Q.dtype+ and K.device == Q.device+ and K.shape[0] == Q.shape[0]+ and K.shape[2] == Q.shape[2]+ and K.shape[3] == Q.shape[3]+ ):+ Q_out, K_out = torch.ops.unsloth.rope_embedding_rows_qk(+ Q.transpose(1, 2), K.transpose(1, 2), cos, sin, False+ )+ return Q_out.transpose(1, 2), K_out.transpose(1, 2)Q_out = torch.ops.unsloth.rope_embedding(Q.transpose(1, 2), cos, sin, False).transpose(1, 2)K_out = torch.ops.unsloth.rope_embedding(K.transpose(1, 2), cos, sin, False).transpose(1, 2)return Q_out, K_out