Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fast decode: rotate Q and K in one launch, and fuse Qwen3 q/k norm with RoPE (#14599)
Daniel Hanc5d885f1ec93
diff --git a/tests/test_decode_fused_rope.py b/tests/test_decode_fused_rope.pynew file mode 100644index 00000000000..defb388b72c--- /dev/null+++ b/tests/test_decode_fused_rope.py@@ -0,0 +1,227 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Single-token decode RoPE (llama path) and q/k norm + RoPE (qwen3 path) in one launch vs the+unfused decode op sequence they replace, both against an fp64 reference, plus the fallbacks."""++import pytest+import torch++if not torch.cuda.is_available():+ pytest.skip("needs a CUDA device", allow_module_level = True)++pytest.importorskip("triton")++import unsloth # noqa: F401+from unsloth.kernels import qk_norm_rope as QK+from unsloth.kernels import rope_embedding as RE++SHAPES = [(1, 16, 8, 128), (4, 16, 8, 128), (8, 32, 8, 64), (3, 14, 2, 64), (5, 8, 8, 256)]+++class _Norm(torch.nn.Module):+ def __init__(+ self,+ dim,+ dtype,+ eps = 1e-6,+ ):+ super().__init__()+ w = 1 + 0.1 * torch.randn(dim, device = "cuda", dtype = dtype)+ self.weight = torch.nn.Parameter(w, requires_grad = False)+ self.variance_epsilon = eps+++def _table(positions, dim, dtype):+ inv = 1.0 / (1e6 ** (torch.arange(0, dim, 2, device = "cuda").float() / dim))+ f = torch.outer(torch.arange(positions, device = "cuda").float(), inv)+ e = torch.cat((f, f), -1)+ return e.cos().to(dtype), e.sin().to(dtype)+++def _unfused_norm(norm, X):+ XX = X.to(torch.float32)+ var = XX.square().mean(-1, keepdim = True)+ var += norm.variance_epsilon+ XX *= var.rsqrt_()+ X = XX.to(X.dtype)+ X *= norm.weight+ return X+++def _unfused_rope(Qn, Kn, cos, sin, pos):+ # The decode block of LlamaAttention_fast_forward_inference, in place.+ cos = cos[pos].unsqueeze(1).to(Qn.dtype)+ sin = sin[pos].unsqueeze(1).to(Qn.dtype)+ h = Qn.shape[-1] // 2+ for X in (Qn, Kn):+ RH = torch.empty_like(X)+ RH[..., :h] = X[..., h:]+ RH[..., h:] = X[..., :h]+ RH[..., :h].neg_()+ X *= cos+ X.addcmul_(RH, sin)+ return Qn, Kn+++def _oracle_rope(X, cos, sin, pos):+ c, s = cos[pos].unsqueeze(1).double(), sin[pos].unsqueeze(1).double()+ h = X.shape[-1] // 2+ return X * c + torch.cat((-X[..., h:], X[..., :h]), -1) * s+++def _oracle_norm(norm, X):+ X = X.double()+ return (+ X+ * torch.rsqrt(X.square().mean(-1, keepdim = True) + norm.variance_epsilon)+ * norm.weight.double()+ )+++def _err(a, b):+ return (a.double() - b).abs().mean().item()++[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("shape", SHAPES)+def test_decode_rope_inplace_matches_unfused(dtype, shape):+ torch.manual_seed(0)+ bsz, nq, nkv, d = shape+ cos, sin = _table(4096, d, dtype)+ pos = torch.randint(0, 4000, (bsz, 1), device = "cuda")+ qbuf = 3 * torch.randn(bsz, 1, nq * d, device = "cuda", dtype = dtype)+ kbuf = 3 * torch.randn(bsz, 1, nkv * d, device = "cuda", dtype = dtype)+ view = lambda b, n: b.view(bsz, 1, n, d).transpose(1, 2) # noqa: E731+ Qo, Ko = (+ _oracle_rope(view(qbuf, nq).double(), cos, sin, pos),+ _oracle_rope(view(kbuf, nkv).double(), cos, sin, pos),+ )+ Qu, Ku = _unfused_rope(view(qbuf, nq).clone(), view(kbuf, nkv).clone(), cos, sin, pos)+ qf, kf = qbuf.clone(), kbuf.clone()+ assert RE.decode_rope_inplace(view(qf, nq), view(kf, nkv), cos, sin, pos)+ Qf, Kf = view(qf, nq), view(kf, nkv)+ ulp = torch.finfo(dtype).eps * max(Qo.abs().max().item(), Ko.abs().max().item())+ assert (Qf - Qu).abs().max().item() <= 2 * ulp and (Kf - Ku).abs().max().item() <= 2 * ulp+ # mean error vs fp64 does not grow (fewer roundings, not more)+ assert _err(Qf, Qo) <= _err(Qu, Qo) * 1.05 and _err(Kf, Ko) <= _err(Ku, Ko) * 1.05++[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("shape", SHAPES)+def test_decode_qk_norm_rope_matches_unfused(dtype, shape):+ torch.manual_seed(0)+ bsz, nq, nkv, d = shape+ cos, sin = _table(4096, d, dtype)+ pos = torch.randint(0, 4000, (bsz, 1), device = "cuda")+ Q = 3 * torch.randn(bsz, 1, nq, d, device = "cuda", dtype = dtype)+ K = 3 * torch.randn(bsz, 1, nkv, d, device = "cuda", dtype = dtype)+ qn, kn = _Norm(d, dtype), _Norm(d, dtype)+ Qo = _oracle_rope(_oracle_norm(qn, Q).transpose(1, 2), cos, sin, pos)+ Ko = _oracle_rope(_oracle_norm(kn, K).transpose(1, 2), cos, sin, pos)+ Qu, Ku = _unfused_rope(+ _unfused_norm(qn, Q.clone()).transpose(1, 2),+ _unfused_norm(kn, K.clone()).transpose(1, 2),+ cos,+ sin,+ pos,+ )+ out = QK.decode_qk_norm_rope(qn, kn, Q, K, cos, sin, pos)+ assert out is not None+ Qf, Kf = out+ assert Qf.shape == Qu.shape and Kf.shape == Ku.shape+ ulp = torch.finfo(dtype).eps * max(Qo.abs().max().item(), Ko.abs().max().item())+ assert (Qf - Qu).abs().max().item() <= 2 * ulp and (Kf - Ku).abs().max().item() <= 2 * ulp+ assert _err(Qf, Qo) <= _err(Qu, Qo) * 1.05 and _err(Kf, Ko) <= _err(Ku, Ko) * 1.05+++def test_decode_fallbacks():+ cos, sin = _table(64, 128, torch.bfloat16)+ q = torch.randn(2, 4, 1, 128, device = "cuda", dtype = torch.bfloat16)+ k = torch.randn(2, 2, 1, 128, device = "cuda", dtype = torch.bfloat16)+ p = torch.zeros(2, 1, dtype = torch.long, device = "cuda")+ before = q.clone()+ # a float32 table: the unfused path multiplies in Q's dtype, so it keeps that path+ assert not RE.decode_rope_inplace(q, k, cos.float(), sin.float(), p)+ assert torch.equal(q, before)+ assert not RE.decode_rope_inplace(+ torch.randn(2, 4, 3, 128, device = "cuda", dtype = torch.bfloat16), k, cos, sin, p+ )+ assert not RE.decode_rope_inplace(q.cpu(), k.cpu(), cos.cpu(), sin.cpu(), p.cpu())+ n32 = _Norm(128, torch.float32)+ assert (+ QK.decode_qk_norm_rope(n32, n32, q.transpose(1, 2), k.transpose(1, 2), cos, sin, p) is None+ )+++def test_decode_kill_switch(monkeypatch):+ cos, sin = _table(64, 128, torch.bfloat16)+ q = torch.randn(2, 4, 1, 128, device = "cuda", dtype = torch.bfloat16)+ k = torch.randn(2, 2, 1, 128, device = "cuda", dtype = torch.bfloat16)+ p = torch.zeros(2, 1, dtype = torch.long, device = "cuda")+ n = _Norm(128, torch.bfloat16)+ monkeypatch.setattr(RE, "DECODE_FUSED_ROPE", False)+ monkeypatch.setattr(QK, "DECODE_FUSED_ROPE", False)+ assert not RE.decode_rope_inplace(q, k, cos, sin, p)+ assert QK.decode_qk_norm_rope(n, n, q.transpose(1, 2), k.transpose(1, 2), cos, sin, p) is None+++def test_decode_strided_int32_positions():+ # int32 last-column slices keep their row stride; the kernels index positions densely.+ torch.manual_seed(0)+ dtype, d = torch.bfloat16, 128+ cos, sin = _table(64, d, dtype)+ pos = torch.tensor([[0, 1, 2], [2, 3, 4], [9, 10, 11]], device = "cuda", dtype = torch.int32)[+ :, -1:+ ]+ assert pos.stride(0) == 3+ Q = 3 * torch.randn(3, 4, 1, d, device = "cuda", dtype = dtype)+ K = 3 * torch.randn(3, 2, 1, d, device = "cuda", dtype = dtype)+ Qu, Ku = _unfused_rope(Q.clone(), K.clone(), cos, sin, pos.long())+ Qf, Kf = Q.clone(), K.clone()+ assert RE.decode_rope_inplace(Qf, Kf, cos, sin, pos)+ assert torch.allclose(Qf.float(), Qu.float(), atol = 0.05) and torch.allclose(+ Kf.float(), Ku.float(), atol = 0.05+ )+ n = _Norm(d, dtype)+ Qs, Ks = Q.transpose(1, 2).contiguous(), K.transpose(1, 2).contiguous()+ Qu, Ku = _unfused_rope(+ _unfused_norm(n, Qs.clone()).transpose(1, 2),+ _unfused_norm(n, Ks.clone()).transpose(1, 2),+ cos,+ sin,+ pos.long(),+ )+ out = QK.decode_qk_norm_rope(n, n, Qs, Ks, cos, sin, pos)+ assert out is not None+ assert torch.allclose(out[0].float(), Qu.float(), atol = 0.05) and torch.allclose(+ out[1].float(), Ku.float(), atol = 0.05+ )+++def test_decode_split_sync_and_mixed_devices(monkeypatch):+ # Split models: both helpers drain the source stream like the other eager RoPE entry points.+ cos, sin = _table(64, 128, torch.bfloat16)+ q = torch.randn(2, 4, 1, 128, device = "cuda", dtype = torch.bfloat16)+ k = torch.randn(2, 2, 1, 128, device = "cuda", dtype = torch.bfloat16)+ p = torch.zeros(2, 1, dtype = torch.long, device = "cuda")+ n = _Norm(128, torch.bfloat16)+ calls = []+ monkeypatch.setattr(RE, "_sync_if_split", lambda X: calls.append(X.device))+ monkeypatch.setattr(QK, "_sync_if_split", lambda X: calls.append(X.device))+ assert RE.decode_rope_inplace(q, k, cos, sin, p)+ assert (+ QK.decode_qk_norm_rope(n, n, q.transpose(1, 2), k.transpose(1, 2), cos, sin, p) is not None+ )+ assert len(calls) == 2+ # any operand off Q's device keeps the unfused path, with nothing written+ before_q, before_k = q.clone(), k.clone()+ assert not RE.decode_rope_inplace(q, k, cos, sin.cpu(), p)+ assert torch.equal(q, before_q) and torch.equal(k, before_k)+ nc = _Norm(128, torch.bfloat16)+ nc.weight.data = nc.weight.data.cpu()+ qs, ks = q.transpose(1, 2), k.transpose(1, 2)+ assert QK.decode_qk_norm_rope(n, nc, qs, ks, cos, sin, p) is None+ assert QK.decode_qk_norm_rope(n, n, qs, ks.cpu(), cos, sin, p) is None+ assert QK.decode_qk_norm_rope(n, n, qs, ks, cos, sin.cpu(), p) is None+ assert len(calls) == 2diff --git a/unsloth/kernels/qk_norm_rope.py b/unsloth/kernels/qk_norm_rope.pyindex e481f119e2f..a1d2d7d6164 100644--- a/unsloth/kernels/qk_norm_rope.py+++ b/unsloth/kernels/qk_norm_rope.py@@ -18,11 +18,12 @@ from .rms_layernorm import (_traced_kernel,fast_rms_layernorm,)-from .rope_embedding import fast_rope_embedding, _sync_if_split+from .rope_embedding import fast_rope_embedding, _sync_if_split, DECODE_FUSED_ROPE__all__ = ["fast_qk_norm_rope","QK_NORM_ROPE_ENABLED",+ "decode_qk_norm_rope",]QK_NORM_ROPE_ENABLED = os.environ.get("UNSLOTH_FUSED_QKNORM_ROPE", "0") == "1"@@ -455,3 +456,51 @@ def fast_qk_norm_rope(else:Qo, Ko, _, _ = torch.ops.unsloth.qk_norm_rope(*args)return Qo.transpose(1, 2), Ko.transpose(1, 2)+++def decode_qk_norm_rope(q_norm, k_norm, Q, K, cos, sin, position_ids):+ """Decode q/k norm + RoPE in one launch; returns new [bsz, n_heads, 1, head_dim] Q, K or None.+ UNSLOTH_DECODE_FUSED_ROPE=0 disables it."""+ if not DECODE_FUSED_ROPE or torch.compiler.is_compiling():+ return None+ Wq, Wk = q_norm.weight, k_norm.weight+ if Q.device.type != "cuda" or Q.dim() != 4 or K.dim() != 4:+ return None+ batch, seq_len, _, head_dim = Q.shape+ if seq_len != 1 or K.shape[0] != batch or K.shape[1] != 1 or K.shape[3] != head_dim:+ return None+ if position_ids is None or position_ids.numel() != batch:+ return None+ if torch.is_grad_enabled() and (+ Q.requires_grad or K.requires_grad or Wq.requires_grad or Wk.requires_grad+ ):+ return None+ cos2, sin2 = cos.squeeze(), sin.squeeze()+ if (+ head_dim % 2 != 0+ or head_dim & (head_dim - 1) != 0+ or head_dim < 16+ or head_dim > 1024+ or cos2.dim() != 2+ or cos2.shape[-1] != head_dim+ or sin2.shape != cos2.shape+ or cos2.stride(-1) != 1+ or sin2.stride(-1) != 1+ or Q.stride(-1) != 1+ or K.stride(-1) != 1+ or Wq.shape != (head_dim,)+ or Wk.shape != (head_dim,)+ # Matches the unfused rounding only when every operand shares Q's dtype.+ or not (Q.dtype == K.dtype == Wq.dtype == Wk.dtype == cos2.dtype == sin2.dtype)+ or not (Q.device == K.device == Wq.device == Wk.device == cos2.device == sin2.device)+ ):+ return None+ # The kernel indexes positions densely; an int32 last-column slice keeps its row stride.+ rope_indices = position_ids.reshape(-1).to(dtype = torch.int32, device = Q.device).contiguous()+ with torch_gpu_device(Q.device):+ Qo, Ko, _, _ = _launch_forward(+ Q, K, Wq.contiguous(), Wk.contiguous(), cos2, sin2, rope_indices,+ float(_eps(q_norm)), float(_eps(k_norm)), True, _eager_kernel,+ ) # fmt: skip+ _sync_if_split(Q)+ return Qo.transpose(1, 2), Ko.transpose(1, 2)diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex f7363566aba..a057e1bd431 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -11,6 +11,7 @@# along with this program. If not, see <https://www.gnu.org/licenses/>.import functools+import osimport weakrefimport tritonimport triton.language as tl@@ -635,6 +636,54 @@ def _fast_rope_embedding_eager(Q, K, cos, sin, rope_embedding_indices):_fast_rope_embedding_untraced = torch.compiler.disable(_fast_rope_embedding_eager)+# Decode RoPE in one launch instead of ~12 per layer per token; UNSLOTH_DECODE_FUSED_ROPE=0 disables it.+DECODE_FUSED_ROPE = os.environ.get("UNSLOTH_DECODE_FUSED_ROPE", "1").strip().lower() not in (+ "0",+ "false",+ "no",+ "off",+)+++def decode_rope_inplace(Q, K, cos, sin, position_ids):+ """Rotate decode Q, K in place; False (nothing written) where the kernel does not apply."""+ if not DECODE_FUSED_ROPE or torch.compiler.is_compiling():+ return False+ if Q.device.type != "cuda" or K.device != Q.device or Q.dim() != 4 or K.dim() != 4:+ return False+ batch, _, seq_len, head_dim = Q.shape+ if (+ seq_len != 1+ or K.shape[0] != batch+ or K.shape[2] != 1+ or K.shape[3] != head_dim+ or head_dim % 2+ ):+ return False+ if position_ids is None or position_ids.numel() != batch:+ return False+ cos2, sin2 = cos.squeeze(), sin.squeeze()+ if (+ cos2.dim() != 2+ or cos2.shape != sin2.shape+ or cos2.shape[-1] != head_dim+ or cos2.dtype != Q.dtype # the unfused path multiplies in Q's dtype+ or sin2.dtype != Q.dtype+ or K.dtype != Q.dtype+ or cos2.device != Q.device+ or sin2.device != Q.device+ or cos2.stride(-1) != 1+ or sin2.stride(-1) != 1+ ):+ return False+ # The kernel indexes positions densely; an int32 last-column slice keeps its row stride.+ rope_ptr = position_ids.reshape(-1).to(dtype = torch.int32, device = Q.device).contiguous()+ with torch_gpu_device(Q.device):+ _rope_qk(Q, K, Q, K, cos2, sin2, rope_ptr, True, False, _eager_kernel)+ _sync_if_split(Q)+ return True++def fast_rope_embedding(Q,K,diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex e39aa7da9d4..e4225421596 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -150,6 +150,7 @@ from unsloth.models._attn_mask_compat import (from transformers.cache_utils import DynamicCache, Cachefrom ..kernels import *from ..kernels.utils import has_mxfp4_base+from ..kernels.rope_embedding import decode_rope_inplacefrom ..kernels.fused_add_rms_layernorm import (fast_add_rms_layernorm_inference,fast_add_rms_layernorm_inference_ok,@@ -763,24 +764,26 @@ def LlamaAttention_fast_forward_inference(self.rotary_emb.extend_rope_embedding(Vn, rotary_seq_len + 1) # +1 slackcos, sin = self.rotary_emb.get_cached(rotary_seq_len, Qn.device.index or 0)- cos = cos[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)- sin = sin[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)-- h = self.half_head_dim-- RH_Q = self.RH_Q- RH_Q[:, :, :, :h] = Qn[:, :, :, h:]- RH_Q[:, :, :, h:] = Qn[:, :, :, :h]- RH_Q[:, :, :, :h].neg_()- Qn *= cos- Qn.addcmul_(RH_Q, sin)-- RH_K = RH_Q[:, :n_kv_heads, :, :]- RH_K[:, :, :, :h] = Kn[:, :, :, h:]- RH_K[:, :, :, h:] = Kn[:, :, :, :h]- RH_K[:, :, :, :h].neg_()- Kn *= cos- Kn.addcmul_(RH_K, sin)+ # One in-place launch for both rotations; False (nothing written) where it does not apply.+ if not decode_rope_inplace(Qn, Kn, cos, sin, position_ids):+ cos = cos[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)+ sin = sin[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)++ h = self.half_head_dim++ RH_Q = self.RH_Q+ RH_Q[:, :, :, :h] = Qn[:, :, :, h:]+ RH_Q[:, :, :, h:] = Qn[:, :, :, :h]+ RH_Q[:, :, :, :h].neg_()+ Qn *= cos+ Qn.addcmul_(RH_Q, sin)++ RH_K = RH_Q[:, :n_kv_heads, :, :]+ RH_K[:, :, :, :h] = Kn[:, :, :, h:]+ RH_K[:, :, :, h:] = Kn[:, :, :, :h]+ RH_K[:, :, :, :h].neg_()+ Kn *= cos+ Kn.addcmul_(RH_K, sin)self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex c9e57eef21b..4ffb91e710b 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -16,7 +16,7 @@ from ._utils import __version__from unsloth_zoo.utils import Version, _get_dtypefrom ..utils.packing import get_packed_info_from_kwargsfrom ..kernels import qk_norm_rope as _qk_norm_rope-from ..kernels.qk_norm_rope import fast_qk_norm_rope+from ..kernels.qk_norm_rope import fast_qk_norm_rope, decode_qk_norm_ropefrom ..utils.attention_dispatch import (AttentionConfig,AttentionContext,@@ -254,12 +254,6 @@ def Qwen3Attention_fast_forward_inference(Kn = Kn.view(bsz, 1, n_kv_heads, head_dim)Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)- Qn = fast_rms_layernorm_inference(self.q_norm, Qn)- Kn = fast_rms_layernorm_inference(self.k_norm, Kn)-- Qn = Qn.transpose(1, 2)- Kn = Kn.transpose(1, 2)-# Must be done 2 steps before hitting full on a short KV cache, or it errors.# rotary_seq_len (from the model-level decode loop, no extra sync) covers explicit positions# past the KV length, since the cache only builds the rows asked for.@@ -270,23 +264,35 @@ def Qwen3Attention_fast_forward_inference(# Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:position_ids = position_ids[:, -1:]- cos = cos[position_ids].unsqueeze(1)- sin = sin[position_ids].unsqueeze(1)- h = self.half_head_dim-- RH_Q = self.RH_Q- RH_Q[:, :, :, :h] = Qn[:, :, :, h:]- RH_Q[:, :, :, h:] = Qn[:, :, :, :h]- RH_Q[:, :, :, :h].neg_()- Qn *= cos- Qn.addcmul_(RH_Q, sin)-- RH_K = RH_Q[:, :n_kv_heads, :, :]- RH_K[:, :, :, :h] = Kn[:, :, :, h:]- RH_K[:, :, :, h:] = Kn[:, :, :, :h]- RH_K[:, :, :, :h].neg_()- Kn *= cos- Kn.addcmul_(RH_K, sin)++ # One launch for both norms and both rotations; None where it does not apply.+ fused = decode_qk_norm_rope(self.q_norm, self.k_norm, Qn, Kn, cos, sin, position_ids)+ if fused is not None:+ Qn, Kn = fused+ else:+ Qn = fast_rms_layernorm_inference(self.q_norm, Qn)+ Kn = fast_rms_layernorm_inference(self.k_norm, Kn)++ Qn = Qn.transpose(1, 2)+ Kn = Kn.transpose(1, 2)++ cos = cos[position_ids].unsqueeze(1)+ sin = sin[position_ids].unsqueeze(1)+ h = self.half_head_dim++ RH_Q = self.RH_Q+ RH_Q[:, :, :, :h] = Qn[:, :, :, h:]+ RH_Q[:, :, :, h:] = Qn[:, :, :, :h]+ RH_Q[:, :, :, :h].neg_()+ Qn *= cos+ Qn.addcmul_(RH_Q, sin)++ RH_K = RH_Q[:, :n_kv_heads, :, :]+ RH_K[:, :, :, :h] = Kn[:, :, :, h:]+ RH_K[:, :, :, h:] = Kn[:, :, :, :h]+ RH_K[:, :, :, :h].neg_()+ Kn *= cos+ Kn.addcmul_(RH_K, sin)self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)