Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fast decode: rotate Q and K in one launch, and fuse Qwen3 q/k norm with RoPE (#14599)

Daniel Hanc5d885f1ec93
diff --git a/tests/test_decode_fused_rope.py b/tests/test_decode_fused_rope.py
new file mode 100644
index 00000000000..defb388b72c
--- /dev/null
+++ b/tests/test_decode_fused_rope.py
@@ -0,0 +1,227 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Single-token decode RoPE (llama path) and q/k norm + RoPE (qwen3 path) in one launch vs the
+unfused decode op sequence they replace, both against an fp64 reference, plus the fallbacks."""
+
+import pytest
+import torch
+
+if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA device", allow_module_level = True)
+
+pytest.importorskip("triton")
+
+import unsloth # noqa: F401
+from unsloth.kernels import qk_norm_rope as QK
+from unsloth.kernels import rope_embedding as RE
+
+SHAPES = [(1, 16, 8, 128), (4, 16, 8, 128), (8, 32, 8, 64), (3, 14, 2, 64), (5, 8, 8, 256)]
+
+
+class _Norm(torch.nn.Module):
+ def __init__(
+ self,
+ dim,
+ dtype,
+ eps = 1e-6,
+ ):
+ super().__init__()
+ w = 1 + 0.1 * torch.randn(dim, device = "cuda", dtype = dtype)
+ self.weight = torch.nn.Parameter(w, requires_grad = False)
+ self.variance_epsilon = eps
+
+
+def _table(positions, dim, dtype):
+ inv = 1.0 / (1e6 ** (torch.arange(0, dim, 2, device = "cuda").float() / dim))
+ f = torch.outer(torch.arange(positions, device = "cuda").float(), inv)
+ e = torch.cat((f, f), -1)
+ return e.cos().to(dtype), e.sin().to(dtype)
+
+
+def _unfused_norm(norm, X):
+ XX = X.to(torch.float32)
+ var = XX.square().mean(-1, keepdim = True)
+ var += norm.variance_epsilon
+ XX *= var.rsqrt_()
+ X = XX.to(X.dtype)
+ X *= norm.weight
+ return X
+
+
+def _unfused_rope(Qn, Kn, cos, sin, pos):
+ # The decode block of LlamaAttention_fast_forward_inference, in place.
+ cos = cos[pos].unsqueeze(1).to(Qn.dtype)
+ sin = sin[pos].unsqueeze(1).to(Qn.dtype)
+ h = Qn.shape[-1] // 2
+ for X in (Qn, Kn):
+ RH = torch.empty_like(X)
+ RH[..., :h] = X[..., h:]
+ RH[..., h:] = X[..., :h]
+ RH[..., :h].neg_()
+ X *= cos
+ X.addcmul_(RH, sin)
+ return Qn, Kn
+
+
+def _oracle_rope(X, cos, sin, pos):
+ c, s = cos[pos].unsqueeze(1).double(), sin[pos].unsqueeze(1).double()
+ h = X.shape[-1] // 2
+ return X * c + torch.cat((-X[..., h:], X[..., :h]), -1) * s
+
+
+def _oracle_norm(norm, X):
+ X = X.double()
+ return (
+ X
+ * torch.rsqrt(X.square().mean(-1, keepdim = True) + norm.variance_epsilon)
+ * norm.weight.double()
+ )
+
+
+def _err(a, b):
+ return (a.double() - b).abs().mean().item()
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("shape", SHAPES)
+def test_decode_rope_inplace_matches_unfused(dtype, shape):
+ torch.manual_seed(0)
+ bsz, nq, nkv, d = shape
+ cos, sin = _table(4096, d, dtype)
+ pos = torch.randint(0, 4000, (bsz, 1), device = "cuda")
+ qbuf = 3 * torch.randn(bsz, 1, nq * d, device = "cuda", dtype = dtype)
+ kbuf = 3 * torch.randn(bsz, 1, nkv * d, device = "cuda", dtype = dtype)
+ view = lambda b, n: b.view(bsz, 1, n, d).transpose(1, 2) # noqa: E731
+ Qo, Ko = (
+ _oracle_rope(view(qbuf, nq).double(), cos, sin, pos),
+ _oracle_rope(view(kbuf, nkv).double(), cos, sin, pos),
+ )
+ Qu, Ku = _unfused_rope(view(qbuf, nq).clone(), view(kbuf, nkv).clone(), cos, sin, pos)
+ qf, kf = qbuf.clone(), kbuf.clone()
+ assert RE.decode_rope_inplace(view(qf, nq), view(kf, nkv), cos, sin, pos)
+ Qf, Kf = view(qf, nq), view(kf, nkv)
+ ulp = torch.finfo(dtype).eps * max(Qo.abs().max().item(), Ko.abs().max().item())
+ assert (Qf - Qu).abs().max().item() <= 2 * ulp and (Kf - Ku).abs().max().item() <= 2 * ulp
+ # mean error vs fp64 does not grow (fewer roundings, not more)
+ assert _err(Qf, Qo) <= _err(Qu, Qo) * 1.05 and _err(Kf, Ko) <= _err(Ku, Ko) * 1.05
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("shape", SHAPES)
+def test_decode_qk_norm_rope_matches_unfused(dtype, shape):
+ torch.manual_seed(0)
+ bsz, nq, nkv, d = shape
+ cos, sin = _table(4096, d, dtype)
+ pos = torch.randint(0, 4000, (bsz, 1), device = "cuda")
+ Q = 3 * torch.randn(bsz, 1, nq, d, device = "cuda", dtype = dtype)
+ K = 3 * torch.randn(bsz, 1, nkv, d, device = "cuda", dtype = dtype)
+ qn, kn = _Norm(d, dtype), _Norm(d, dtype)
+ Qo = _oracle_rope(_oracle_norm(qn, Q).transpose(1, 2), cos, sin, pos)
+ Ko = _oracle_rope(_oracle_norm(kn, K).transpose(1, 2), cos, sin, pos)
+ Qu, Ku = _unfused_rope(
+ _unfused_norm(qn, Q.clone()).transpose(1, 2),
+ _unfused_norm(kn, K.clone()).transpose(1, 2),
+ cos,
+ sin,
+ pos,
+ )
+ out = QK.decode_qk_norm_rope(qn, kn, Q, K, cos, sin, pos)
+ assert out is not None
+ Qf, Kf = out
+ assert Qf.shape == Qu.shape and Kf.shape == Ku.shape
+ ulp = torch.finfo(dtype).eps * max(Qo.abs().max().item(), Ko.abs().max().item())
+ assert (Qf - Qu).abs().max().item() <= 2 * ulp and (Kf - Ku).abs().max().item() <= 2 * ulp
+ assert _err(Qf, Qo) <= _err(Qu, Qo) * 1.05 and _err(Kf, Ko) <= _err(Ku, Ko) * 1.05
+
+
+def test_decode_fallbacks():
+ cos, sin = _table(64, 128, torch.bfloat16)
+ q = torch.randn(2, 4, 1, 128, device = "cuda", dtype = torch.bfloat16)
+ k = torch.randn(2, 2, 1, 128, device = "cuda", dtype = torch.bfloat16)
+ p = torch.zeros(2, 1, dtype = torch.long, device = "cuda")
+ before = q.clone()
+ # a float32 table: the unfused path multiplies in Q's dtype, so it keeps that path
+ assert not RE.decode_rope_inplace(q, k, cos.float(), sin.float(), p)
+ assert torch.equal(q, before)
+ assert not RE.decode_rope_inplace(
+ torch.randn(2, 4, 3, 128, device = "cuda", dtype = torch.bfloat16), k, cos, sin, p
+ )
+ assert not RE.decode_rope_inplace(q.cpu(), k.cpu(), cos.cpu(), sin.cpu(), p.cpu())
+ n32 = _Norm(128, torch.float32)
+ assert (
+ QK.decode_qk_norm_rope(n32, n32, q.transpose(1, 2), k.transpose(1, 2), cos, sin, p) is None
+ )
+
+
+def test_decode_kill_switch(monkeypatch):
+ cos, sin = _table(64, 128, torch.bfloat16)
+ q = torch.randn(2, 4, 1, 128, device = "cuda", dtype = torch.bfloat16)
+ k = torch.randn(2, 2, 1, 128, device = "cuda", dtype = torch.bfloat16)
+ p = torch.zeros(2, 1, dtype = torch.long, device = "cuda")
+ n = _Norm(128, torch.bfloat16)
+ monkeypatch.setattr(RE, "DECODE_FUSED_ROPE", False)
+ monkeypatch.setattr(QK, "DECODE_FUSED_ROPE", False)
+ assert not RE.decode_rope_inplace(q, k, cos, sin, p)
+ assert QK.decode_qk_norm_rope(n, n, q.transpose(1, 2), k.transpose(1, 2), cos, sin, p) is None
+
+
+def test_decode_strided_int32_positions():
+ # int32 last-column slices keep their row stride; the kernels index positions densely.
+ torch.manual_seed(0)
+ dtype, d = torch.bfloat16, 128
+ cos, sin = _table(64, d, dtype)
+ pos = torch.tensor([[0, 1, 2], [2, 3, 4], [9, 10, 11]], device = "cuda", dtype = torch.int32)[
+ :, -1:
+ ]
+ assert pos.stride(0) == 3
+ Q = 3 * torch.randn(3, 4, 1, d, device = "cuda", dtype = dtype)
+ K = 3 * torch.randn(3, 2, 1, d, device = "cuda", dtype = dtype)
+ Qu, Ku = _unfused_rope(Q.clone(), K.clone(), cos, sin, pos.long())
+ Qf, Kf = Q.clone(), K.clone()
+ assert RE.decode_rope_inplace(Qf, Kf, cos, sin, pos)
+ assert torch.allclose(Qf.float(), Qu.float(), atol = 0.05) and torch.allclose(
+ Kf.float(), Ku.float(), atol = 0.05
+ )
+ n = _Norm(d, dtype)
+ Qs, Ks = Q.transpose(1, 2).contiguous(), K.transpose(1, 2).contiguous()
+ Qu, Ku = _unfused_rope(
+ _unfused_norm(n, Qs.clone()).transpose(1, 2),
+ _unfused_norm(n, Ks.clone()).transpose(1, 2),
+ cos,
+ sin,
+ pos.long(),
+ )
+ out = QK.decode_qk_norm_rope(n, n, Qs, Ks, cos, sin, pos)
+ assert out is not None
+ assert torch.allclose(out[0].float(), Qu.float(), atol = 0.05) and torch.allclose(
+ out[1].float(), Ku.float(), atol = 0.05
+ )
+
+
+def test_decode_split_sync_and_mixed_devices(monkeypatch):
+ # Split models: both helpers drain the source stream like the other eager RoPE entry points.
+ cos, sin = _table(64, 128, torch.bfloat16)
+ q = torch.randn(2, 4, 1, 128, device = "cuda", dtype = torch.bfloat16)
+ k = torch.randn(2, 2, 1, 128, device = "cuda", dtype = torch.bfloat16)
+ p = torch.zeros(2, 1, dtype = torch.long, device = "cuda")
+ n = _Norm(128, torch.bfloat16)
+ calls = []
+ monkeypatch.setattr(RE, "_sync_if_split", lambda X: calls.append(X.device))
+ monkeypatch.setattr(QK, "_sync_if_split", lambda X: calls.append(X.device))
+ assert RE.decode_rope_inplace(q, k, cos, sin, p)
+ assert (
+ QK.decode_qk_norm_rope(n, n, q.transpose(1, 2), k.transpose(1, 2), cos, sin, p) is not None
+ )
+ assert len(calls) == 2
+ # any operand off Q's device keeps the unfused path, with nothing written
+ before_q, before_k = q.clone(), k.clone()
+ assert not RE.decode_rope_inplace(q, k, cos, sin.cpu(), p)
+ assert torch.equal(q, before_q) and torch.equal(k, before_k)
+ nc = _Norm(128, torch.bfloat16)
+ nc.weight.data = nc.weight.data.cpu()
+ qs, ks = q.transpose(1, 2), k.transpose(1, 2)
+ assert QK.decode_qk_norm_rope(n, nc, qs, ks, cos, sin, p) is None
+ assert QK.decode_qk_norm_rope(n, n, qs, ks.cpu(), cos, sin, p) is None
+ assert QK.decode_qk_norm_rope(n, n, qs, ks, cos, sin.cpu(), p) is None
+ assert len(calls) == 2
diff --git a/unsloth/kernels/qk_norm_rope.py b/unsloth/kernels/qk_norm_rope.py
index e481f119e2f..a1d2d7d6164 100644
--- a/unsloth/kernels/qk_norm_rope.py
+++ b/unsloth/kernels/qk_norm_rope.py
@@ -18,11 +18,12 @@ from .rms_layernorm import (
_traced_kernel,
fast_rms_layernorm,
)
-from .rope_embedding import fast_rope_embedding, _sync_if_split
+from .rope_embedding import fast_rope_embedding, _sync_if_split, DECODE_FUSED_ROPE
__all__ = [
"fast_qk_norm_rope",
"QK_NORM_ROPE_ENABLED",
+ "decode_qk_norm_rope",
]
QK_NORM_ROPE_ENABLED = os.environ.get("UNSLOTH_FUSED_QKNORM_ROPE", "0") == "1"
@@ -455,3 +456,51 @@ def fast_qk_norm_rope(
else:
Qo, Ko, _, _ = torch.ops.unsloth.qk_norm_rope(*args)
return Qo.transpose(1, 2), Ko.transpose(1, 2)
+
+
+def decode_qk_norm_rope(q_norm, k_norm, Q, K, cos, sin, position_ids):
+ """Decode q/k norm + RoPE in one launch; returns new [bsz, n_heads, 1, head_dim] Q, K or None.
+ UNSLOTH_DECODE_FUSED_ROPE=0 disables it."""
+ if not DECODE_FUSED_ROPE or torch.compiler.is_compiling():
+ return None
+ Wq, Wk = q_norm.weight, k_norm.weight
+ if Q.device.type != "cuda" or Q.dim() != 4 or K.dim() != 4:
+ return None
+ batch, seq_len, _, head_dim = Q.shape
+ if seq_len != 1 or K.shape[0] != batch or K.shape[1] != 1 or K.shape[3] != head_dim:
+ return None
+ if position_ids is None or position_ids.numel() != batch:
+ return None
+ if torch.is_grad_enabled() and (
+ Q.requires_grad or K.requires_grad or Wq.requires_grad or Wk.requires_grad
+ ):
+ return None
+ cos2, sin2 = cos.squeeze(), sin.squeeze()
+ if (
+ head_dim % 2 != 0
+ or head_dim & (head_dim - 1) != 0
+ or head_dim < 16
+ or head_dim > 1024
+ or cos2.dim() != 2
+ or cos2.shape[-1] != head_dim
+ or sin2.shape != cos2.shape
+ or cos2.stride(-1) != 1
+ or sin2.stride(-1) != 1
+ or Q.stride(-1) != 1
+ or K.stride(-1) != 1
+ or Wq.shape != (head_dim,)
+ or Wk.shape != (head_dim,)
+ # Matches the unfused rounding only when every operand shares Q's dtype.
+ or not (Q.dtype == K.dtype == Wq.dtype == Wk.dtype == cos2.dtype == sin2.dtype)
+ or not (Q.device == K.device == Wq.device == Wk.device == cos2.device == sin2.device)
+ ):
+ return None
+ # The kernel indexes positions densely; an int32 last-column slice keeps its row stride.
+ rope_indices = position_ids.reshape(-1).to(dtype = torch.int32, device = Q.device).contiguous()
+ with torch_gpu_device(Q.device):
+ Qo, Ko, _, _ = _launch_forward(
+ Q, K, Wq.contiguous(), Wk.contiguous(), cos2, sin2, rope_indices,
+ float(_eps(q_norm)), float(_eps(k_norm)), True, _eager_kernel,
+ ) # fmt: skip
+ _sync_if_split(Q)
+ return Qo.transpose(1, 2), Ko.transpose(1, 2)
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index f7363566aba..a057e1bd431 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -11,6 +11,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import functools
+import os
import weakref
import triton
import triton.language as tl
@@ -635,6 +636,54 @@ def _fast_rope_embedding_eager(Q, K, cos, sin, rope_embedding_indices):
_fast_rope_embedding_untraced = torch.compiler.disable(_fast_rope_embedding_eager)
+# Decode RoPE in one launch instead of ~12 per layer per token; UNSLOTH_DECODE_FUSED_ROPE=0 disables it.
+DECODE_FUSED_ROPE = os.environ.get("UNSLOTH_DECODE_FUSED_ROPE", "1").strip().lower() not in (
+ "0",
+ "false",
+ "no",
+ "off",
+)
+
+
+def decode_rope_inplace(Q, K, cos, sin, position_ids):
+ """Rotate decode Q, K in place; False (nothing written) where the kernel does not apply."""
+ if not DECODE_FUSED_ROPE or torch.compiler.is_compiling():
+ return False
+ if Q.device.type != "cuda" or K.device != Q.device or Q.dim() != 4 or K.dim() != 4:
+ return False
+ batch, _, seq_len, head_dim = Q.shape
+ if (
+ seq_len != 1
+ or K.shape[0] != batch
+ or K.shape[2] != 1
+ or K.shape[3] != head_dim
+ or head_dim % 2
+ ):
+ return False
+ if position_ids is None or position_ids.numel() != batch:
+ return False
+ cos2, sin2 = cos.squeeze(), sin.squeeze()
+ if (
+ cos2.dim() != 2
+ or cos2.shape != sin2.shape
+ or cos2.shape[-1] != head_dim
+ or cos2.dtype != Q.dtype # the unfused path multiplies in Q's dtype
+ or sin2.dtype != Q.dtype
+ or K.dtype != Q.dtype
+ or cos2.device != Q.device
+ or sin2.device != Q.device
+ or cos2.stride(-1) != 1
+ or sin2.stride(-1) != 1
+ ):
+ return False
+ # The kernel indexes positions densely; an int32 last-column slice keeps its row stride.
+ rope_ptr = position_ids.reshape(-1).to(dtype = torch.int32, device = Q.device).contiguous()
+ with torch_gpu_device(Q.device):
+ _rope_qk(Q, K, Q, K, cos2, sin2, rope_ptr, True, False, _eager_kernel)
+ _sync_if_split(Q)
+ return True
+
+
def fast_rope_embedding(
Q,
K,
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index e39aa7da9d4..e4225421596 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -150,6 +150,7 @@ from unsloth.models._attn_mask_compat import (
from transformers.cache_utils import DynamicCache, Cache
from ..kernels import *
from ..kernels.utils import has_mxfp4_base
+from ..kernels.rope_embedding import decode_rope_inplace
from ..kernels.fused_add_rms_layernorm import (
fast_add_rms_layernorm_inference,
fast_add_rms_layernorm_inference_ok,
@@ -763,24 +764,26 @@ def LlamaAttention_fast_forward_inference(
self.rotary_emb.extend_rope_embedding(Vn, rotary_seq_len + 1) # +1 slack
cos, sin = self.rotary_emb.get_cached(rotary_seq_len, Qn.device.index or 0)
- cos = cos[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)
- sin = sin[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)
-
- h = self.half_head_dim
-
- RH_Q = self.RH_Q
- RH_Q[:, :, :, :h] = Qn[:, :, :, h:]
- RH_Q[:, :, :, h:] = Qn[:, :, :, :h]
- RH_Q[:, :, :, :h].neg_()
- Qn *= cos
- Qn.addcmul_(RH_Q, sin)
-
- RH_K = RH_Q[:, :n_kv_heads, :, :]
- RH_K[:, :, :, :h] = Kn[:, :, :, h:]
- RH_K[:, :, :, h:] = Kn[:, :, :, :h]
- RH_K[:, :, :, :h].neg_()
- Kn *= cos
- Kn.addcmul_(RH_K, sin)
+ # One in-place launch for both rotations; False (nothing written) where it does not apply.
+ if not decode_rope_inplace(Qn, Kn, cos, sin, position_ids):
+ cos = cos[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)
+ sin = sin[position_ids].unsqueeze(1).to(device = Qn.device, dtype = Qn.dtype)
+
+ h = self.half_head_dim
+
+ RH_Q = self.RH_Q
+ RH_Q[:, :, :, :h] = Qn[:, :, :, h:]
+ RH_Q[:, :, :, h:] = Qn[:, :, :, :h]
+ RH_Q[:, :, :, :h].neg_()
+ Qn *= cos
+ Qn.addcmul_(RH_Q, sin)
+
+ RH_K = RH_Q[:, :n_kv_heads, :, :]
+ RH_K[:, :, :, :h] = Kn[:, :, :, h:]
+ RH_K[:, :, :, h:] = Kn[:, :, :, :h]
+ RH_K[:, :, :, :h].neg_()
+ Kn *= cos
+ Kn.addcmul_(RH_K, sin)
self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)
self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index c9e57eef21b..4ffb91e710b 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -16,7 +16,7 @@ from ._utils import __version__
from unsloth_zoo.utils import Version, _get_dtype
from ..utils.packing import get_packed_info_from_kwargs
from ..kernels import qk_norm_rope as _qk_norm_rope
-from ..kernels.qk_norm_rope import fast_qk_norm_rope
+from ..kernels.qk_norm_rope import fast_qk_norm_rope, decode_qk_norm_rope
from ..utils.attention_dispatch import (
AttentionConfig,
AttentionContext,
@@ -254,12 +254,6 @@ def Qwen3Attention_fast_forward_inference(
Kn = Kn.view(bsz, 1, n_kv_heads, head_dim)
Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
- Qn = fast_rms_layernorm_inference(self.q_norm, Qn)
- Kn = fast_rms_layernorm_inference(self.k_norm, Kn)
-
- Qn = Qn.transpose(1, 2)
- Kn = Kn.transpose(1, 2)
-
# Must be done 2 steps before hitting full on a short KV cache, or it errors.
# rotary_seq_len (from the model-level decode loop, no extra sync) covers explicit positions
# past the KV length, since the cache only builds the rows asked for.
@@ -270,23 +264,35 @@ def Qwen3Attention_fast_forward_inference(
# Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.
if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
position_ids = position_ids[:, -1:]
- cos = cos[position_ids].unsqueeze(1)
- sin = sin[position_ids].unsqueeze(1)
- h = self.half_head_dim
-
- RH_Q = self.RH_Q
- RH_Q[:, :, :, :h] = Qn[:, :, :, h:]
- RH_Q[:, :, :, h:] = Qn[:, :, :, :h]
- RH_Q[:, :, :, :h].neg_()
- Qn *= cos
- Qn.addcmul_(RH_Q, sin)
-
- RH_K = RH_Q[:, :n_kv_heads, :, :]
- RH_K[:, :, :, :h] = Kn[:, :, :, h:]
- RH_K[:, :, :, h:] = Kn[:, :, :, :h]
- RH_K[:, :, :, :h].neg_()
- Kn *= cos
- Kn.addcmul_(RH_K, sin)
+
+ # One launch for both norms and both rotations; None where it does not apply.
+ fused = decode_qk_norm_rope(self.q_norm, self.k_norm, Qn, Kn, cos, sin, position_ids)
+ if fused is not None:
+ Qn, Kn = fused
+ else:
+ Qn = fast_rms_layernorm_inference(self.q_norm, Qn)
+ Kn = fast_rms_layernorm_inference(self.k_norm, Kn)
+
+ Qn = Qn.transpose(1, 2)
+ Kn = Kn.transpose(1, 2)
+
+ cos = cos[position_ids].unsqueeze(1)
+ sin = sin[position_ids].unsqueeze(1)
+ h = self.half_head_dim
+
+ RH_Q = self.RH_Q
+ RH_Q[:, :, :, :h] = Qn[:, :, :, h:]
+ RH_Q[:, :, :, h:] = Qn[:, :, :, :h]
+ RH_Q[:, :, :, :h].neg_()
+ Qn *= cos
+ Qn.addcmul_(RH_Q, sin)
+
+ RH_K = RH_Q[:, :n_kv_heads, :, :]
+ RH_K[:, :, :, :h] = Kn[:, :, :, h:]
+ RH_K[:, :, :, h:] = Kn[:, :, :, :h]
+ RH_K[:, :, :, :h].neg_()
+ Kn *= cos
+ Kn.addcmul_(RH_K, sin)
self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)
self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)