FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
perf(kernel): split triton decode attention by context length and sm count (#729)
Xiaoze Fan11c1adb88f58
diff --git a/python/freetoken/attention/triton.py b/python/freetoken/attention/triton.pyindex 30bedf4f..7f29c481 100644--- a/python/freetoken/attention/triton.py+++ b/python/freetoken/attention/triton.py@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Listimport torchfrom freetoken.core import Batch, get_global_ctx+from freetoken.kernel.triton.attention import decode_kv_split_capfrom .base import AttentionSpec, BaseAttnBackend, BaseAttnMetadatafrom .utils import BaseCaptureData@@ -16,9 +17,9 @@ if TYPE_CHECKING:@dataclassclass TritonCaptureData(BaseCaptureData):q_to_req: torch.Tensor+ # flat decode split scratch: each graph batch size views it with its own split countattn_logits: torch.Tensorattn_lse: torch.Tensor- num_kv_splits: torch.Tensorswa_page_table: torch.Tensor | None = None@classmethod@@ -30,7 +31,7 @@ class TritonCaptureData(BaseCaptureData):*,num_q_heads: int,max_head_dim: int,- max_kv_splits: int,+ scratch_rows: int,**kwargs,):return cls(@@ -40,22 +41,8 @@ class TritonCaptureData(BaseCaptureData):cu_seqlens_q=torch.arange(0, max_bs + 1, dtype=torch.int32, device=device),page_table=torch.zeros((max_bs, max_seq_len), dtype=torch.int32, device=device),q_to_req=torch.arange(max_bs, dtype=torch.int32, device=device),- attn_logits=torch.empty(- (max_bs, num_q_heads, max_kv_splits, max_head_dim),- dtype=torch.float32,- device=device,- ),- attn_lse=torch.empty(- (max_bs, num_q_heads, max_kv_splits),- dtype=torch.float32,- device=device,- ),- num_kv_splits=torch.full(- (max_bs,),- max_kv_splits,- dtype=torch.int32,- device=device,- ),+ attn_logits=torch.empty((scratch_rows * num_q_heads * max_head_dim,), dtype=torch.float32, device=device),+ attn_lse=torch.empty((scratch_rows * num_q_heads,), dtype=torch.float32, device=device),**kwargs,)@@ -72,7 +59,6 @@ class TritonMetadata(BaseAttnMetadata):max_q_len: intattn_logits: torch.Tensor | None = Noneattn_lse: torch.Tensor | None = None- num_kv_splits: torch.Tensor | None = Noneswa_indices: torch.Tensor | None = Nonedef get_last_indices(self, bs: int) -> torch.Tensor:@@ -87,7 +73,6 @@ class TritonAttentionBackend(BaseAttnBackend):self.capture: TritonCaptureData | None = Noneself.capture_bs: List[int] = []self.max_graph_bs = 0- self.max_kv_splits = 8self.prefill_tile_min_q = 128self.num_q_heads = int(getattr(config, "num_qo_heads", 1))kv_groups = getattr(config, "kv_cache_group_specs", lambda: ())()@@ -95,6 +80,11 @@ class TritonAttentionBackend(BaseAttnBackend):(group.head_dim for group in kv_groups),default=int(getattr(config, "head_dim", 1)),)+ self.kv_heads = sorted({group.num_kv_heads for group in kv_groups}) or [int(getattr(config, "num_kv_heads", 1))]++ def _split_cap(self, bs: int) -> int:+ """The most decode KV splits any layer group takes at this batch size."""+ return max(decode_kv_split_cap(bs, max(1, self.num_q_heads), heads, self.device.index) for heads in self.kv_heads)def _ensure_decode_scratch(self,@@ -103,32 +93,27 @@ class TritonAttentionBackend(BaseAttnBackend):num_q_heads: int,head_dim: int,) -> None:+ splits = self._split_cap(bs)if (metadata.attn_logits is not Noneand metadata.attn_lse is not None- and metadata.num_kv_splits is not Noneand metadata.attn_logits.shape[0] >= bsand metadata.attn_logits.shape[1] >= num_q_heads+ and metadata.attn_logits.shape[2] >= splitsand metadata.attn_logits.shape[3] >= head_dim):returnscratch_head_dim = max(self.max_head_dim, head_dim)metadata.attn_logits = torch.empty(- (bs, num_q_heads, self.max_kv_splits, scratch_head_dim),+ (bs, num_q_heads, splits, scratch_head_dim),dtype=torch.float32,device=self.device,)metadata.attn_lse = torch.empty(- (bs, num_q_heads, self.max_kv_splits),+ (bs, num_q_heads, splits),dtype=torch.float32,device=self.device,)- metadata.num_kv_splits = torch.full(- (bs,),- self.max_kv_splits,- dtype=torch.int32,- device=self.device,- )def forward(self,@@ -167,7 +152,6 @@ class TritonAttentionBackend(BaseAttnBackend):self._ensure_decode_scratch(metadata, bs, q.shape[1], q.shape[-1])assert metadata.attn_logits is not Noneassert metadata.attn_lse is not None- assert metadata.num_kv_splits is not Nonereturn decode_paged_attention(q=q,k_cache=k_cache,@@ -177,8 +161,6 @@ class TritonAttentionBackend(BaseAttnBackend):q_positions=metadata.q_positions,attn_logits=metadata.attn_logits[:bs],attn_lse=metadata.attn_lse[:bs],- num_kv_splits=metadata.num_kv_splits[:bs],- max_kv_splits=self.max_kv_splits,sm_scale=scale,sliding_window=spec.sliding_window,sinks=spec.sinks,@@ -280,7 +262,7 @@ class TritonAttentionBackend(BaseAttnBackend):self.device,num_q_heads=max(1, self.num_q_heads),max_head_dim=max(1, self.max_head_dim),- max_kv_splits=self.max_kv_splits,+ scratch_rows=max(bs * self._split_cap(bs) for bs in bs_list),)if self._swa_capture_enabled():self.capture.swa_page_table = torch.zeros(@@ -324,14 +306,20 @@ class TritonAttentionBackend(BaseAttnBackend):metadata.indices = indicesmetadata.q_to_req = self.capture.q_to_req[: metadata.q_to_req.numel()]metadata.q_positions = self.capture.positions[:q_tokens]- metadata.attn_logits = self.capture.attn_logits[:bs]- metadata.attn_lse = self.capture.attn_lse[:bs]- metadata.num_kv_splits = self.capture.num_kv_splits[:bs]+ metadata.attn_logits, metadata.attn_lse = self._capture_scratch(bs)++ def _capture_scratch(self, bs: int) -> tuple[torch.Tensor, torch.Tensor]:+ assert self.capture is not None+ splits, heads, dim = self._split_cap(bs), max(1, self.num_q_heads), max(1, self.max_head_dim)+ logits = self.capture.attn_logits[: bs * heads * splits * dim].view(bs, heads, splits, dim)+ lse = self.capture.attn_lse[: bs * heads * splits].view(bs, heads, splits)+ return logits, lsedef prepare_for_capture(self, batch: Batch) -> None:bs = batch.sizeassert bs in self.capture_bs and self.capture is not Nonecapture = self.capture+ attn_logits, attn_lse = self._capture_scratch(bs)batch.attn_metadata = TritonMetadata(cu_seqlens_q_gpu=capture.cu_seqlens_q[: bs + 1],indptr=capture.cu_seqlens_k[: bs + 1],@@ -341,9 +329,8 @@ class TritonAttentionBackend(BaseAttnBackend):is_decode=True,prefix_lens=capture.positions[:bs],max_q_len=1,- attn_logits=capture.attn_logits[:bs],- attn_lse=capture.attn_lse[:bs],- num_kv_splits=capture.num_kv_splits[:bs],+ attn_logits=attn_logits,+ attn_lse=attn_lse,swa_indices=self._capture_swa_indices(),)diff --git a/python/freetoken/kernel/triton/attention.py b/python/freetoken/kernel/triton/attention.pyindex c9087e03..d19457cf 100644--- a/python/freetoken/kernel/triton/attention.py+++ b/python/freetoken/kernel/triton/attention.py@@ -7,10 +7,28 @@ import tritonimport triton.language as tl-_MAX_KV_SPLITS = 8_MIN_BLOCK_KV = 32+# a long decode request takes splits of at least this many keys, so the partial result each one writes stays small next to the keys it reads+_MIN_SPLIT_LEN = 128+# a short request still takes up to this many splits of _MIN_BLOCK_KV keys, so tiny requests stay parallel+_MIN_SPLITS = 8+_MAX_KV_SPLITS = 64+# stage 2 merges at most this many fp32 partial elements per step+_STAGE2_TILE_ELEMS = 8192+++def _stage2_launch(max_kv_splits: int, block_dv: int) -> tuple[int, int]:+ """(splits merged per step, num_warps) for stage 2; the tile is at most the split cap rounded up to a power of two."""+ tile = min(triton.next_power_of_2(max_kv_splits), max(4, _STAGE2_TILE_ELEMS // block_dv), 64)+ return tile, max(1, min(8, tile * block_dv // 2048))+++def _max_request_splits(keys: int) -> int:+ return max(triton.cdiv(keys, _MIN_SPLIT_LEN), min(_MIN_SPLITS, triton.cdiv(keys, _MIN_BLOCK_KV)), 1)++@functools.lru_cache(maxsize=None)def _optin_smem_bytes(device_index: int) -> int:"""Per-block opt-in shared-memory budget for a CUDA device (0 if unavailable)."""@@ -23,6 +41,12 @@ def _num_sms(device_index: int) -> int:return int(torch.cuda.get_device_properties(device_index).multi_processor_count)+def decode_kv_split_cap(batch: int, num_q_heads: int, num_kv_heads: int, device_index: int) -> int:+ """Most KV splits per request for a decode batch: about four programs per SM, within [_MIN_SPLITS, _MAX_KV_SPLITS]."""+ head_blocks = triton.cdiv(num_q_heads, min(16, max(1, num_q_heads // num_kv_heads)))+ return max(_MIN_SPLITS, min(_MAX_KV_SPLITS, triton.cdiv(4 * _num_sms(device_index), batch * head_blocks)))++def _select_extend_tile(head_dim: int, block_d: int, smem_optin: int) -> tuple[int, int]:"""Pick ``(BLOCK_M, BLOCK_N)`` for the extend/prefill kernel, shared-memory aware.@@ -150,6 +174,13 @@ def _paged_attention_kernel(@triton.jit+def _decode_num_splits(effective_len, max_kv_splits, MIN_SPLIT_LEN: tl.constexpr, MIN_SPLITS: tl.constexpr, MIN_BLOCK_KV: tl.constexpr):+ # same rule as _max_request_splits: long requests split by length, short ones still take up to MIN_SPLITS small splits+ by_len = tl.maximum(tl.cdiv(effective_len, MIN_SPLIT_LEN), tl.minimum(MIN_SPLITS, tl.cdiv(effective_len, MIN_BLOCK_KV)))+ return tl.minimum(tl.maximum(by_len, 1), max_kv_splits)++[email protected](do_not_specialize=["max_kv_splits"])def _decode_grouped_stage1_kernel(q_ptr,k_ptr,@@ -160,7 +191,7 @@ def _decode_grouped_stage1_kernel(q_pos_ptr,mid_o_ptr,mid_lse_ptr,- num_kv_splits_ptr,+ max_kv_splits,stride_qt,stride_qh,stride_ks,@@ -181,6 +212,8 @@ def _decode_grouped_stage1_kernel(BLOCK_H: tl.constexpr,VALID_BLOCK_H: tl.constexpr,MIN_BLOCK_KV: tl.constexpr,+ MIN_SPLIT_LEN: tl.constexpr,+ MIN_SPLITS: tl.constexpr,D: tl.constexpr,DV: tl.constexpr,SLIDING_WINDOW: tl.constexpr,@@ -212,7 +245,7 @@ def _decode_grouped_stage1_kernel(effective_start = tl.maximum(0, q_pos - SLIDING_WINDOW + 1)effective_len = tl.maximum(0, effective_end - effective_start)- kv_splits = tl.load(num_kv_splits_ptr + batch_id)+ kv_splits = _decode_num_splits(effective_len, max_kv_splits, MIN_SPLIT_LEN, MIN_SPLITS, MIN_BLOCK_KV)kv_len_per_split = (tl.cdiv(tl.cdiv(effective_len, kv_splits), MIN_BLOCK_KV) * MIN_BLOCK_KV)@@ -275,14 +308,14 @@ def _decode_grouped_stage1_kernel(tl.store(mid_lse_ptr + lse_offsets, m_i + tl.log(l_i), mask=mask_h)[email protected](do_not_specialize=["max_kv_splits"])def _decode_stage2_kernel(mid_o_ptr,mid_lse_ptr,o_ptr,indptr_ptr,q_pos_ptr,- num_kv_splits_ptr,+ max_kv_splits,sinks_ptr,stride_mid_ob,stride_mid_oh,@@ -292,12 +325,14 @@ def _decode_stage2_kernel(stride_lse_s,stride_ot,stride_oh,- MAX_KV_SPLITS: tl.constexpr,MIN_BLOCK_KV: tl.constexpr,+ MIN_SPLIT_LEN: tl.constexpr,+ MIN_SPLITS: tl.constexpr,BLOCK_DV: tl.constexpr,DV: tl.constexpr,SLIDING_WINDOW: tl.constexpr,HAS_SINKS: tl.constexpr,+ SPLIT_TILE: tl.constexpr,):batch_id = tl.program_id(0)q_head = tl.program_id(1)@@ -310,13 +345,14 @@ def _decode_stage2_kernel(effective_start = tl.maximum(0, q_pos - SLIDING_WINDOW + 1)effective_len = tl.maximum(0, effective_end - effective_start)- kv_splits = tl.load(num_kv_splits_ptr + batch_id)+ kv_splits = _decode_num_splits(effective_len, max_kv_splits, MIN_SPLIT_LEN, MIN_SPLITS, MIN_BLOCK_KV)kv_len_per_split = (tl.cdiv(tl.cdiv(effective_len, kv_splits), MIN_BLOCK_KV) * MIN_BLOCK_KV)offs_d = tl.arange(0, BLOCK_DV)mask_d = offs_d < DV+ offs_s = tl.arange(0, SPLIT_TILE)if HAS_SINKS:m_i = tl.load(sinks_ptr + q_head).to(tl.float32)l_i = 1.0@@ -325,26 +361,27 @@ def _decode_stage2_kernel(l_i = 0.0acc = tl.zeros((BLOCK_DV,), dtype=tl.float32)- mid_base = batch_id * stride_mid_ob + q_head * stride_mid_oh + offs_d+ mid_base = batch_id * stride_mid_ob + q_head * stride_mid_ohlse_base = batch_id * stride_lse_b + q_head * stride_lse_h- for split_id in tl.range(0, MAX_KV_SPLITS, num_stages=2):- split_start = kv_len_per_split * split_id- split_end = tl.minimum(split_start + kv_len_per_split, effective_len)-- if split_end > split_start:- partial = tl.load(- mid_o_ptr + mid_base + split_id * stride_mid_os,- mask=mask_d,- other=0.0,- )- partial_lse = tl.load(mid_lse_ptr + lse_base + split_id * stride_lse_s)- m_new = tl.maximum(partial_lse, m_i)- alpha = tl.exp(m_i - m_new)- beta = tl.exp(partial_lse - m_new)- acc = acc * alpha + partial * beta- l_i = l_i * alpha + beta- m_i = m_new+ # one load round trip per SPLIT_TILE splits+ for s0 in range(0, kv_splits, SPLIT_TILE):+ split_id = s0 + offs_s+ valid = (split_id < kv_splits) & (kv_len_per_split * split_id < effective_len)+ partial_lse = tl.load(mid_lse_ptr + lse_base + split_id * stride_lse_s, mask=valid, other=-float("inf"))+ partial = tl.load(+ mid_o_ptr + mid_base + split_id[:, None] * stride_mid_os + offs_d[None, :],+ mask=valid[:, None] & mask_d[None, :],+ other=0.0,+ )+ m_new = tl.maximum(m_i, tl.max(partial_lse, axis=0))+ # a tile of empty splits with no sink leaves m at -inf, where exp(-inf - -inf) is nan+ m_safe = tl.where(m_new == -float("inf"), 0.0, m_new)+ alpha = tl.exp(m_i - m_safe)+ beta = tl.exp(partial_lse - m_safe)+ acc = acc * alpha + tl.sum(partial * beta[:, None], axis=0)+ l_i = l_i * alpha + tl.sum(beta, axis=0)+ m_i = m_newout = tl.where(l_i == 0.0, 0.0, acc / l_i)tl.store(@@ -363,14 +400,17 @@ def decode_paged_attention(q_positions: torch.Tensor,attn_logits: torch.Tensor,attn_lse: torch.Tensor,- num_kv_splits: torch.Tensor,- max_kv_splits: int,sm_scale: float,sliding_window: int | None = None,sinks: torch.Tensor | None = None,out: torch.Tensor | None = None,+ max_kv_splits: int | None = None,) -> torch.Tensor:- """SGLang-style split-k grouped decode attention for one query per request."""+ """SGLang-style split-k grouped decode attention for one query per request.++ Each request takes ``max(ceil(keys / 128), min(8, ceil(keys / 32)))`` splits, capped by ``max_kv_splits`` (default :func:`decode_kv_split_cap`) and by the split dim of ``attn_logits`` / ``attn_lse``.+ The kernels compute the split count on the device, so a captured graph serves any context length.+ """assert q.is_cuda and k_cache.is_cuda and v_cache.is_cudaassert q.dim() == 3 and k_cache.dim() == 3 and v_cache.dim() == 3@@ -382,11 +422,18 @@ def decode_paged_attention(assert num_q_heads % num_kv_heads == 0assert attn_logits.shape[0] >= batchassert attn_logits.shape[1] >= num_q_heads- assert attn_logits.shape[2] >= max_kv_splitsassert attn_logits.shape[3] >= head_dimassert attn_lse.shape[0] >= batchassert attn_lse.shape[1] >= num_q_heads- assert attn_lse.shape[2] >= max_kv_splits+ if sliding_window is not None and sliding_window <= 0:+ # the kernels treat a window <= 0 as no window+ sliding_window = None+ if max_kv_splits is None:+ max_kv_splits = decode_kv_split_cap(batch, num_q_heads, num_kv_heads, q.device.index)+ if sliding_window:+ # no request attends more than the window, so programs past its splits would only exit+ max_kv_splits = min(max_kv_splits, _max_request_splits(sliding_window))+ max_kv_splits = min(max_kv_splits, attn_logits.shape[2], attn_lse.shape[2])if sinks is not None:assert sinks.is_cudaassert sinks.dim() == 1@@ -404,12 +451,17 @@ def decode_paged_attention(block_d = triton.next_power_of_2(head_dim)block_dv = triton.next_power_of_2(head_dim)grid = (batch, triton.cdiv(num_q_heads, valid_block_h), max_kv_splits)- block_n = 32- if sliding_window and head_dim <= 256:+ # a grid that fits in one wave runs about one program per SM, which then needs wider tiles to keep enough loads in flight+ block_n = 64 if grid[0] * grid[1] * grid[2] <= _num_sms(q.device.index) else _MIN_BLOCK_KV+ num_stages = 3+ # stage 1 keeps one K tile and one V tile in shared memory+ if 2 * block_n * block_d * k_cache.element_size() > 0.8 * _optin_smem_bytes(q.device.index):+ block_n = _MIN_BLOCK_KV+ if sliding_window:split_len = triton.cdiv(triton.cdiv(sliding_window, max_kv_splits), _MIN_BLOCK_KV) * _MIN_BLOCK_KV- # A window split of several tiles in a single-wave grid is latency bound; 64-key tiles halve its serial steps.- if split_len >= 128 and grid[0] * grid[1] * grid[2] <= _num_sms(q.device.index):- block_n = 64+ if split_len <= block_n:+ # a full window's split fits in one tile, where a deeper pipeline only adds overhead+ block_n, num_stages = split_len, 2_decode_grouped_stage1_kernel[grid](q,@@ -421,7 +473,7 @@ def decode_paged_attention(q_positions,attn_logits,attn_lse,- num_kv_splits,+ max_kv_splits,q.stride(0),q.stride(1),k_cache.stride(0),@@ -442,19 +494,22 @@ def decode_paged_attention(BLOCK_H=block_h,VALID_BLOCK_H=valid_block_h,MIN_BLOCK_KV=_MIN_BLOCK_KV,+ MIN_SPLIT_LEN=_MIN_SPLIT_LEN,+ MIN_SPLITS=_MIN_SPLITS,D=head_dim,DV=head_dim,SLIDING_WINDOW=sliding_window or 0,num_warps=4,- num_stages=2,+ num_stages=num_stages,)+ split_tile, stage2_warps = _stage2_launch(max_kv_splits, block_dv)_decode_stage2_kernel[(batch, num_q_heads)](attn_logits,attn_lse,o,indptr,q_positions,- num_kv_splits,+ max_kv_splits,sinks_arg,attn_logits.stride(0),attn_logits.stride(1),@@ -464,14 +519,15 @@ def decode_paged_attention(attn_lse.stride(2),o.stride(0),o.stride(1),- MAX_KV_SPLITS=max_kv_splits,MIN_BLOCK_KV=_MIN_BLOCK_KV,+ MIN_SPLIT_LEN=_MIN_SPLIT_LEN,+ MIN_SPLITS=_MIN_SPLITS,BLOCK_DV=block_dv,DV=head_dim,SLIDING_WINDOW=sliding_window or 0,HAS_SINKS=sinks is not None,- num_warps=4,- num_stages=2,+ SPLIT_TILE=split_tile,+ num_warps=stage2_warps,)return odiff --git a/tests/kernels/test_triton_attention.py b/tests/kernels/test_triton_attention.pyindex a29f621d..60a5e274 100644--- a/tests/kernels/test_triton_attention.py+++ b/tests/kernels/test_triton_attention.py@@ -286,7 +286,6 @@ def test_decode_triton_attention_matches_reference(device=device,)attn_lse = torch.empty(batch, num_q_heads, max_kv_splits, dtype=torch.float32, device=device)- num_kv_splits = torch.full((batch,), max_kv_splits, dtype=torch.int32, device=device)actual = decode_paged_attention(q,@@ -297,8 +296,6 @@ def test_decode_triton_attention_matches_reference(q_positions,attn_logits,attn_lse,- num_kv_splits,- max_kv_splits,sm_scale,sliding_window=sliding_window,)@@ -344,11 +341,10 @@ def test_decode_triton_attention_non_pow2_group(num_q_heads: int, num_kv_heads:batch, num_q_heads, max_kv_splits, head_dim, dtype=torch.float32, device=device)attn_lse = torch.empty(batch, num_q_heads, max_kv_splits, dtype=torch.float32, device=device)- num_kv_splits = torch.full((batch,), max_kv_splits, dtype=torch.int32, device=device)actual = decode_paged_attention(q, k_cache, v_cache, indptr, indices, q_positions,- attn_logits, attn_lse, num_kv_splits, max_kv_splits, sm_scale,+ attn_logits, attn_lse, sm_scale,)expected = _reference_paged_attention(q, k_cache, v_cache, indptr, indices, q_to_req, q_positions, sm_scale, None,@@ -392,7 +388,6 @@ def test_decode_triton_attention_with_sinks_matches_reference():device=device,)attn_lse = torch.empty(batch, num_q_heads, max_kv_splits, dtype=torch.float32, device=device)- num_kv_splits = torch.full((batch,), max_kv_splits, dtype=torch.int32, device=device)actual = decode_paged_attention(q,@@ -403,8 +398,6 @@ def test_decode_triton_attention_with_sinks_matches_reference():q_positions,attn_logits,attn_lse,- num_kv_splits,- max_kv_splits,sm_scale,sinks=sinks,)@@ -424,6 +417,54 @@ def test_decode_triton_attention_with_sinks_matches_reference():torch.testing.assert_close(actual.float(), expected.float(), atol=2e-2, rtol=2e-2)[email protected](not torch.cuda.is_available(), reason="Triton attention needs CUDA")+ ("seq_lens", "num_kv_heads", "sliding_window", "scratch_splits", "use_sinks"),+ [+ ([1, 129, 1000, 8193], 2, None, 64, False),+ ([1, 129, 1000, 8193], 2, None, 8, False),+ ([300, 2000, 70], 8, 300, 64, True),+ ],+)+def test_decode_triton_attention_splits_long_requests(+ seq_lens: list[int],+ num_kv_heads: int,+ sliding_window: int | None,+ scratch_splits: int,+ use_sinks: bool,+):+ """Requests of many splits next to short ones, a scratch smaller than the split cap, and windows that span several splits."""+ from freetoken.kernel.triton.attention import decode_paged_attention++ torch.manual_seed(5)+ device = torch.device("cuda")+ batch = len(seq_lens)+ num_q_heads = 16+ head_dim = 256+ total_kv = sum(seq_lens)+ q = torch.randn(batch, num_q_heads, head_dim, device=device, dtype=torch.bfloat16)+ k_cache = torch.randn(total_kv, num_kv_heads, head_dim, device=device, dtype=torch.bfloat16)+ v_cache = torch.randn(total_kv, num_kv_heads, head_dim, device=device, dtype=torch.bfloat16)+ indptr = torch.tensor([0, *seq_lens], dtype=torch.int32, device=device).cumsum(0).to(torch.int32)+ indices = torch.randperm(total_kv, device=device).to(torch.int32)+ q_positions = torch.tensor([n - 1 for n in seq_lens], dtype=torch.int64, device=device)+ q_to_req = torch.arange(batch, dtype=torch.int32, device=device)+ sinks = torch.randn(num_q_heads, dtype=torch.float32, device=device) if use_sinks else None+ sm_scale = head_dim**-0.5+ attn_logits = torch.empty(batch, num_q_heads, scratch_splits, head_dim, dtype=torch.float32, device=device)+ attn_lse = torch.empty(batch, num_q_heads, scratch_splits, dtype=torch.float32, device=device)++ actual = decode_paged_attention(+ q, k_cache, v_cache, indptr, indices, q_positions,+ attn_logits, attn_lse, sm_scale,+ sliding_window=sliding_window, sinks=sinks,+ )+ expected = _reference_paged_attention(+ q, k_cache, v_cache, indptr, indices, q_to_req, q_positions, sm_scale, sliding_window, sinks=sinks,+ )+ torch.testing.assert_close(actual.float(), expected.float(), atol=2e-2, rtol=2e-2)++@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton attention needs CUDA")@pytest.mark.parametrize("use_split_inputs", [False, True])@pytest.mark.parametrize(@@ -837,10 +878,8 @@ def test_triton_backend_replay_metadata_uses_capture_buffers(monkeypatch):assert replay_metadata.q_positions.data_ptr() == backend.capture.positions.data_ptr()assert replay_metadata.attn_logits is not Noneassert replay_metadata.attn_lse is not None- assert replay_metadata.num_kv_splits is not Noneassert replay_metadata.attn_logits.data_ptr() == backend.capture.attn_logits.data_ptr()assert replay_metadata.attn_lse.data_ptr() == backend.capture.attn_lse.data_ptr()- assert replay_metadata.num_kv_splits.data_ptr() == backend.capture.num_kv_splits.data_ptr()torch.testing.assert_close(backend.capture.cu_seqlens_k[:3], expected_indptr)torch.testing.assert_close(backend.capture.page_table.view(-1)[:8], expected_indices)