Accurate, fast FP4 KV-cache decoding through algorithm-system co-design.
"""SM100 CuTeDSL kernel for hybrid FP4/BF16 paged decode."""
import enum
import math
from typing import Type, Tuple, Callable, Optional, Literal
from functools import partial
from dataclasses import dataclass
import cuda.bindings.driver as cuda
import cutlass
import cutlass.cute as cute
from cutlass import Float32, Int32, const_expr, Float8E4M3FN, Float4E2M1FN
from cutlass.cute.experimental import iket
from cutlass.cute.nvgpu import cpasync
import cutlass.cute.nvgpu.tcgen05 as tcgen05
from cutlass.cute.nvgpu.tcgen05.mma import MmaMXF4NVF4Op, MmaMXF4Op
import cutlass.utils.blackwell_helpers as sm100_utils_basic
from ._fa4.block_scaled_layout import make_smem_layout_sfa, make_smem_layout_sfb
import cutlass.utils.blockscaled_layout as blockscaled_utils
from ._fa4.paged_kv import PagedKVManager
from ._fa4 import utils as utils
from ._fa4 import copy_utils
from ._fa4 import pipeline as pipeline
from ._fa4.mask import AttentionMask
# from ._fa4.softmax import SoftmaxSm100, apply_score_mod_inner
from ._fa4.softmax import (
SoftmaxSm100,
_round_probability_code,
_round_probability_scale,
_amax2_or_amax6_probability_scale,
_mxfp4_probability_exponent,
apply_score_mod_inner,
)
from ._fa4.seqlen_info import SeqlenInfoQK
from ._fa4.block_info import BlockInfo
from ._fa4.block_sparsity import BlockSparseTensors
from ._fa4.block_sparse_utils import (
get_total_block_count,
produce_block_sparse_loads_sm100,
softmax_block_sparse_sm100,
handle_block_sparse_empty_tile_correction_sm100,
)
from ._fa4.pack_gqa import PackGQA
from ._fa4 import mma_sm100_desc as sm100_desc
from ._fa4 import blackwell_helpers as sm100_utils
from ._fa4.blackwell_helpers import packed_float_to_ue4m3, packed_float_to_e2m1
from ._fa4.fast_math import FastDivmod
from ._block_scale import inverse_power_of_two, round_up_exponent
from ._fa4.tile_scheduler import (
TileSchedulerArguments,
SingleTileScheduler,
StaticPersistentTileScheduler,
SingleTileLPTScheduler,
SingleTileVarlenScheduler,
ParamsBase,
)
def _sf_layout_with_page_stride(layout, page_stride):
"""Override the page pitch of a K/V block-scale layout.
``tile_to_shape`` derives every stride from the shape, which assumes the
scale factors of consecutive pages sit back to back. When a page is a region
of a wider cache page the pitch is larger, and it is the only stride that
changes: the swizzle inside a page is fixed by the MMA. The page axis is
mode 3, whose stride is ``(0, pitch)``.
Reconstructing the layout drops the divisibility facts ``tile_to_shape``
attached to the dynamic strides, so they are reapplied; the SFB TMA atom
needs the 512-byte atom alignment to remain known.
"""
if page_stride is None:
return layout
def assume_atom_aligned(value):
if isinstance(value, int):
return value
return cute.assume(value, divby=512)
stride = layout.stride
return cute.make_layout(
layout.shape,
stride=(
(stride[0][0], assume_atom_aligned(stride[0][1])),
stride[1],
(stride[2][0], assume_atom_aligned(stride[2][1])),
(stride[3][0], assume_atom_aligned(page_stride)),
),
)
# A rescale whose factor is within 2**-RESCALE_THRESHOLD of one is skipped:
# the correction pass over O costs more than the exponent headroom it buys.
RESCALE_THRESHOLD = 8.0
class NamedBarrierFwd(enum.IntEnum):
Epilogue = enum.auto() # starts from 1 as barrier 0 is reserved for sync_threads()
# Publishes the per-warp partial row maxima between the four softmax warps
# of one warpgroup, once per KV block when S is transposed. The warpgroups
# own disjoint query rows and must not synchronise with each other, so each
# takes its own barrier; there is one slot per possible row group.
SoftmaxReduce = enum.auto()
SoftmaxReduce1 = enum.auto()
SoftmaxReduce2 = enum.auto()
SoftmaxReduce3 = enum.auto()
# WarpSchedulerWG1 = enum.auto()
# WarpSchedulerWG2 = enum.auto()
# WarpSchedulerWG3 = enum.auto()
# PFull = enum.auto()
# PEmpty = enum.auto()
class SplitRole(enum.Enum):
"""Which blocks one CTA of the launch merges.
``ALL`` is every CTA when there is no split of its own for the BF16 blocks:
the tail, the CTA's FP4 pages and the sink, each as far as it is fused.
With one, the FP4 splits are ``PAGES``, their pages alone, and the last
split is ``BF16``, the tail and the sink alone. Each role is a separately
specialised body of every warp role, chosen once per CTA.
"""
ALL = enum.auto()
PAGES = enum.auto()
BF16 = enum.auto()
# How many ways the transposed softmax may split the query rows of a tile. The
# reduction takes one named barrier per group, counting up from SoftmaxReduce,
# so this many consecutive slots must exist in the enum above.
MAX_SOFTMAX_ROW_GROUPS = 4
assert (
int(NamedBarrierFwd.SoftmaxReduce) + MAX_SOFTMAX_ROW_GROUPS - 1
== int(NamedBarrierFwd.SoftmaxReduce3)
)
# A group runs one cross-warp reduction per KV tile — a butterfly, a named
# barrier and a round trip through shared memory — and that cost is per group,
# not per row, so it does not shrink as the group is handed fewer rows. Below
# this many rows a group spends more on its own reduction than the shorter
# serial chain saves: measured at batch 16, seqlen 128k, splitting four query
# rows four ways is 10 percent slower than splitting them two ways, while
# splitting eight and thirty-two rows four ways wins 5 and 10 percent.
MIN_ROWS_PER_SOFTMAX_GROUP = 2
class FP4DecodeKernel:
arch = 100
def __init__(
self,
# dtype: Type[cutlass.Numeric],
head_dim: int,
head_dim_v: Optional[int] = None,
qhead_per_kvhead: cutlass.Constexpr[int] = 1,
is_causal: bool = False,
is_local: bool = False,
is_split_kv: bool = False,
pack_gqa: bool = False,
m_block_size: int = 128,
n_block_size: int = 128,
is_persistent: bool = True,
score_mod: cutlass.Constexpr | None = None,
mask_mod: cutlass.Constexpr | None = None,
has_aux_tensors: cutlass.Constexpr = False,
paged_kv_non_tma: bool = False,
is_varlen_q: bool = False,
sf_dtype: Optional[Type[cutlass.Numeric]] = None,
sf_vec_size: Optional[int] = None,
fused_residual_first_block: bool = False,
fused_sink_block: bool = False,
bf16_split: bool = False,
residual_source: str = "contiguous",
use_out_indices: bool = False,
qhead_per_kvhead_O: cutlass.Constexpr[int] = None,
seqlen_q_static_one: bool = False,
transpose_s: bool = False,
query_terms: cutlass.Constexpr[int] = 1,
p_scale_divisor: cutlass.Constexpr[float] = 6.0,
p_scale_grid_search: cutlass.Constexpr[bool] = False,
p_scale_amax2_or_amax6: cutlass.Constexpr[bool] = False,
p_scale_two_level: cutlass.Constexpr[bool] = False,
p_scale_rescale_log2: cutlass.Constexpr[int] = 0,
):
assert (sf_vec_size, sf_dtype) in (
(16, cutlass.Float8E4M3FN), (32, cutlass.Float8E8M0FNU),
), "scales must be NVFP4 (16, E4M3) or MXFP4 (32, E8M0)"
self.is_mxfp4 = sf_vec_size == 32
assert not self.is_mxfp4 or (
p_scale_divisor == 6.0 and not p_scale_grid_search
and not p_scale_amax2_or_amax6 and not p_scale_two_level
and p_scale_rescale_log2 == 0
), "MXFP4 P uses its own power-of-two scale search"
assert query_terms in (1, 2)
self.query_terms = query_terms
assert p_scale_divisor in (2.0, 3.0, 4.0, 6.0)
self.p_scale_divisor = p_scale_divisor
self.p_scale_grid_search = p_scale_grid_search
self.p_scale_amax2_or_amax6 = p_scale_amax2_or_amax6
self.p_scale_two_level = p_scale_two_level
if p_scale_rescale_log2 not in (0, 3):
raise ValueError("P exponent offset must be zero or log2(8)")
if p_scale_rescale_log2 and (not p_scale_amax2_or_amax6 or p_scale_two_level or p_scale_grid_search):
raise ValueError("P exponent offset requires the amax2_or_amax6 ladder")
self.p_scale_rescale_log2 = p_scale_rescale_log2
assert not p_scale_two_level or (p_scale_divisor == 6.0 and not p_scale_grid_search and not p_scale_amax2_or_amax6)
assert not (p_scale_grid_search and p_scale_amax2_or_amax6)
self.fused_residual_first_block = fused_residual_first_block
self.fused_sink_block = fused_sink_block
assert not fused_sink_block or fused_residual_first_block, (
"the BF16 sink reuses the residual cache and requires its prologue"
)
# The residual and the sink are single blocks outside the page range
# split-k divides, so under split-k exactly one split may count them.
# With bf16_split that is a split of their own, launched after the
# FP4 splits: mO carries one more partial than there are page
# partitions, the FP4 splits divide the pages among themselves alone,
# and none of them waits for, loads or holds a BF16 block.
# Without it every split runs both blocks and all but split 0 with a
# length of zero, which the zero-length contract already makes
# contribute nothing; split 0 is the one the scheduler always runs,
# even when it holds no FP4 page.
self.bf16_split = bf16_split
assert not bf16_split or (is_split_kv and fused_residual_first_block), (
"a split of its own for the BF16 blocks needs split-k and a residual"
)
self.residual_split_idx = 0
assert residual_source in ("contiguous", "paged_bf16"), (
f"unknown residual_source={residual_source!r}; "
f"expected 'contiguous' or 'paged_bf16'"
)
self.residual_source = residual_source
self.use_out_indices = use_out_indices
self.seqlen_q_static_one = seqlen_q_static_one
# Hold S as (kv position, query row) instead of (query row, kv position).
# A tensor-memory load hands a thread one row and the same columns as
# every other lane, so only the row axis can be narrowed per thread.
# Decode leaves almost all of the query axis empty, and putting it in
# the columns is what lets a thread exponentiate qhead_per_kvhead values
# instead of a whole row of keys. The row reductions become cross-thread
# in exchange. Resolved against the quantization flags in __call__.
self.transpose_s_requested = transpose_s
self.transpose_s = False
self.tmem_sfpv_offset = None
self.transposed_query_rows = 0
self.stacked_query_terms = False
self.narrow_pv = False
self.pv_rows = 0
self.softmax_row_groups = 1
self.softmax_rows_per_group = 0
self.softmax_scatters_rows = False
self.softmax_red_slots = 0
self.use_tma_KV = not paged_kv_non_tma
# self.dtype = dtype
# padding head_dim to a multiple of 16 as k_block_size
hdim_multiple_of = 16
self.head_dim_padded = int(math.ceil(head_dim / hdim_multiple_of) * hdim_multiple_of)
head_dim_v = head_dim_v if head_dim_v is not None else head_dim
self.same_hdim_kv = head_dim == head_dim_v
self.head_dim_v_padded = int(math.ceil(head_dim_v / hdim_multiple_of) * hdim_multiple_of)
self.same_hdim_kv_padded = self.head_dim_padded == self.head_dim_v_padded
self.check_hdim_oob = head_dim != self.head_dim_padded
self.check_hdim_v_oob = head_dim_v != self.head_dim_v_padded
self.m_block_size = m_block_size
self.n_block_size = n_block_size
# A second Q stage only earns its keep when a CTA carries two Q tiles.
# With seqlen_q statically one the packed M extent is qhead_per_kvhead
# rows under PackGQA and a single row without it, either way at most one
# m_block_size, so stage 1 would TMA an out-of-range Q, get zeros, and
# run a full softmax, TMEM round trip and PV MMA over them.
if seqlen_q_static_one:
assert qhead_per_kvhead <= m_block_size
if is_split_kv or seqlen_q_static_one:
self.q_stage = 1
elif getattr(type(self), "_force_q_stage_1", False):
self.q_stage = 1
else:
self.q_stage = 2
assert self.q_stage in [1, 2]
assert self.query_terms == 1 or self.q_stage == 1
# 2 Q tile per CTA
self.cta_tiler = (self.q_stage * m_block_size, n_block_size, self.head_dim_padded)
self.mma_tiler_qk = (m_block_size, n_block_size, self.head_dim_padded)
self.mma_tiler_pv = (m_block_size, self.head_dim_v_padded, n_block_size)
self.qk_acc_dtype = Float32
self.pv_acc_dtype = Float32
self.cluster_shape_mn = (1, 1)
self.is_persistent = is_persistent
self.is_causal = is_causal
self.is_local = is_local
self.is_varlen_q = is_varlen_q
self.use_correction_warps_for_epi = is_varlen_q
self.qhead_per_kvhead = qhead_per_kvhead
self.qhead_per_kvhead_O = (
qhead_per_kvhead_O if qhead_per_kvhead_O is not None else qhead_per_kvhead
)
assert self.qhead_per_kvhead_O >= self.qhead_per_kvhead, (
"qhead_per_kvhead_O must be >= qhead_per_kvhead (pad up, never down)"
)
self.is_split_kv = is_split_kv
self.pack_gqa = pack_gqa
# A single query position packs onto a flat head axis (see the PackGQA
# layouts in __call__), which tiles for any ratio up to m_block_size.
# Only the (head, seqlen) packing needs the tile to be a whole number
# of query groups.
if pack_gqa and not seqlen_q_static_one:
assert m_block_size % self.qhead_per_kvhead == 0, (
"For PackGQA, m_block_size must be divisible by qhead_per_kvhead"
)
assert m_block_size % self.qhead_per_kvhead_O == 0, (
"For PackGQA with separate O-axis, m_block_size must be divisible by qhead_per_kvhead_O"
)
# The MMA fixes the M extent at 128 rows while decode fills only
# qhead_per_kvhead of them, and a softmax thread owns one row, so the
# rows that carry no query still cost a full row of exp2. Repeating each
# query row across the tile puts a real row under every thread, which is
# what lets the column range a thread owns shrink by this factor. The
# copies of one query row are consecutive, so a group of them lands
# inside a single warp and the row reductions stay warp-local.
# The split path writes FP32 partials and an LSE per row for the combine
# kernel to consume, both keyed by the unreplicated row order, so it is
# left alone until those two writers learn the same row selection the
# unsplit epilogue uses.
self.q_replicate = 1
if (
pack_gqa
and seqlen_q_static_one
and not is_split_kv
and getattr(type(self), "_enable_q_replicate", False)
):
self.q_replicate = m_block_size // self.qhead_per_kvhead
assert not (self.is_split_kv and self.head_dim_v_padded >= 192), (
"SplitKV is not supported for hdim >= 192"
)
self.score_mod = score_mod
self.mask_mod = mask_mod
if cutlass.const_expr(has_aux_tensors):
self.vec_size: cutlass.Constexpr = 1
else:
self.vec_size: cutlass.Constexpr = 2
# Does S1 need to wait for S0 to finish
# self.s0_s1_barrier = self.head_dim_padded in [64, 96] and (not self.is_causal and not self.is_local)
self.s0_s1_barrier = False
self.overlap_sO_sQ = (
(self.head_dim_padded == 192 and self.head_dim_v_padded >= 64) or
(self.head_dim_v_padded >= 128 and self.is_split_kv)
)
# BF16 blocks merged ahead of and behind a CTA's own FP4 pages hold
# their tiles until the tile's last MMA, and O is staged only once that
# MMA has completed, so on a CTA that owns one tile O takes the tiles'
# bytes instead of widening sQ or holding its own.
self.overlap_sO_bf16 = fused_residual_first_block and not bf16_split
if self.overlap_sO_bf16:
assert not is_persistent, (
"O staged in the BF16 tiles needs one tile per CTA"
)
self.overlap_sO_sQ = False
if self.overlap_sO_sQ:
self.is_persistent = False
assert self.use_tma_KV or not (self.check_hdim_oob or self.check_hdim_v_oob), (
"Paged KV does not support irregular head dim"
)
SM100_TMEM_CAPACITY_COLUMNS = 512
self.tmem_alloc_cols = SM100_TMEM_CAPACITY_COLUMNS
self.tmem_s_offset = [0, self.n_block_size] # e.g., 0, 128
self.tmem_o_offset = [
self.tmem_s_offset[-1] + self.n_block_size + i * self.head_dim_v_padded
for i in range(self.q_stage)
] # e.g., 256, 384
# self.tmem_o_offset = self.tmem_s_offset
self.tmem_total = self.tmem_o_offset[-1] + self.head_dim_v_padded
assert self.tmem_total <= SM100_TMEM_CAPACITY_COLUMNS
self.tmem_s_to_p_offset = self.n_block_size // 2
self.tmem_p_offset = [
self.tmem_s_offset[i] + self.tmem_s_to_p_offset for i in range(2)
] # e.g., 64, 192
self.tmem_p_bf16_offset = [
self.tmem_s_offset[i] + self.tmem_s_to_p_offset for i in range(2)
] # same as FP4 but aliased temporally — see comment above.
# vec buffer for row_max & row_sum
self.tmem_vec_offset = self.tmem_s_offset
# The count a softmax warpgroup was tuned at. A CTA wide enough that it
# cannot afford this narrows it; see _assign_warp_roles.
if self.head_dim_padded < 96:
self.num_regs_softmax_tuned = 200
self.num_regs_correction = 64
self.num_regs_other = 48
else:
# self.num_regs_softmax_tuned = 192 if self.is_causal or self.is_local else 184
self.num_regs_softmax_tuned = 216
# self.num_regs_softmax_tuned = 176
# self.num_regs_correction = 96
# self.num_regs_correction = 80
# self.num_regs_correction = 64 if self.is_causal or self.is_local else 80
self.num_regs_correction = 48
# self.num_regs_other = 32
# self.num_regs_other = 64
# self.num_regs_other = 80
self.num_regs_other = 24
# self.num_regs_other = 96 if self.is_causal or self.is_local else 80
# self.num_regs_other = 64 if self.is_causal or self.is_local else 80
self.num_regs_empty = 24
# Two softmax warpgroups is the layout every configuration starts from:
# one per Q stage. A transposed decode that splits its query rows four
# ways asks for four and re-runs this once __call__ has resolved that.
self._assign_warp_roles(2)
self.buffer_align_bytes = 1024
# Scale factor parameters for block-scaled quantization (FP4)
self.sf_dtype = sf_dtype
self.sf_vec_size = sf_vec_size
self.mma_inst_bits_k = 256
self.mma_inst_tile_k = self.head_dim_padded // (self.mma_inst_bits_k // 4)
self.sf_groups_per_mma_k = self.mma_inst_bits_k // 4 // self.sf_vec_size
# An SM holds this many 32-bit registers, handed out in blocks of eight per
# thread, and the kernel runs one CTA per SM.
REGISTER_FILE_PER_SM = 65536
REGISTER_ALLOC_GRANULARITY = 8
WARPS_PER_WARPGROUP = 4
def _assign_warp_roles(self, num_softmax_warpgroups: int):
"""Lay out the warp roles for a CTA with this many softmax warpgroups.
Each softmax warpgroup is a contiguous run of four warps starting on a
128-thread boundary, which is what makes ``thread_idx % 128`` the kv
position inside the tile that the transposed softmax indexes by. The
remaining roles follow: one correction warpgroup, then the mma,
epilogue, load and idle warps that share the last warpgroup.
Widening the CTA costs registers, so the softmax warpgroups split
whatever the correction and service warpgroups leave of the CTA's
register allocation rather than each keeping the count a narrower CTA
could afford.
"""
warps = self.WARPS_PER_WARPGROUP
threads_per_warpgroup = warps * cute.arch.WARP_SIZE
self.softmax_warp_ids = tuple(
tuple(range(g * warps, (g + 1) * warps))
for g in range(num_softmax_warpgroups)
)
self.softmax0_warp_ids = self.softmax_warp_ids[0] # stage 0
self.softmax1_warp_ids = self.softmax_warp_ids[1] # stage 1
service = num_softmax_warpgroups * warps
self.correction_warp_ids = tuple(range(service, service + warps))
self.mma_warp_id = service + warps
self.epilogue_warp_ids = (service + warps + 1,)
self.load_warp_ids = (service + warps + 2,)
self.empty_warp_ids = (service + warps + 3,)
self.threads_per_cta = cute.arch.WARP_SIZE * (service + 2 * warps)
if not self.use_tma_KV:
self.load_warp_ids = self.load_warp_ids + self.empty_warp_ids
self.empty_warp_ids = ()
if self.use_correction_warps_for_epi:
self.empty_warp_ids = self.empty_warp_ids + self.epilogue_warp_ids
self.epilogue_warp_ids = self.correction_warp_ids
elif self.is_varlen_q: # fallback
self.epilogue_warp_ids = (service + warps + 1, service + warps + 2)
# setmaxnreg only redistributes the registers the CTA was given at
# launch, which is threads_per_cta times the largest per-thread count
# that fits the register file, rounded down to the allocation
# granularity. A CTA of 768 threads loses that rounding twice over: the
# per-thread count drops to 80 and the CTA's pool with it, so the
# arithmetic has to be done against the pool and not against the whole
# register file. Over-subscribe it and setmaxnreg.inc waits for
# registers no one will release.
regs_at_launch = (
self.REGISTER_FILE_PER_SM
// self.threads_per_cta
// self.REGISTER_ALLOC_GRANULARITY
* self.REGISTER_ALLOC_GRANULARITY
)
softmax_pool = regs_at_launch * self.threads_per_cta - threads_per_warpgroup * (
self.num_regs_correction + max(self.num_regs_other, self.num_regs_empty)
)
affordable = (
softmax_pool
// (threads_per_warpgroup * num_softmax_warpgroups)
// self.REGISTER_ALLOC_GRANULARITY
* self.REGISTER_ALLOC_GRANULARITY
)
self.num_regs_softmax = min(self.num_regs_softmax_tuned, affordable)
def _softmax_row_group_count(self) -> int:
"""How many ways the transposed softmax splits this tile's query rows.
A decode carries one Q tile, so only the first softmax warpgroup has a
stage to run. Spreading the query rows over more groups shortens the
serial chain every thread runs — its butterfly, its exponentials and
its packing — and the groups write disjoint rows of P and of the
scales. Two groups is what any tile with more than one query row
already split into; going past two is worth it only while every group
keeps enough rows to amortize its own reduction.
Plain Python, evaluated while ``__call__`` is traced, so the count and
everything derived from it are compile-time constants.
"""
# The BF16 steps share one P tile between the groups, and nothing
# orders one group's stores against another's clear, so each clears
# only rows no group writes (see zero_transposed_p_bf16). A second Q
# stage gives the second warpgroup its own work.
if self.q_stage != 1:
return 1
if self.transposed_query_rows < 2:
return 1
return min(
MAX_SOFTMAX_ROW_GROUPS,
max(2, self.transposed_query_rows // MIN_ROWS_PER_SOFTMAX_GROUP),
)
def _setup_attributes(self):
"""Set up configurations and parameters for the FMHA kernel operation.
This method initializes and configures various attributes required for the
execution of the fused multi-head attention kernel, mainly about the pipeline stages:
- Sets up staging parameters for Q, K, V inputs and accumulator data
- Configures pipeline stages for softmax, correction, and epilogue operations
"""
self.acc_stage = 1
# The epilogue indexes sO by q tile, so a deeper ring than there are q
# tiles is memory no one can name. Decode runs a single q tile, where
# the second buffer was costing 32KB of output, or 64KB once the split
# path widens it to fp32 partials.
self.epi_stage = self.q_stage
# Compute kv_stage from SMEM budget.
# Blackwell: 228KB per SM, 227KB optin per block.
# K and V alias when same dtype or when K is smaller (FP4 K in BF16 V).
smem_budget = 227 * 1024
align = self.buffer_align_bytes # 128B struct field alignment
def align_up(x, a): return (x + a - 1) // a * a
# Fixed fields (not scaled by kv_stage): mbar, tmem_holding_buf, sScale, sO, sQ, SFQ, SFP
# mbar_total depends on kv_stage but is small (~40 barriers * 8B = 320B); use upper bound
smem_mbar = 512 # generous upper bound for mbarrier storage
smem_tmem = 4 # Int32
smem_sScale = align_up(max(self.q_stage * self.m_block_size * 2, self.m_block_size * 4) * 4, align) # Float32
smem_q_per_stage = self.m_block_size * self.head_dim_padded * self.q_dtype.width // 8
smem_o_per_stage = self.m_block_size * self.head_dim_v_padded * self.o_dtype.width // 8
# SFQ/SFP are per q_stage, SF layout cosize: m_block * head_dim / sf_vec_size
sfq_per_stage = self.m_block_size * self.head_dim_padded // self.sf_vec_size
sfp_per_stage = self.m_block_size * self.head_dim_v_padded // self.sf_vec_size
sfq_bytes = (
sfq_per_stage * self.q_stage * self.query_terms if self.quant_qk else 0
)
# When the two overlap, sQ is widened to cover sO rather than sO being
# dropped, so the budget has to charge the wider of the two. The Q side
# is every query term's tile and its scale factors, which are dead by
# the time sO is written, just like the tiles.
if self.overlap_sO_sQ:
smem_sO = 0
smem_sQ = align_up(
max(
smem_q_per_stage * self.q_stage * self.query_terms + sfq_bytes,
smem_o_per_stage * self.epi_stage,
),
align,
)
smem_sSFQ = 0
else:
smem_sO = align_up(smem_o_per_stage * self.epi_stage, align)
smem_sQ = align_up(
smem_q_per_stage * self.q_stage * self.query_terms,
align,
)
smem_sSFQ = align_up(sfq_bytes, align)
smem_sSFP = align_up(sfp_per_stage * self.q_stage, align) if self.quant_pv else 0
# A transposed P reaches the tensor core from shared memory, and the
# cross-warp row maximum needs one float per softmax warp per query row,
# double buffered so a single named barrier per KV block suffices.
if self.transpose_s:
smem_sP = align_up(
self.m_block_size * self.n_block_size * self.v_dtype.width // 8, align
)
smem_sRed = align_up(self.softmax_red_slots * 4, align)
else:
smem_sP = 0
smem_sRed = 0
if self.fused_residual_first_block:
smem_sK_bf16 = align_up(self.m_block_size * self.head_dim_padded * 2, align)
smem_sV_bf16 = align_up(self.m_block_size * self.head_dim_v_padded * 2, align)
smem_sQ_bf16 = align_up(self.m_block_size * self.head_dim_padded * 2, align)
else:
smem_sK_bf16 = 0
smem_sV_bf16 = 0
smem_sQ_bf16 = 0
smem_bf16_tiles = smem_sK_bf16 + smem_sV_bf16 + smem_sQ_bf16
if self.overlap_sO_bf16:
smem_bf16_tiles = max(smem_bf16_tiles, smem_sO)
smem_sO = 0
# On a split of their own the BF16 tiles share their bytes with the
# K/V stages (see SharedStorage), so they are charged further down.
smem_fixed = smem_mbar + smem_tmem + smem_sScale + smem_sO + smem_sQ + smem_sSFQ + smem_sSFP + smem_sP + smem_sRed
if not self.bf16_split:
smem_fixed += smem_bf16_tiles
# Per-kv_stage fields: sK (or aliased), sV, SFK, SFV
smem_k_per_stage = self.m_block_size * self.head_dim_padded * self.k_dtype.width // 8
smem_v_per_stage = self.m_block_size * self.head_dim_v_padded * self.v_dtype.width // 8
if self.v_dtype == self.k_dtype or self.k_dtype.width < self.v_dtype.width:
smem_kv_per_stage = max(smem_k_per_stage, smem_v_per_stage)
else:
smem_kv_per_stage = smem_k_per_stage + smem_v_per_stage
# SF layout cosize per stage: n_block * head_dim / sf_vec_size (MMA-tiled)
sfk_per_stage = self.n_block_size * self.head_dim_padded // self.sf_vec_size
sfv_per_stage = self.n_block_size * self.head_dim_v_padded // self.sf_vec_size
if self.quant_qk:
smem_kv_per_stage += sfk_per_stage
if self.quant_pv:
smem_kv_per_stage += sfv_per_stage
# Add per-stage padding for swizzle/layout cosize inflation (~128B per staged field)
num_kv_staged_fields = 2 # sK + sV (or 1 if aliased, but sV still has cosize overhead)
if self.quant_qk:
num_kv_staged_fields += 1 # sSFK
if self.quant_pv:
num_kv_staged_fields += 1 # sSFV
smem_kv_per_stage += num_kv_staged_fields * 128
self.kv_stage = (smem_budget - smem_fixed) // smem_kv_per_stage
if self.quant_qk and self.quant_pv:
# Depth buys latency hiding only until the mainloop stops waiting on
# KV. Past that the extra buffers cost cosize and issue slots: on the
# non-split path a depth sweep is flat from 8 to 10 and 1% worse at
# 12 and beyond, so spending the whole SMEM budget (which lands at
# 14) is a loss. The split path carries an FP32 partial-output buffer
# and cannot fit more than 9 stages anyway.
KV_STAGE_FP4_SPLITK_CAP = 8
KV_STAGE_FP4_CAP = 10
cap = KV_STAGE_FP4_SPLITK_CAP if self.is_split_kv else KV_STAGE_FP4_CAP
self.kv_stage = min(self.kv_stage, cap)
if self.bf16_split:
# The tiles overlay the stages' K/V data, so only what they need
# past it adds to the budget.
smem_kv_data_per_stage = max(smem_k_per_stage, smem_v_per_stage)
def smem_total(stages):
overhang = smem_bf16_tiles - stages * smem_kv_data_per_stage
return smem_fixed + stages * smem_kv_per_stage + max(0, overhang)
while smem_total(self.kv_stage) > smem_budget:
self.kv_stage -= 1
assert self.kv_stage >= 2, (
f"kv_stage={self.kv_stage} < 2: FP4 mainloop requires K/V double-buffer"
)
# For hdim 192,128, we don't have enough smem to store all 3 stages of KV:
# 128 x 192 x 2 bytes x 3 stages = 144KB, and we need 96KB for Q.
# Instead we store smem as [smem_large, smem_small, smem_large], where smem_large is
# 128 x 192 and smem_small is 128 x 128. We set the stride between the stages to be
# 128 * 160, so that indexing the 0th and 2nd stages will get the right address,
# but for the 1st stage we need to add or subtract (depending on phase) 128 x 64.
# self.uneven_kv_smem = (
# self.head_dim_padded == 192 and self.head_dim_v_padded == 128 and self.kv_stage == 3
# )
self.uneven_kv_smem = False
self.uneven_kv_smem_offset = (
self.m_block_size * (self.head_dim_padded - self.head_dim_v_padded) // 2
if self.uneven_kv_smem
else 0
)
assert self.uneven_kv_smem_offset % 1024 == 0
@cute.jit
def __call__(
self,
mQ, # cute.Tensor or cute.Pointer (b, s_q, h, d)
mK, # cute.Tensor or cute.Pointer (b_k, s_k, h_k, d)
mV, # cute.Tensor or cute.Pointer (b_k, s_k, h_k, dv)
mO: cute.Tensor, # (b, s_q, h, dv)
mLSE: Optional[cute.Tensor],
softmax_scale: Float32,
stream: cuda.CUstream,
mCuSeqlensQ: Optional[cute.Tensor] = None,
mCuSeqlensK: Optional[cute.Tensor] = None,
mSeqUsedQ: Optional[cute.Tensor] = None,
mSeqUsedK: Optional[cute.Tensor] = None,
mPageTable: Optional[cute.Tensor] = None,
window_size_left: Int32 | int | None = None,
window_size_right: Int32 | int | None = None,
learnable_sink: Optional[cute.Tensor] = None,
blocksparse_tensors: Optional[BlockSparseTensors] = None,
aux_tensors: Optional[list] = None,
mSFQ: Optional[cute.Tensor] = None,
mSFK: Optional[cute.Tensor] = None,
mSFV: Optional[cute.Tensor] = None,
# For pointer-based Q/K: separate shapes to handle cross-attention (seqlen_q != seqlen_k)
q_ptr_shape: tuple = (),
k_ptr_shape: tuple = (),
# For pointer-based V (FP4 K-major): full headdim shape (b, s, h, d)
v_ptr_shape: tuple = (),
# Distance between consecutive K/V pages, in FP4 elements for the packed
# data and in bytes for the scale factors. None means the pages are
# densely packed, which is what the shapes alone imply. A larger stride
# lets one page be a region of a wider cache page that also holds the
# scale factors and the other of K/V.
k_page_stride=None,
v_page_stride=None,
k_sf_page_stride=None,
v_sf_page_stride=None,
compute_sp1: cutlass.Constexpr[bool] = False,
mResidualQ: Optional[cute.Tensor] = None,
mResidualK: Optional[cute.Tensor] = None,
mResidualV: Optional[cute.Tensor] = None,
mResidualSeqUsedK: Optional[cute.Tensor] = None,
mResidualKCache: Optional[cute.Tensor] = None,
mResidualVCache: Optional[cute.Tensor] = None,
mResidualBlockIds: Optional[cute.Tensor] = None,
mSinkSeqUsedK: Optional[cute.Tensor] = None,
mOutIndices: Optional[cute.Tensor] = None,
mQSecond=None,
mSFQSecond: Optional[cute.Tensor] = None,
):
"""Execute the Fused Multi-Head Attention operation on the provided tensors.
For FP4, mQ/mK can be cute.Pointer with q/k_ptr_shape providing (b, s, h, d).
The kernel builds tensors from the pointer using make_ordered_layout.
"""
# Build Q/K tensors from pointer/tensor + shape
# For pointers: mQ is a Pointer, .iterator not needed
# For tensors: mQ is a Tensor, use .iterator to extract pointer
q_iter = mQ.iterator if hasattr(mQ, 'iterator') else mQ
q_second_iter = (
mQSecond.iterator if hasattr(mQSecond, "iterator") else mQSecond
)
k_iter = mK.iterator if hasattr(mK, 'iterator') else mK
mQ = cute.make_tensor(q_iter, cute.make_ordered_layout(
q_ptr_shape, order=tuple(range(len(q_ptr_shape) - 1, -1, -1))
))
mQSecond = cute.make_tensor(
q_second_iter,
cute.make_ordered_layout(
q_ptr_shape,
order=tuple(range(len(q_ptr_shape) - 1, -1, -1)),
),
)
if const_expr(k_page_stride is None):
mK = cute.make_tensor(k_iter, cute.make_ordered_layout(
k_ptr_shape, order=tuple(range(len(k_ptr_shape) - 1, -1, -1))
))
else:
# Same row-major (b, s, h, d) layout the ordered form produces, with
# the page stride supplied instead of derived. The innermost stride
# stays a literal 1 so the major mode is still statically known.
_, _, k_h, k_d = k_ptr_shape
mK = cute.make_tensor(k_iter, cute.make_layout(
k_ptr_shape, stride=(k_page_stride, k_h * k_d, k_d, 1)
))
# FP4 K-major V: build from pointer with explicit (b, s, h, d) shape and
# K-major strides (S*H*D, 1, S, S*H). The host transposes V's underlying
# buffer so that seqlen has stride 1 in the FP4 byte buffer.
if const_expr(len(v_ptr_shape) > 0):
v_iter = mV.iterator if hasattr(mV, 'iterator') else mV
# K-major V: nvfp4_quantize on `v.permute(0,2,3,1).reshape(b*h*d, s)`
# produces an FP4 byte buffer of physical shape (b, h, d, s/2) row-major,
# where each int8 byte holds two seqlen-adjacent FP4 in the high/low
# nibble. Logically the V tensor has FP4 shape (b, s, h, d) with element
# strides (h*d*s, 1, h*d, d) — order=(3, 0, 2, 1):
# s (dim 1) order 0 → stride 1
# d (dim 3) order 1 → stride s
# h (dim 2) order 2 → stride s*d
# b (dim 0) order 3 → stride s*d*h
# Int64 shape so make_ordered_layout produces Int64 strides
# (tma_partition for SFV requires Int64).
from cutlass import Int64
v_b, v_s, v_h, v_d = v_ptr_shape
v_shape = (Int64(v_b), Int64(v_s), Int64(v_h), Int64(v_d))
if const_expr(v_page_stride is None):
mV = cute.make_tensor(v_iter, cute.make_ordered_layout(
v_shape, order=(3, 0, 2, 1),
))
else:
# The order=(3, 0, 2, 1) strides written out, with the page
# stride supplied. Seqlen keeps a literal stride of 1 so V stays
# statically K-major.
mV = cute.make_tensor(v_iter, cute.make_layout(
v_shape,
stride=(
Int64(v_page_stride),
1,
Int64(v_s) * Int64(v_d),
Int64(v_s),
),
))
self.q_dtype = mQ.element_type
self.k_dtype = mK.element_type
self.v_dtype = mV.element_type
self.o_dtype = mO.element_type
self.compute_sp1 = const_expr(compute_sp1)
if const_expr(mSFQ is None):
assert self.q_dtype.width >= 8
assert const_expr(mSFK is None), "Must provide both QK sfs or None"
self.quant_qk = const_expr(mSFQ is not None)
self.quant_pv = const_expr(mSFV is not None)
assert not (not self.quant_qk and self.quant_pv)
# The transposed layout only pays off where the query axis is short and
# statically known, and it reroutes P through shared memory, so it is
# confined to the pure FP4 decode shape. Everything else keeps the
# untransposed path. The query count is padded to a power of two below,
# because it becomes the repetition count of a single tcgen05 load.
# The conditions are combined with all() over a list rather than a chain
# of `and`. Preprocessing rewrites each short-circuit operator into a
# nested conditional that carries a copy of the continuation, so an
# n-term chain in a traced function grows the transformed AST by 2**n;
# sixteen terms here took this function from 14k nodes to over 4M and
# its preprocessing from 0.01s to more than ten minutes. Every term is a
# plain attribute read, so evaluating all of them eagerly is free.
self.transpose_s = const_expr(
all(
[
self.transpose_s_requested,
self.quant_qk,
self.quant_pv,
self.pack_gqa,
self.seqlen_q_static_one,
not self.compute_sp1,
not self.is_causal,
not self.is_local,
# self.use_block_sparsity is not derived until much later in
# this function, so read its source argument directly.
blocksparse_tensors is None,
self.score_mod is None,
self.qhead_per_kvhead <= 128,
]
)
)
assert not self.p_scale_two_level or self.transpose_s, "two_level P requires transposed decode"
assert not self.p_scale_rescale_log2 or self.transpose_s, (
"rescaled P requires transposed decode"
)
assert self.query_terms == 1 or self.transpose_s, (
"two-term query decode requires the transposed score path"
)
assert not self.is_mxfp4 or self.transpose_s, (
"MXFP4 decode requires the transposed score path"
)
if const_expr(self.transpose_s):
# Replication addresses the same padding by repeating rows, which
# the transposed layout makes pointless.
self.q_replicate = 1
# Transposing swaps the two leading modes of the QK tiler. Requiring
# them equal keeps that tuple, and every partition derived from it,
# unchanged, so only the operand roles move.
assert self.m_block_size == self.n_block_size, (
"transpose_s requires a square QK tile"
)
# PackGQA with a statically-one query length fills qhead_per_kvhead
# rows of the M tile; transposed, they are the live columns of S.
# The count becomes the repetition of a single tcgen05 load, which
# must be a power of two, so it is rounded up. The extra columns
# carry the zero rows TMA fills past the flat head axis: their
# scores are finite and their rows of O are never stored.
self.transposed_query_rows = 1 << (self.qhead_per_kvhead - 1).bit_length()
# Two independently scaled terms occupy disjoint query rows of
# one narrow QK instruction. The logical softmax row count stays
# unchanged; its FP32 addition joins the two score columns.
self.stacked_query_terms = (
self.query_terms == 2
and self.q_stage == 1
and self.qhead_per_kvhead in (4, 8, 16)
)
# PV accumulates O^T: V is the A operand over the head dimension and
# P the B operand over the live query rows, rounded up to the eight
# an MMA needs. Padding the rows to the 128 of an A operand instead
# spends sixteen times the tensor-core energy at a ratio of eight,
# which under the power cap costs clock. Each correction thread then
# holds one head-dimension lane of every live row, so the row count
# is bounded by the correction warps' registers; the two-level P
# scale accumulates rows of O per thread and keeps the wide tile.
self.pv_rows = max(8, self.transposed_query_rows)
self.narrow_pv = all(
[
self.transposed_query_rows <= 16,
self.q_stage == 1,
not self.p_scale_two_level,
learnable_sink is None,
self.head_dim_v_padded == self.m_block_size,
not self.use_correction_warps_for_epi,
]
)
self.softmax_row_groups = self._softmax_row_group_count()
# transposed_query_rows is a power of two, so the group count
# always divides it evenly.
self.softmax_rows_per_group = (
self.transposed_query_rows // self.softmax_row_groups
)
# The row maximum reduce-scatters a thread's rows across lanes when
# it holds more than one and each lane's row still has the lanes of
# a whole scale group to itself. The grid-search P scale reduces an
# error per row over the group and keeps the row-per-thread path.
self.softmax_scatters_rows = (
2 <= self.softmax_rows_per_group <= self.sf_vec_size
and not self.p_scale_grid_search
)
# One softmax warpgroup per row group, except that the CTA never
# narrows below the two warpgroups the Q stages need.
self._assign_warp_roles(max(2, self.softmax_row_groups))
# Two buffers per warp per row, so a single named barrier per KV
# block suffices for the row reductions. Splitting the rows across
# groups moves slots between the groups without changing the total.
self.softmax_red_slots = (
2 * len(self.softmax0_warp_ids) * self.transposed_query_rows
)
# P and its scale factors normally alias the tensor memory S was
# read out of, which is why the QK of the next block has to wait
# behind this block's PV. Transposed, P lives in shared memory, so
# only the scale factors still alias, and a decode leaves 128
# tensor-memory columns free above O to move them into. Once they
# are out of the way the only thing the next QK waits on is the
# softmax having drained S, which it signals as soon as the load
# completes rather than at the end of the step.
self.tmem_sfpv_offset = self.tmem_total
if const_expr(self.fused_sink_block):
assert self.transpose_s, (
"the two-island BF16 path requires transposed scores"
)
# Each CTA picks its role once, from its split, so it must own exactly
# one tile for its whole life. Its BF16 tiles share the bytes of sK,
# which therefore has to hold V as well.
if const_expr(self.bf16_split):
assert (
self.transpose_s
and not self.is_persistent
and self.v_dtype == self.k_dtype
), (
"a split of its own for the BF16 blocks needs transposed "
"scores, one tile per CTA, and K and V of one dtype"
)
# Assume all strides are divisible by 128 bits except the last stride
def _assume_strides(t):
divby = 128 // t.element_type.width
return tuple(
s if isinstance(s, int) else cute.assume(s, divby=divby)
for s in t.stride[:-1]
) + (t.stride[-1],)
mV, mO = [
cute.make_tensor(t.iterator, cute.make_layout(t.shape, stride=_assume_strides(t)))
for t in (mV, mO)
]
Q_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensQ is None) else [0, 2, 1]
mQ = cute.make_tensor(mQ.iterator, cute.select(mQ.layout, mode=Q_layout_transpose)) # (s_q, d, h, b)
mQSecond = cute.make_tensor(
mQSecond.iterator,
cute.select(mQSecond.layout, mode=Q_layout_transpose),
)
# (s_k, d, h_k, b_k) or (total_k, d, h_k) if there's cu_seqlens_k or (page_size, d, h_k, num_pages) if there's page_table
KV_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensK is None) else [0, 2, 1]
mK, mV = [
cute.make_tensor(t.iterator, cute.select(t.layout, mode=KV_layout_transpose))
for t in (mK, mV)
]
if const_expr(self.is_split_kv):
O_layout_transpose = [2, 4, 3, 1, 0] if const_expr(mCuSeqlensQ is None) else [1, 3, 2, 0]
LSE_layout_transpose = [3, 2, 1, 0] if const_expr(mCuSeqlensQ is None) else [2, 1, 0]
num_splits = mO.shape[0]
else:
O_layout_transpose = [1, 3, 2, 0] if const_expr(mCuSeqlensQ is None) else [0, 2, 1]
LSE_layout_transpose = [2, 1, 0] if const_expr(mCuSeqlensQ is None) else [1, 0]
num_splits = Int32(1)
mO = cute.make_tensor(mO.iterator, cute.select(mO.layout, mode=O_layout_transpose))
mLSE = (
cute.make_tensor(mLSE.iterator, cute.select(mLSE.layout, mode=LSE_layout_transpose))
if const_expr(mLSE is not None)
else None
)
# (s, d, h, b) -> (d, s, h, b)
# For FP4 block-scaled MMA, B (V) must be K-major (K=seqlen contiguous).
# Skip V transpose when V is already K-major (headdim contiguous = N contiguous after transpose).
# Without transpose: mV = (s, d, h, b) → mode 0=s(K), mode 1=d(N) with d contiguous → K-major: NO!
# With transpose: mV = (d, s, h, b) → mode 0=d(N), mode 1=s(K) with d contiguous → MN-major
# For FP4 we need K-major: mode 1 (K=seqlen) contiguous. Need V physically transposed on host.
V_layout_transpose = [1, 0, 2, 3] if const_expr(mCuSeqlensK is None) else [1, 0, 2]
mV = cute.make_tensor(mV.iterator, cute.select(mV.layout, mode=V_layout_transpose))
self.q_major_mode = cutlass.utils.LayoutEnum.from_tensor(mQ).mma_major_mode()
self.k_major_mode = cutlass.utils.LayoutEnum.from_tensor(mK).mma_major_mode()
self.v_major_mode = cutlass.utils.LayoutEnum.from_tensor(mV).mma_major_mode()
self.o_layout = cutlass.utils.LayoutEnum.from_tensor(mO)
if const_expr(self.q_major_mode != tcgen05.OperandMajorMode.K):
raise RuntimeError("The layout of mQ is not supported")
if const_expr(self.k_major_mode != tcgen05.OperandMajorMode.K):
raise RuntimeError("The layout of mK is not supported")
# FP4 block-scaled MMA requires K-major B; standard MMA uses MN-major
if const_expr(mSFV is not None):
if const_expr(self.v_major_mode != tcgen05.OperandMajorMode.K):
raise RuntimeError("The layout of mV must be K-major for FP4 block-scaled MMA")
else:
if const_expr(self.v_major_mode != tcgen05.OperandMajorMode.MN):
raise RuntimeError("The layout of mV is not supported")
# check type consistency
if const_expr(self.q_dtype == cutlass.Int8):
assert self.q_dtype == self.k_dtype
self.q_dtype = self.k_dtype = cutlass.Float4E2M1FN
if const_expr(mSFV is not None):
self.v_dtype = cutlass.Float4E2M1FN
if const_expr(self.q_dtype != self.k_dtype):
raise TypeError(f"Type mismatch: {self.q_dtype} != {self.k_dtype}")
if const_expr(mSFV is not None and self.q_dtype != self.v_dtype):
raise TypeError(f"Type mismatch: {self.q_dtype} != {self.v_dtype} (V quantization requires matching dtype)")
self._setup_attributes()
self.use_tma_O = (
self.arch >= 90
and mCuSeqlensQ is None
and mSeqUsedQ is None
and not self.seqlen_q_static_one
)
# This can be tuned
self.e2e_freq = 16
if const_expr(
self.head_dim_padded > 64 and not self.is_causal and not self.is_local and self.pack_gqa
):
self.e2e_freq = 32 if mCuSeqlensQ is not None or mSeqUsedQ is not None else 10
use_2cta_instrs = self.mma_tiler_qk[0] == 256
assert use_2cta_instrs == False, "Two-CTA instructions not supported yet"
self.cta_group = (
tcgen05.CtaGroup.TWO if use_2cta_instrs else tcgen05.CtaGroup.ONE
)
# the intermediate tensor p is from tmem & mK-major. Transposed, softmax
# holds P in registers rather than tensor memory, and OperandSource only
# selects the source of A, so P has to travel through shared memory.
p_source = (
tcgen05.OperandSource.SMEM
if const_expr(self.transpose_s)
else tcgen05.OperandSource.TMEM
)
p_major_mode = tcgen05.OperandMajorMode.K
# Transposing S makes K the A operand and Q the B operand; both are
# K-major over the head dimension either way, so only the roles move.
qk_a_major = self.k_major_mode if const_expr(self.transpose_s) else self.q_major_mode
qk_b_major = self.q_major_mode if const_expr(self.transpose_s) else self.k_major_mode
# Use block-scaled MMA for PV only if V is being quantized (mSFV is provided)
if const_expr(self.quant_qk):
tiled_mma_qk = sm100_utils_basic.make_blockscaled_trivial_tiled_mma(
self.q_dtype,
qk_a_major,
qk_b_major,
self.sf_dtype,
self.sf_vec_size,
self.cta_group,
self.mma_tiler_qk[:2],
)
else:
tiled_mma_qk = sm100_utils_basic.make_trivial_tiled_mma(
self.q_dtype,
qk_a_major,
qk_b_major,
self.qk_acc_dtype,
self.cta_group,
self.mma_tiler_qk[:2],
)
if const_expr(self.quant_pv):
tiled_mma_pv = sm100_utils_basic.make_blockscaled_trivial_tiled_mma(
self.v_dtype,
p_major_mode,
self.v_major_mode,
self.sf_dtype,
self.sf_vec_size,
self.cta_group,
self.mma_tiler_pv[:2],
p_source,
)
else:
tiled_mma_pv = sm100_utils_basic.make_trivial_tiled_mma(
self.v_dtype,
p_major_mode,
self.v_major_mode,
self.pv_acc_dtype,
self.cta_group,
self.mma_tiler_pv[:2],
p_source,
)
self.cluster_shape_mnk = (*self.cluster_shape_mn, 1)
self.cluster_layout_vmnk = cute.tiled_divide(
cute.make_layout(self.cluster_shape_mnk),
(tiled_mma_qk.thr_id.shape,),
)
self.epi_tile = self.mma_tiler_pv[:2]
# Which builder a tensor uses follows its operand role, not its name.
make_q_smem_layout = (
sm100_utils_basic.make_smem_layout_b
if const_expr(self.transpose_s)
else sm100_utils_basic.make_smem_layout_a
)
make_k_smem_layout = (
sm100_utils_basic.make_smem_layout_a
if const_expr(self.transpose_s)
else sm100_utils_basic.make_smem_layout_b
)
make_sfq_smem_layout = (
make_smem_layout_sfb if const_expr(self.transpose_s) else make_smem_layout_sfa
)
make_sfk_smem_layout = (
make_smem_layout_sfa if const_expr(self.transpose_s) else make_smem_layout_sfb
)
# ((Atom_Inst_M, Atom_Inst_K), MMA_M, MMA_K, STAGE)
sQ_layout = make_q_smem_layout(
tiled_mma_qk,
self.mma_tiler_qk,
self.q_dtype,
self.q_stage,
)
sK_layout = make_k_smem_layout(
tiled_mma_qk,
self.mma_tiler_qk,
self.k_dtype,
self.kv_stage,
)
tP_layout = sm100_utils_basic.make_smem_layout_a(
tiled_mma_pv,
self.mma_tiler_pv,
self.v_dtype,
self.acc_stage,
)
sV_layout = sm100_utils_basic.make_smem_layout_b(
tiled_mma_pv,
self.mma_tiler_pv,
self.v_dtype,
self.kv_stage,
)
sO_layout = sm100_utils_basic.make_smem_layout_epi(
self.o_dtype,
self.o_layout,
self.epi_tile,
self.epi_stage,
)
sfv_smem_layout_staged = None
sfp_smem_layout_staged = None
# # (((Atom_Inst_M, Rest_M),(Atom_Inst_K, Rest_K)), MMA_M, MMA_K, STAGE)
sfq_smem_layout_staged = make_sfq_smem_layout(
tiled_mma_qk,
self.mma_tiler_qk,
self.sf_vec_size,
self.q_stage,
mma_tile_inst_k=self.mma_inst_tile_k,
)
sfk_smem_layout_staged = make_sfk_smem_layout(
tiled_mma_qk,
self.mma_tiler_qk,
self.sf_vec_size,
self.kv_stage,
mma_tile_inst_k=self.mma_inst_tile_k,
)
# Create P scale factor layout for P*V operation (P is the A matrix)
if const_expr(self.quant_pv):
sfp_smem_layout_staged = make_smem_layout_sfa(
tiled_mma_pv,
self.mma_tiler_pv,
self.sf_vec_size,
self.q_stage,
mma_tile_inst_k=self.mma_inst_tile_k,
)
sfv_smem_layout_staged = make_smem_layout_sfb(
tiled_mma_pv,
self.mma_tiler_pv,
self.sf_vec_size,
self.kv_stage,
mma_tile_inst_k=self.mma_inst_tile_k,
)
if const_expr(not self.same_hdim_kv_padded):
# sK and sV are using the same physical smem so we need to adjust the stride so that they line up
stride_sK = const_expr(
max(sK_layout.outer.stride[-1], 0)
) # take max to turn tuple to Int32
stride_sV = const_expr(max(sV_layout.outer.stride[-1], 0))
stage_stride = const_expr(
max(stride_sK, stride_sV)
if not self.uneven_kv_smem
else (stride_sK + stride_sV) // 2
)
sK_layout = cute.make_composed_layout(
sK_layout.inner,
0,
cute.make_layout(
(*sK_layout.outer.shape[:-1], self.kv_stage),
stride=(*sK_layout.outer.stride[:-1], stage_stride),
),
)
sV_layout = cute.make_composed_layout(
sV_layout.inner,
0,
cute.make_layout(
(*sV_layout.outer.shape[:-1], self.kv_stage),
stride=(*sV_layout.outer.stride[:-1], stage_stride),
),
)
mQ_shape_unpacked = mQ.shape
def packed_rows(t, head_mode: int, heads: int):
"""Shape and stride of the packed row mode of a (sq, ..., h, ...) tensor.
With a single query position the (head, seqlen) mode collapses to
the head axis alone. A flat axis of ``heads`` rows tiles by the
M tile for any ratio, with TMA zero-filling the rows past it; the
hierarchical mode only tiles when the tile is a whole number of
query groups.
"""
if const_expr(self.seqlen_q_static_one):
return heads, t.stride[head_mode]
return (heads, t.shape[0]), (t.stride[head_mode], t.stride[0])
if const_expr(self.pack_gqa):
q_rows_shape, q_rows_stride = packed_rows(mQ, 2, self.qhead_per_kvhead)
shape_Q_packed = (
q_rows_shape, # qhead_per_kvhead, or (qhead_per_kvhead, sq)
mQ.shape[1], # d
mK.shape[2], # h_k
*mQ.shape[3:], # b
)
stride_Q_packed = (
q_rows_stride,
mQ.stride[1],
mQ.stride[2] * self.qhead_per_kvhead,
*mQ.stride[3:],
)
mQ = cute.make_tensor(
mQ.iterator, cute.make_layout(shape_Q_packed, stride=stride_Q_packed)
)
mQSecond = cute.make_tensor(
mQSecond.iterator,
cute.make_layout(shape_Q_packed, stride=stride_Q_packed),
)
o_rows_shape, o_rows_stride = packed_rows(mO, 2, self.qhead_per_kvhead_O)
shape_O_packed = (
o_rows_shape,
mO.shape[1],
mK.shape[2],
*mO.shape[3:],
)
stride_O_packed = (
o_rows_stride,
mO.stride[1],
mO.stride[2] * self.qhead_per_kvhead_O,
*mO.stride[3:],
)
mO = cute.make_tensor(
mO.iterator, cute.make_layout(shape_O_packed, stride=stride_O_packed)
)
if const_expr(mLSE is not None):
lse_rows_shape, lse_rows_stride = packed_rows(
mLSE, 1, self.qhead_per_kvhead
)
shape_LSE_packed = (
lse_rows_shape,
mK.shape[2],
*mLSE.shape[2:],
)
stride_LSE_packed = (
lse_rows_stride,
mLSE.stride[1] * self.qhead_per_kvhead,
*mLSE.stride[2:],
)
mLSE = cute.make_tensor(
mLSE.iterator, cute.make_layout(shape_LSE_packed, stride=stride_LSE_packed)
)
self.tma_copy_bytes = {
name: cute.size_in_bytes(mX.element_type, cute.select(layout, mode=[0, 1, 2]))
for name, mX, layout in [
("Q", mQ, sQ_layout),
("K", mK, sK_layout),
("V", mV, sV_layout),
]
}
# Add scale factor copy bytes to Q/K/V since they use the same barrier
if const_expr(self.quant_qk):
self.tma_copy_bytes["Q"] += cute.size_in_bytes(mSFQ.element_type, cute.select(sfq_smem_layout_staged, mode=[0, 1, 2]))
self.tma_copy_bytes["K"] += cute.size_in_bytes(mSFK.element_type, cute.select(sfk_smem_layout_staged, mode=[0, 1, 2]))
self.tma_copy_bytes["Q"] *= self.query_terms
if const_expr(self.quant_pv):
self.tma_copy_bytes["V"] += cute.size_in_bytes(mSFV.element_type, cute.select(sfv_smem_layout_staged, mode=[0, 1, 2]))
# TMA load for Q
tma_load_op = cpasync.CopyBulkTensorTileG2SOp(self.cta_group)
tma_store_op = cpasync.CopyBulkTensorTileS2GOp()
mQ_shape = mQ.shape
# The FP4 Q tile is the B operand once S is transposed.
make_q_tma_atom = (
cute.nvgpu.make_tiled_tma_atom_B
if const_expr(self.transpose_s)
else cute.nvgpu.make_tiled_tma_atom_A
)
tma_atom_Q, mQ = make_q_tma_atom(
tma_load_op,
mQ,
cute.select(sQ_layout, mode=[0, 1, 2]),
self.mma_tiler_qk,
tiled_mma_qk,
self.cluster_layout_vmnk.shape,
)
tma_atom_Q_second, mQ_second_tma = make_q_tma_atom(
tma_load_op,
mQSecond,
cute.select(sQ_layout, mode=[0, 1, 2]),
self.mma_tiler_qk,
tiled_mma_qk,
self.cluster_layout_vmnk.shape,
)
tma_atom_K = None
tma_atom_V = None
mK_shape = mK.shape
mV_shape = mV.shape
if const_expr(self.use_tma_KV):
# TMA load for K — the A operand once S is transposed.
make_k_tma_atom = (
cute.nvgpu.make_tiled_tma_atom_A
if const_expr(self.transpose_s)
else cute.nvgpu.make_tiled_tma_atom_B
)
tma_atom_K, mK = make_k_tma_atom(
tma_load_op,
mK,
cute.select(sK_layout, mode=[0, 1, 2]),
self.mma_tiler_qk,
tiled_mma_qk,
self.cluster_layout_vmnk.shape,
)
# TMA load for V
tma_atom_V, mV = cute.nvgpu.make_tiled_tma_atom_B(
tma_load_op,
mV,
cute.select(sV_layout, mode=[0, 1, 2]),
self.mma_tiler_pv,
tiled_mma_pv,
self.cluster_layout_vmnk.shape,
)
tiled_mma_qk_bf16 = None
tiled_mma_pv_bf16 = None
sQ_bf16_layout = None
sK_bf16_layout = None
sV_bf16_layout = None
tP_bf16_layout = None
tma_atom_Q_bf16 = None
tma_atom_K_bf16 = None
tma_atom_V_bf16 = None
mResidualQ_t = None
mResidualK_t = None
mResidualV_t = None
if const_expr(self.fused_residual_first_block):
if const_expr(self.residual_source == "contiguous"):
mResK_src = mResidualK
mResV_src = mResidualV
else:
# paged_bf16 — caller must supply both cache tensors.
mResK_src = mResidualKCache
mResV_src = mResidualVCache
# mResK_src has logical shape (B-or-num_blocks, S=128, H_kv, D) bf16.
# Apply the same transpose pattern as the BF16 reference kernel:
# (b, s, h, d) -> (s, d, h, b) via mode=[1, 3, 2, 0].
mResK = cute.make_tensor(
mResK_src.iterator,
cute.select(mResK_src.layout, mode=[1, 3, 2, 0]),
)
mResV = cute.make_tensor(
mResV_src.iterator,
cute.select(mResV_src.layout, mode=[1, 3, 2, 0]),
)
# BF16 V is MN-major (not transposed): (s, d, h, b) -> (d, s, h, b)
mResV = cute.make_tensor(
mResV.iterator,
cute.select(mResV.layout, mode=[1, 0, 2, 3]),
)
# BF16 GEMMs use a non-blockscaled tiled MMA (standard SM100 BF16 tcgen05).
# We re-use the existing q_major_mode/k_major_mode/v_major_mode
# since the residual K/V are constructed with the same logical layout
# transposes as the BF16 reference kernel.
res_k_dtype = mResK.element_type # cutlass.BFloat16
res_v_dtype = mResV.element_type # cutlass.BFloat16
res_k_major = cutlass.utils.LayoutEnum.from_tensor(mResK).mma_major_mode()
res_v_major = cutlass.utils.LayoutEnum.from_tensor(mResV).mma_major_mode()
# The residual follows the FP4 blocks' orientation so that both write
# the same tensor-memory tile the same way round and one softmax
# reads them. Transposed, K is the A operand and Q the B operand.
tiled_mma_qk_bf16 = sm100_utils_basic.make_trivial_tiled_mma(
res_k_dtype,
res_k_major if const_expr(self.transpose_s) else self.q_major_mode,
self.q_major_mode if const_expr(self.transpose_s) else res_k_major,
self.qk_acc_dtype,
self.cta_group,
self.mma_tiler_qk[:2],
)
tiled_mma_pv_bf16 = sm100_utils_basic.make_trivial_tiled_mma(
res_v_dtype,
p_major_mode,
res_v_major,
self.pv_acc_dtype,
self.cta_group,
self.mma_tiler_pv[:2],
p_source,
)
# SMEM layouts for BF16 residual K/V (single stage — block 0 only).
make_k_bf16_smem_layout = (
sm100_utils_basic.make_smem_layout_a
if const_expr(self.transpose_s)
else sm100_utils_basic.make_smem_layout_b
)
sK_bf16_layout = make_k_bf16_smem_layout(
tiled_mma_qk_bf16,
self.mma_tiler_qk,
res_k_dtype,
1, # 1 stage suffices: residual block 0 is loaded once per CTA-tile.
)
sV_bf16_layout = sm100_utils_basic.make_smem_layout_b(
tiled_mma_pv_bf16,
self.mma_tiler_pv,
res_v_dtype,
1,
)
tP_bf16_layout = sm100_utils_basic.make_smem_layout_a(
tiled_mma_pv_bf16,
self.mma_tiler_pv,
res_v_dtype, # BFloat16
1,
)
# TMA atoms for BF16 residual K/V.
make_k_bf16_tma_atom = (
cute.nvgpu.make_tiled_tma_atom_A
if const_expr(self.transpose_s)
else cute.nvgpu.make_tiled_tma_atom_B
)
tma_atom_K_bf16, mResidualK_t = make_k_bf16_tma_atom(
tma_load_op,
mResK,
cute.select(sK_bf16_layout, mode=[0, 1, 2]),
self.mma_tiler_qk,
tiled_mma_qk_bf16,
self.cluster_layout_vmnk.shape,
)
tma_atom_V_bf16, mResidualV_t = cute.nvgpu.make_tiled_tma_atom_B(
tma_load_op,
mResV,
cute.select(sV_bf16_layout, mode=[0, 1, 2]),
self.mma_tiler_pv,
tiled_mma_pv_bf16,
self.cluster_layout_vmnk.shape,
)
mResQ = cute.make_tensor(
mResidualQ.iterator,
cute.select(mResidualQ.layout, mode=[1, 3, 2, 0]),
)
if const_expr(self.pack_gqa):
res_rows_shape, res_rows_stride = packed_rows(
mResQ, 2, self.qhead_per_kvhead
)
shape_ResQ_packed = (
res_rows_shape,
mResQ.shape[1],
mK.shape[2],
*mResQ.shape[3:],
)
stride_ResQ_packed = (
res_rows_stride,
mResQ.stride[1],
mResQ.stride[2] * self.qhead_per_kvhead,
*mResQ.stride[3:],
)
mResQ = cute.make_tensor(
mResQ.iterator,
cute.make_layout(shape_ResQ_packed, stride=stride_ResQ_packed),
)
res_q_dtype = mResQ.element_type # cutlass.BFloat16
make_q_bf16_smem_layout = (
sm100_utils_basic.make_smem_layout_b
if const_expr(self.transpose_s)
else sm100_utils_basic.make_smem_layout_a
)
sQ_bf16_layout = make_q_bf16_smem_layout(
tiled_mma_qk_bf16,
self.mma_tiler_qk,
res_q_dtype,
1,
)
make_q_bf16_tma_atom = (
cute.nvgpu.make_tiled_tma_atom_B
if const_expr(self.transpose_s)
else cute.nvgpu.make_tiled_tma_atom_A
)
tma_atom_Q_bf16, mResidualQ_t = make_q_bf16_tma_atom(
tma_load_op,
mResQ,
cute.select(sQ_bf16_layout, mode=[0, 1, 2]),
self.mma_tiler_qk,
tiled_mma_qk_bf16,
self.cluster_layout_vmnk.shape,
)
# TMA load for scale factors
tma_atom_sfq = None
tma_tensor_sfq = None
tma_atom_sfk = None
tma_tensor_sfk = None
tma_atom_sfv = None
tma_tensor_sfv = None
if const_expr(self.quant_qk):
sfq_layout = cute.tile_to_shape(blockscaled_utils.BlockScaledBasicChunk(self.sf_vec_size).layout, mQ_shape_unpacked, (2, 1, 3, 4))
sfq_op = sm100_utils_basic.cluster_shape_to_tma_atom_A(
self.cluster_shape_mn, tiled_mma_qk.thr_id
)
mSFQ = cute.make_tensor(mSFQ.iterator, sfq_layout)
mSFQSecond = cute.make_tensor(mSFQSecond.iterator, sfq_layout)
if const_expr(self.pack_gqa):
shape_SFQ_packed = (
mSFQ.shape[0], # ((32,4), rest_m=1) unchanged
mSFQ.shape[1], # K modes unchanged
(mSFQ.shape[2][0], mK_shape[2]), # (1, h_kv=8) — use pre-TMA shape
mSFQ.shape[3], # (1, b=1)
)
stride_SFQ_packed = (
mSFQ.stride[0], # ((16,4), rest_m_stride) unchanged
mSFQ.stride[1], # K strides unchanged
mSFQ.stride[2], # (0, h_sf_stride=1024) unchanged
mSFQ.stride[3], # batch unchanged
)
mSFQ = cute.make_tensor(
mSFQ.iterator,
cute.make_layout(shape_SFQ_packed, stride=stride_SFQ_packed),
)
mSFQSecond = cute.make_tensor(
mSFQSecond.iterator,
cute.make_layout(
shape_SFQ_packed, stride=stride_SFQ_packed
),
)
tma_atom_sfq, tma_tensor_sfq = cute.nvgpu.make_tiled_tma_atom_A(
sfq_op,
mSFQ,
cute.select(sfq_smem_layout_staged, mode=[0, 1, 2]),
self.mma_tiler_qk,
tiled_mma_qk,
self.cluster_layout_vmnk.shape,
internal_type=cutlass.Int16,
)
(
tma_atom_sfq_second,
tma_tensor_sfq_second,
) = cute.nvgpu.make_tiled_tma_atom_A(
sfq_op,
mSFQSecond,
cute.select(sfq_smem_layout_staged, mode=[0, 1, 2]),
self.mma_tiler_qk,
tiled_mma_qk,
self.cluster_layout_vmnk.shape,
internal_type=cutlass.Int16,
)
if const_expr(self.quant_qk):
# Setup TMA load for SFK (scale factor for K, like SFB).
sfk_op = sm100_utils_basic.cluster_shape_to_tma_atom_SFB(
self.cluster_shape_mn, tiled_mma_qk.thr_id
)
sfk_layout = cute.tile_to_shape(blockscaled_utils.BlockScaledBasicChunk(self.sf_vec_size).layout, mK_shape, (2, 1, 3, 4))
sfk_layout = _sf_layout_with_page_stride(sfk_layout, k_sf_page_stride)
mSFK = cute.make_tensor(mSFK.iterator, sfk_layout)
# For SFB, compute mma_inst_shape_mnk_sfb: (M // (2 if use_2cta_instrs else 1), round_up(N, 128), K)
mma_inst_shape_mnk_qk = (
self.mma_tiler_qk[0],
self.mma_tiler_qk[1],
self.mma_inst_bits_k // self.k_dtype.width,
)
mma_inst_shape_mnk_sfb_qk = (
mma_inst_shape_mnk_qk[0] // (2 if use_2cta_instrs else 1),
cute.round_up(mma_inst_shape_mnk_qk[1], 128),
mma_inst_shape_mnk_qk[2],
)
mma_tiler_sfb_qk = (
mma_inst_shape_mnk_sfb_qk[0],
mma_inst_shape_mnk_sfb_qk[1],
mma_inst_shape_mnk_sfb_qk[2] * self.mma_inst_tile_k,
)
# For SFB, we need a separate tiled_mma_sfb with CtaGroup.ONE
tiled_mma_sfb_qk = sm100_utils_basic.make_blockscaled_trivial_tiled_mma(
self.k_dtype,
self.k_major_mode,
self.k_major_mode,
self.sf_dtype,
self.sf_vec_size,
self.cta_group,
mma_inst_shape_mnk_sfb_qk[:2],
)
cluster_layout_sfb_vmnk = cute.tiled_divide(
cute.make_layout(self.cluster_shape_mnk),
(tiled_mma_sfb_qk.thr_id.shape,),
)
tma_atom_sfk, tma_tensor_sfk = cute.nvgpu.make_tiled_tma_atom_B(
sfk_op,
mSFK,
cute.select(sfk_smem_layout_staged, mode=[0, 1, 2]),
mma_tiler_sfb_qk,
tiled_mma_sfb_qk,
cluster_layout_sfb_vmnk.shape,
internal_type=cutlass.Int16,
)
if const_expr(self.quant_pv):
# Setup TMA load for SFV (scale factor for V, like SFB)
sfv_op = sm100_utils_basic.cluster_shape_to_tma_atom_SFB(
self.cluster_shape_mn, tiled_mma_pv.thr_id
)
# Setup scale factor tensor layout
sfv_layout = cute.tile_to_shape(blockscaled_utils.BlockScaledBasicChunk(self.sf_vec_size).layout, mV_shape, (2, 1, 3, 4))
sfv_layout = _sf_layout_with_page_stride(sfv_layout, v_sf_page_stride)
mSFV = cute.make_tensor(mSFV.iterator, sfv_layout)
# For SFB, compute mma_inst_shape_mnk_sfb: (M // (2 if use_2cta_instrs else 1), round_up(N, 128), K)
mma_inst_shape_mnk_pv = ( # the same processed by one tcgen05.mma instruction
self.mma_tiler_pv[0],
self.mma_tiler_pv[1],
self.mma_inst_bits_k // self.v_dtype.width,
)
use_2cta_instrs = self.mma_tiler_pv[0] == 256
mma_inst_shape_mnk_sfb_pv = (
mma_inst_shape_mnk_pv[0] // (2 if use_2cta_instrs else 1),
cute.round_up(mma_inst_shape_mnk_pv[1], 128),
mma_inst_shape_mnk_pv[2],
)
mma_tiler_sfb_pv = (
mma_inst_shape_mnk_sfb_pv[0],
mma_inst_shape_mnk_sfb_pv[1],
mma_inst_shape_mnk_sfb_pv[2] * self.mma_inst_tile_k,
)
# For SFB, we need a separate tiled_mma_sfb with CtaGroup.ONE
tiled_mma_sfb_pv = sm100_utils_basic.make_blockscaled_trivial_tiled_mma(
self.v_dtype,
p_major_mode,
self.v_major_mode,
self.sf_dtype,
self.sf_vec_size,
self.cta_group,
mma_inst_shape_mnk_sfb_pv[:2],
p_source,
)
cluster_layout_sfb_pv_vmnk = cute.tiled_divide(
cute.make_layout(self.cluster_shape_mnk),
(tiled_mma_sfb_pv.thr_id.shape,),
)
tma_atom_sfv, tma_tensor_sfv = cute.nvgpu.make_tiled_tma_atom_B(
sfv_op,
mSFV,
cute.select(sfv_smem_layout_staged, mode=[0, 1, 2]),
mma_tiler_sfb_pv,
tiled_mma_sfb_pv,
cluster_layout_sfb_pv_vmnk.shape,
internal_type=cutlass.Int16,
)
o_cta_v_layout = cute.composition(cute.make_identity_layout(mO.shape), self.epi_tile)
self.num_epilogue_threads = cute.arch.WARP_SIZE * len(self.epilogue_warp_ids)
if const_expr(self.use_tma_O):
tma_atom_O, mO = cpasync.make_tiled_tma_atom(
tma_store_op,
mO,
cute.select(sO_layout, mode=[0, 1]),
o_cta_v_layout,
)
gmem_tiled_copy_O = None
else:
tma_atom_O = None
universal_copy_bits = 128
async_copy_elems = universal_copy_bits // self.o_dtype.width
atom_universal_copy = cute.make_copy_atom(
cute.nvgpu.CopyUniversalOp(),
self.o_dtype,
num_bits_per_copy=universal_copy_bits,
)
tO_shape_dim_1 = sO_layout.outer.shape[1][0] // async_copy_elems
tO_layout = cute.make_ordered_layout(
(self.num_epilogue_threads // tO_shape_dim_1, tO_shape_dim_1),
order=(1, 0),
)
# So that we don't have to check if we overshoot kBlockM when we store O
assert self.m_block_size % tO_layout.shape[0] == 0
vO_layout = cute.make_layout((1, async_copy_elems))
gmem_tiled_copy_O = cute.make_tiled_copy_tv(atom_universal_copy, tO_layout, vO_layout)
if const_expr(mCuSeqlensQ is not None or mSeqUsedQ is not None):
TileScheduler = SingleTileVarlenScheduler
else:
if const_expr(self.is_causal or self.is_local):
TileScheduler = SingleTileLPTScheduler
else:
TileScheduler = (
SingleTileScheduler
if const_expr(not self.is_persistent)
else StaticPersistentTileScheduler
)
tile_sched_args = TileSchedulerArguments(
cute.ceil_div(cute.size(mQ.shape[0]), self.cta_tiler[0]),
cute.size(mQ.shape[2]),
cute.size(mQ.shape[3])
if const_expr(mCuSeqlensQ is None)
else cute.size(mCuSeqlensQ.shape[0] - 1),
num_splits,
cute.size(mK.shape[0])
if const_expr(mPageTable is None)
else mK.shape[0] * mPageTable.shape[1],
mQ.shape[1],
mV.shape[0], # Note that this is different from Sm90 since we transpose mV in Sm100
total_q=cute.size(mQ.shape[0])
if const_expr(mCuSeqlensQ is not None)
else cute.size(mQ.shape[0]) * cute.size(mQ.shape[3]),
tile_shape_mn=self.cta_tiler[:2],
mCuSeqlensQ=mCuSeqlensQ,
mSeqUsedQ=mSeqUsedQ,
qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1,
# For sub-byte dtypes (FP4), width=4 rounds to 0 under integer division;
# clamp so size_one_head * element_size in the scheduler's L2 swizzle calc
# doesn't become 0 and divide by zero in the else branch of the ifexp.
element_size=max(self.k_dtype.width // 8, 1),
is_persistent=self.is_persistent,
lpt=self.is_causal or self.is_local,
is_split_kv=self.is_split_kv,
)
tile_sched_params = TileScheduler.to_underlying_arguments(tile_sched_args)
self.tile_scheduler_cls = TileScheduler
grid_dim = TileScheduler.get_grid_shape(tile_sched_params)
self.mbar_load_q_full_offset = 0
self.mbar_load_q_empty_offset = self.mbar_load_q_full_offset + self.q_stage
self.mbar_load_kv_full_offset = self.mbar_load_q_empty_offset + self.q_stage
self.mbar_load_kv_empty_offset = self.mbar_load_kv_full_offset + self.kv_stage
self.mbar_P_full_O_rescaled_offset = self.mbar_load_kv_empty_offset + self.kv_stage
self.mbar_S_full_offset = self.mbar_P_full_O_rescaled_offset + self.q_stage
self.mbar_O_full_offset = self.mbar_S_full_offset + self.q_stage
self.mbar_softmax_corr_full_offset = self.mbar_O_full_offset + self.q_stage
self.mbar_softmax_corr_empty_offset = self.mbar_softmax_corr_full_offset + self.q_stage
self.mbar_corr_epi_full_offset = self.mbar_softmax_corr_empty_offset + self.epi_stage
self.mbar_corr_epi_empty_offset = self.mbar_corr_epi_full_offset + self.epi_stage
self.mbar_s0_s1_sequence_offset = self.mbar_corr_epi_empty_offset + self.q_stage
self.mbar_tmem_dealloc_offset = self.mbar_s0_s1_sequence_offset + 8
self.mbar_P_full_2_offset = self.mbar_tmem_dealloc_offset + 1
# QK and PV SF tmem load wait for softmax t2r store
self.mbar_sfqk_load_offset = self.mbar_P_full_2_offset + self.q_stage
# The PV GEMM has read the shared-memory P and its scale factors.
self.mbar_P_consumed_offset = self.mbar_sfqk_load_offset + self.q_stage
self.mbar_q_fp4_ready_offset = self.mbar_P_consumed_offset + self.q_stage
self.mbar_residual_kv_full_offset = self.mbar_q_fp4_ready_offset + self.q_stage
# `self.fused_residual_first_block` is a constant (set in __init__) so
# the branch is selected at trace time; no const_expr wrapper needed
# since these are plain Python int assignments.
_residual_kv_slots = 3 if self.fused_residual_first_block else 0
self.mbar_residual_kv_empty_offset = self.mbar_residual_kv_full_offset + _residual_kv_slots
self.mbar_bf16_S_full_offset = self.mbar_residual_kv_empty_offset + _residual_kv_slots
_bf16_phase_slots = 3 if self.fused_residual_first_block else 0
self.mbar_bf16_P_full_offset = self.mbar_bf16_S_full_offset + (1 if self.fused_residual_first_block else 0)
self.mbar_bf16_P_full_2_offset = self.mbar_bf16_P_full_offset + (1 if self.fused_residual_first_block else 0)
self.mbar_bf16_PV_done_offset = self.mbar_bf16_P_full_2_offset + (1 if self.fused_residual_first_block else 0)
self.mbar_total = self.mbar_bf16_PV_done_offset + (1 if self.fused_sink_block else 0)
# self.mbar_total = self.mbar_P_full_2_offset + self.q_stage
self.mbar_p_split = lambda k: (k // 4 * 3 if cutlass.const_expr(self.v_dtype.width >= 8) else k // 2 )
sO_size = (
cute.cosize(sO_layout)
if const_expr(not (self.overlap_sO_sQ or self.overlap_sO_bf16))
else 1
)
# The query terms and their scale factors live in sQ only while QK
# runs and sO only after the last MMA, so an overlapped region holds
# the larger of the two, not one O tile per term. The scale factors
# follow the Q tiles there, which leaves sSFQ a placeholder.
q_terms_size = self.query_terms * cute.cosize(sQ_layout)
sfq_terms_size = (
self.query_terms * cute.cosize(sfq_smem_layout_staged)
if const_expr(self.quant_qk)
else 0
)
sQ_size = (
cutlass.max(
q_terms_size + sfq_terms_size * 8 // self.q_dtype.width,
cute.cosize(sO_layout) * self.o_dtype.width // self.q_dtype.width,
)
if const_expr(self.overlap_sO_sQ)
else q_terms_size
)
# Calculate scale factor shared memory sizes
# Use size 1 as minimum to avoid alignment issues when size is 0
sfq_smem_size = (
sfq_terms_size
if const_expr(self.quant_qk and not self.overlap_sO_sQ)
else 1
)
sfk_smem_size = cute.cosize(sfk_smem_layout_staged) if const_expr(self.quant_qk) else 1
sfp_smem_size = cute.cosize(sfp_smem_layout_staged) if const_expr(self.quant_pv) else 1
sfv_smem_size = cute.cosize(sfv_smem_layout_staged) if const_expr(self.quant_pv) else 1
# P leaves tensor memory under the transposed layout, so the same tile
# the MMA reads as its A operand needs real storage. sSoftmaxRed carries
# the per-warp partial row maxima between the four softmax warps.
sP_smem_size = cute.cosize(tP_layout) if const_expr(self.transpose_s) else 1
softmax_red_size = (
self.softmax_red_slots if const_expr(self.transpose_s) else 1
)
sK_bf16_smem_size = (
cute.cosize(sK_bf16_layout) if const_expr(self.fused_residual_first_block) else 1
)
sV_bf16_smem_size = (
cute.cosize(sV_bf16_layout) if const_expr(self.fused_residual_first_block) else 1
)
# Transposed, the residual's P also reaches the tensor core from shared
# memory. It is the same 128x128 of bf16 as the residual Q tile and the
# QK GEMM, whose completion softmax waits on before writing P, is
# exactly when that tile dies, so the two share the bytes.
sQ_bf16_smem_size = (
max(cute.cosize(sQ_bf16_layout), cute.cosize(tP_bf16_layout))
if const_expr(self.fused_residual_first_block)
else 1
)
@cute.struct
class BF16Tiles:
sK_bf16: cute.struct.Align[
cute.struct.MemRange[cutlass.BFloat16, sK_bf16_smem_size],
self.buffer_align_bytes,
]
sV_bf16: cute.struct.Align[
cute.struct.MemRange[cutlass.BFloat16, sV_bf16_smem_size],
self.buffer_align_bytes,
]
sQ_bf16: cute.struct.Align[
cute.struct.MemRange[cutlass.BFloat16, sQ_bf16_smem_size],
self.buffer_align_bytes,
]
# K reuses V's buffer when K is smaller (FP4 K in BF16 V), or same dtype
k_aliases_v = self.k_dtype.width < self.v_dtype.width
kv_stages_storage = cute.struct.Align[
cute.struct.MemRange[self.k_dtype, 1] if const_expr(k_aliases_v) else cute.struct.MemRange[self.k_dtype, cute.cosize(sK_layout)],
1 if const_expr(k_aliases_v) else self.buffer_align_bytes,
]
# A CTA on the split of the BF16 blocks holds no FP4 K/V stage and the
# others hold no BF16 tile, so with that split the two share one
# region in place of sK.
@cute.union
class KVStagesOrBF16Tiles:
stages: kv_stages_storage
bf16: BF16Tiles
@cute.union
class BF16TilesOrO:
bf16: BF16Tiles
o: cute.struct.Align[
cute.struct.MemRange[self.o_dtype, cute.cosize(sO_layout)],
self.buffer_align_bytes,
]
o_storage = (
BF16TilesOrO
if const_expr(self.overlap_sO_bf16)
else cute.struct.Align[
cute.struct.MemRange[self.o_dtype, sO_size],
self.buffer_align_bytes,
]
)
# Every block the tensor core reads its operands out of the K/V stages,
# P and Q while TMA refills the stages. The softmax's cross-warp row
# reduction goes through sSoftmaxRed on its critical path every block,
# and in the same half of shared memory as that traffic its loads wait
# behind it: with sSoftmaxRed, P and the stages in one half,
# Qwen3-30B's merged decode ran 8% slower and Qwen3-4B's 3% slower.
# So the streamed buffers come first, O (or the BF16 tiles), cold
# through the main loop, carries what follows into the other half,
# and sSoftmaxRed is last.
@cute.struct
class SharedStorage:
sK: KVStagesOrBF16Tiles if const_expr(self.bf16_split) else kv_stages_storage
sV: cute.struct.Align[
cute.struct.MemRange[self.v_dtype, cute.cosize(sV_layout)] if not const_expr(self.v_dtype == self.k_dtype) else cute.struct.MemRange[self.k_dtype, 1],
self.buffer_align_bytes if not const_expr(self.v_dtype == self.k_dtype) else 1,
]
# Scale factor shared memory (if block-scaled quantization is used)
sSFK: cute.struct.Align[
cute.struct.MemRange[self.sf_dtype, sfk_smem_size],
self.buffer_align_bytes,
]
sSFV: cute.struct.Align[
cute.struct.MemRange[self.sf_dtype, sfv_smem_size],
self.buffer_align_bytes,
]
sP: cute.struct.Align[
cute.struct.MemRange[self.v_dtype, sP_smem_size],
self.buffer_align_bytes if const_expr(self.transpose_s) else 1,
]
sQ: cute.struct.Align[
cute.struct.MemRange[self.q_dtype, sQ_size],
self.buffer_align_bytes,
]
sSFQ: cute.struct.Align[
cute.struct.MemRange[self.sf_dtype, sfq_smem_size],
self.buffer_align_bytes,
]
sO: o_storage
sScale: cute.struct.Align[cute.struct.MemRange[Float32, max(self.q_stage * self.m_block_size * 2, self.m_block_size * 4)], self.buffer_align_bytes]
sSFP: cute.struct.Align[
cute.struct.MemRange[self.sf_dtype, sfp_smem_size],
self.buffer_align_bytes,
]
# m_barriers for pipelines
mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.mbar_total]
# Tmem holding buffer
tmem_holding_buf: Int32
sSoftmaxRed: cute.struct.Align[
cute.struct.MemRange[Float32, softmax_red_size],
self.buffer_align_bytes if const_expr(self.transpose_s) else 1,
]
# Remove scale factors to avoid OOM. Seems I can't set their size to 0
@cute.struct
class SharedStorageBF16:
# m_barriers for pipelines
mbar_ptr: cute.struct.MemRange[cutlass.Int64, self.mbar_total]
# Tmem holding buffer
tmem_holding_buf: Int32
sScale: cute.struct.Align[cute.struct.MemRange[Float32, max(self.q_stage * self.m_block_size * 2, self.m_block_size * 4)], self.buffer_align_bytes]
sO: cute.struct.Align[
cute.struct.MemRange[self.o_dtype, sO_size],
self.buffer_align_bytes,
]
sQ: cute.struct.Align[
cute.struct.MemRange[self.q_dtype, sQ_size],
self.buffer_align_bytes,
]
sK: cute.struct.Align[
# cute.cosize(sK_layout) is correct even in the case of self.uneven_kv_smem
cute.struct.MemRange[self.k_dtype, cute.cosize(sK_layout)],
self.buffer_align_bytes,
]
self.shared_storage = SharedStorage if const_expr(self.quant_qk) or const_expr(self.quant_pv) else SharedStorageBF16
# Verify shared memory fits within budget
total_smem_bytes = self.shared_storage.size_in_bytes()
assert total_smem_bytes <= 227 * 1024, (
f"SharedStorage {total_smem_bytes // 1024}KB exceeds 227KB limit. "
f"Reduce kv_stage (currently {self.kv_stage})."
)
print(f"Total shared memory used: {total_smem_bytes / 1024:.2f} KB")
sO_bytes = (
cute.size_in_bytes(self.o_dtype, sO_layout)
if const_expr(not (self.overlap_sO_sQ or self.overlap_sO_bf16))
else 0
)
sQ_bytes = cute.size_in_bytes(self.q_dtype, sQ_layout)
sK_bytes = cute.size_in_bytes(self.k_dtype, sK_layout)
sV_bytes = cute.size_in_bytes(self.v_dtype, sV_layout)
sfq_bytes = cute.size_in_bytes(cutlass.Uint8, sfq_smem_layout_staged)
sfk_bytes = cute.size_in_bytes(cutlass.Uint8, sfk_smem_layout_staged)
sfp_bytes = cute.size_in_bytes(cutlass.Uint8, sfp_smem_layout_staged) if const_expr(sfp_smem_layout_staged is not None) else 0
sfv_bytes = cute.size_in_bytes(cutlass.Uint8, sfv_smem_layout_staged) if const_expr(sfv_smem_layout_staged is not None) else 0
print(f"sO_size: {sO_bytes / 1024:.2f} KB")
print(f"sQ_size: {sQ_bytes / 1024:.2f} KB")
print(f"sK_size: {sK_bytes / 1024:.2f} KB")
if const_expr(self.v_dtype != self.k_dtype):
print(f"sV_size: {sV_bytes / 1024:.2f} KB")
print(f"sfq_smem_size: {sfq_bytes / 1024:.2f} KB")
print(f"sfk_smem_size: {sfk_bytes / 1024:.2f} KB")
print(f"sfp_smem_size: {sfp_bytes / 1024:.2f} KB")
print(f"sfv_smem_size: {sfv_bytes / 1024:.2f} KB")
LOG2_E = math.log2(math.e)
if const_expr(self.score_mod is None):
softmax_scale_log2 = softmax_scale * LOG2_E
softmax_scale = None
else:
# NB: If a users passes in a score mod, we want to apply the score-mod in the sm_scaled qk
# But in the original base 10. We hijack softmax_scale_log2 to just be the change of base
# and correctly apply the softmax_scale prior to score_mod in the softmax step
softmax_scale_log2 = LOG2_E
softmax_scale = softmax_scale
if const_expr(window_size_left is not None):
window_size_left = Int32(window_size_left)
if const_expr(window_size_right is not None):
window_size_right = Int32(window_size_right)
fastdiv_mods = None
if cutlass.const_expr(aux_tensors is not None):
seqlen_q = cute.size(mQ.shape[0]) // (
self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1
)
seqlen_k = cute.size(mK.shape[0])
seqlen_q_divmod = FastDivmod.create(seqlen_q)
seqlen_k_divmod = FastDivmod.create(seqlen_k)
fastdiv_mods = (seqlen_q_divmod, seqlen_k_divmod)
self.use_block_sparsity = cutlass.const_expr(blocksparse_tensors is not None)
if cutlass.const_expr(self.use_block_sparsity and mPageTable is not None):
raise NotImplementedError("Block sparsity + paged KV not supported on SM100")
# Launch the kernel synchronously
self.kernel(
mQ,
mQ_second_tma,
mK,
mV,
mO,
mLSE,
mCuSeqlensQ,
mCuSeqlensK,
mSeqUsedQ,
mSeqUsedK,
mPageTable,
tma_atom_Q,
tma_atom_Q_second,
tma_atom_K,
tma_atom_V,
tma_atom_O,
softmax_scale_log2,
softmax_scale,
window_size_left,
window_size_right,
learnable_sink,
blocksparse_tensors,
sQ_layout,
sK_layout,
tP_layout,
sV_layout,
sO_layout,
gmem_tiled_copy_O,
tiled_mma_qk,
tiled_mma_pv,
tile_sched_params,
num_splits,
aux_tensors,
fastdiv_mods,
tma_atom_sfq,
tma_tensor_sfq,
tma_atom_sfq_second,
tma_tensor_sfq_second,
tma_atom_sfk,
tma_tensor_sfk,
tma_atom_sfv,
tma_tensor_sfv,
sfq_smem_layout_staged,
sfk_smem_layout_staged,
sfp_smem_layout_staged,
sfv_smem_layout_staged,
sQ_bf16_layout,
tiled_mma_qk_bf16,
mResidualK_t,
mResidualV_t,
mResidualSeqUsedK,
mSinkSeqUsedK,
tma_atom_K_bf16,
tma_atom_V_bf16,
sK_bf16_layout,
sV_bf16_layout,
tiled_mma_pv_bf16,
mResidualQ_t,
tma_atom_Q_bf16,
tP_bf16_layout,
mResidualBlockIds,
mOutIndices,
).launch(
grid=grid_dim,
block=[self.threads_per_cta, 1, 1],
cluster=self.cluster_shape_mnk,
smem=self.shared_storage.size_in_bytes(),
stream=stream,
min_blocks_per_mp=1,
)
# GPU device kernel
@cute.kernel
def kernel(
self,
mQ: cute.Tensor, # (s_q, d, h, b) or (total_q, d, h) if there is cu_seqlens_q
mQSecond: cute.Tensor,
mK: cute.Tensor, # (s_k, d, h_k, b_k) or (total_k, d, h_k) if there is cu_seqlens_k or (page_size, d, h_k, num_pages) if there is page_table
mV: cute.Tensor, # (d, s_k, h_k, b_k) or (d, total_k, h_k) if there is cu_seqlens_k or (d, page_size, h_k, num_pages) if there is page_table
mO: cute.Tensor,
mLSE: Optional[cute.Tensor],
mCuSeqlensQ: Optional[cute.Tensor],
mCuSeqlensK: Optional[cute.Tensor],
mSeqUsedQ: Optional[cute.Tensor],
mSeqUsedK: Optional[cute.Tensor],
mPageTable: Optional[cute.Tensor],
tma_atom_Q: cute.CopyAtom,
tma_atom_Q_second: cute.CopyAtom,
tma_atom_K: Optional[cute.CopyAtom],
tma_atom_V: Optional[cute.CopyAtom],
tma_atom_O: Optional[cute.CopyAtom],
softmax_scale_log2: Float32,
softmax_scale: Float32 | None,
window_size_left: Optional[Int32],
window_size_right: Optional[Int32],
learnable_sink: Optional[cute.Tensor],
blocksparse_tensors: Optional[BlockSparseTensors],
sQ_layout: cute.ComposedLayout,
sK_layout: cute.ComposedLayout,
tP_layout: cute.ComposedLayout,
sV_layout: cute.ComposedLayout,
sO_layout: cute.ComposedLayout,
gmem_tiled_copy_O: Optional[cute.TiledCopy],
tiled_mma_qk: cute.TiledMma,
tiled_mma_pv: cute.TiledMma,
tile_sched_params: ParamsBase,
num_splits: Int32,
aux_tensors: Optional[list] = None,
fastdiv_mods=(None, None),
tma_atom_sfq: Optional[cute.CopyAtom] = None,
tma_tensor_sfq: Optional[cute.Tensor] = None,
tma_atom_sfq_second: Optional[cute.CopyAtom] = None,
tma_tensor_sfq_second: Optional[cute.Tensor] = None,
tma_atom_sfk: Optional[cute.CopyAtom] = None,
tma_tensor_sfk: Optional[cute.Tensor] = None,
tma_atom_sfv: Optional[cute.CopyAtom] = None,
tma_tensor_sfv: Optional[cute.Tensor] = None,
sfq_smem_layout_staged: Optional[cute.Layout] = None,
sfk_smem_layout_staged: Optional[cute.Layout] = None,
sfp_smem_layout_staged: Optional[cute.Layout] = None,
sfv_smem_layout_staged: Optional[cute.Layout] = None,
sQ_bf16_layout: Optional[cute.ComposedLayout] = None,
tiled_mma_qk_bf16: Optional[cute.TiledMma] = None,
mResidualK: Optional[cute.Tensor] = None,
mResidualV: Optional[cute.Tensor] = None,
mResidualSeqUsedK: Optional[cute.Tensor] = None,
mSinkSeqUsedK: Optional[cute.Tensor] = None,
tma_atom_K_bf16: Optional[cute.CopyAtom] = None,
tma_atom_V_bf16: Optional[cute.CopyAtom] = None,
sK_bf16_layout: Optional[cute.ComposedLayout] = None,
sV_bf16_layout: Optional[cute.ComposedLayout] = None,
tiled_mma_pv_bf16: Optional[cute.TiledMma] = None,
mResidualQ: Optional[cute.Tensor] = None,
tma_atom_Q_bf16: Optional[cute.CopyAtom] = None,
tP_bf16_layout: Optional[cute.ComposedLayout] = None,
mResidualBlockIds: Optional[cute.Tensor] = None,
mOutIndices: Optional[cute.Tensor] = None,
):
"""The device kernel implementation of the Fused Multi-Head Attention.
This kernel coordinates multiple specialized warps to perform different phases of the FMHA computation:
1. Load warp: Loads Q, K, V data from global memory to shared memory using TMA
2. MMA warp: Performs matrix multiplications (Q*K^T and P*V)
3. Softmax warps: Compute softmax normalization on attention scores
4. Correction warps: Apply adjustments to intermediate results
5. Epilogue warp: Handles final output transformation and storage
The kernel implements a complex pipeline with overlapping computation and memory operations,
using tensor memory access (TMA) for efficient data loading, warp specialization for different
computation phases, and optional attention masking.
"""
warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx())
iket.range_push("pro_init")
# Prefetch tma descriptor
if warp_idx == 0:
cpasync.prefetch_descriptor(tma_atom_Q)
if const_expr(tma_atom_K is not None):
cpasync.prefetch_descriptor(tma_atom_K)
if const_expr(tma_atom_V is not None):
cpasync.prefetch_descriptor(tma_atom_V)
if const_expr(tma_atom_O is not None):
cpasync.prefetch_descriptor(tma_atom_O)
if const_expr(tma_atom_sfq is not None):
cpasync.prefetch_descriptor(tma_atom_sfq)
if const_expr(tma_atom_sfk is not None):
cpasync.prefetch_descriptor(tma_atom_sfk)
if const_expr(tma_atom_sfv is not None):
cpasync.prefetch_descriptor(tma_atom_sfv)
# Alloc
smem = cutlass.utils.SmemAllocator()
storage = smem.allocate(self.shared_storage)
mbar_ptr = storage.mbar_ptr.data_ptr()
# Use the first N warps to initialize barriers
if warp_idx == 1:
# Init "full" barrier with number of producers, "empty" barrier with number of consumers
for i in cutlass.range_constexpr(self.q_stage):
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_load_q_full_offset + i, 1
)
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_load_q_empty_offset + i, len([self.mma_warp_id])
)
if warp_idx == 2:
for i in cutlass.range_constexpr(self.q_stage):
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_softmax_corr_empty_offset + i,
cute.arch.WARP_SIZE * len(self.correction_warp_ids),
)
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_softmax_corr_full_offset + i,
cute.arch.WARP_SIZE
* len(self.softmax0_warp_ids)
* self.softmax_row_groups,
)
if warp_idx == 3:
if const_expr(self.s0_s1_barrier):
for i in cutlass.range_constexpr(8):
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_s0_s1_sequence_offset + i, cute.arch.WARP_SIZE
)
if const_expr(not self.use_correction_warps_for_epi) and warp_idx == 4:
for i in cutlass.range_constexpr(self.q_stage):
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_corr_epi_full_offset + i,
cute.arch.WARP_SIZE * len(self.correction_warp_ids),
)
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_corr_epi_empty_offset + i,
cute.arch.WARP_SIZE * len(self.epilogue_warp_ids),
)
if warp_idx == 5:
for i in cutlass.range_constexpr(self.q_stage):
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_P_full_O_rescaled_offset + i,
cute.arch.WARP_SIZE
* (
len(self.softmax0_warp_ids) * self.softmax_row_groups
+ len(self.correction_warp_ids)
),
)
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_S_full_offset + i, len([self.mma_warp_id])
)
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_O_full_offset + i, len([self.mma_warp_id])
)
if warp_idx == 6:
for i in cutlass.range_constexpr(self.q_stage):
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_P_full_2_offset + i,
cute.arch.WARP_SIZE * len(self.softmax0_warp_ids),
)
if warp_idx == 7:
# Every softmax warp in the CTA arrives here once it leaves its
# role, whether or not it was handed a row group, and so does every
# correction warp.
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_tmem_dealloc_offset,
cute.arch.WARP_SIZE
* (
sum(len(ids) for ids in self.softmax_warp_ids)
+ len(self.correction_warp_ids)
),
)
if warp_idx == 8:
for i in cutlass.range_constexpr(self.q_stage):
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_sfqk_load_offset + i,
cute.arch.WARP_SIZE
* len(self.softmax0_warp_ids)
* self.softmax_row_groups,
)
if const_expr(self.fused_residual_first_block):
for i in cutlass.range_constexpr(3):
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_residual_kv_full_offset + i, 1
)
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_residual_kv_empty_offset + i,
len([self.mma_warp_id]),
)
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_bf16_S_full_offset, 1
)
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_bf16_P_full_offset,
cute.arch.WARP_SIZE
* len(self.softmax0_warp_ids)
* self.softmax_row_groups,
)
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_bf16_P_full_2_offset,
cute.arch.WARP_SIZE
* len(self.softmax0_warp_ids)
* self.softmax_row_groups,
)
if const_expr(self.fused_sink_block):
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_bf16_PV_done_offset, 1
)
if warp_idx == 9:
for i in cutlass.range_constexpr(self.q_stage):
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_q_fp4_ready_offset + i,
1, # one arrive: the load warp (after cast)
)
cute.arch.mbarrier_init(
mbar_ptr + self.mbar_P_consumed_offset + i, len([self.mma_warp_id])
)
iket.range_pop()
# Relying on pipeline_kv constructor to call mbarrier_init_fence and sync
iket.range_push("pro_sync")
pipeline_kv = self.make_and_init_load_kv_pipeline(mbar_ptr + self.mbar_load_kv_full_offset)
iket.range_pop()
iket.range_push("pro_setup")
# Generate smem tensor Q/K/V/O
# (MMA, MMA_Q, MMA_D, PIPE)
sQ = storage.sQ.get_tensor(sQ_layout.outer, swizzle=sQ_layout.inner)
sQ_terms = tuple(
cute.make_tensor(
sQ.iterator + term * cute.cosize(sQ_layout),
sQ.layout,
)
for term in range(self.query_terms)
)
sQ_bf16 = None
sQ_uint8 = None
# Byte view for moving complete FP4 rows: a thread owns whole bytes,
# never a single nibble. Stacking and replication share this layout.
if const_expr(self.q_replicate > 1 or self.stacked_query_terms):
uint8_swizzle = cute.make_swizzle(2, 4, 3)
# FP4 atom_K (per kH) = head_dim_padded//2 elements; Uint8
# atom_K_byte = atom_K_FP4 // 2 = head_dim_padded // 4.
atom_K_b_size = self.head_dim_padded // 4
# M-stride in Uint8 bytes = head_dim_padded // 2 (one byte per 2
# FP4 elements; one row spans head_dim_padded FP4 = head_dim/2 byte).
row_bytes = self.head_dim_padded // 2
kH_stride_bytes = atom_K_b_size # one kH atom = atom_K_b_size bytes
stage_stride_bytes = self.m_block_size * row_bytes
uint8_outer = cute.make_layout(
((self.m_block_size, atom_K_b_size), 1, 2, self.q_stage),
stride=(((row_bytes, 1)), 0, kH_stride_bytes, stage_stride_bytes),
)
sQ_uint8 = cute.make_tensor(
cute.recast_ptr(sQ.iterator, uint8_swizzle, cutlass.Uint8),
uint8_outer,
)
# (MMA, MMA_K, MMA_D, PIPE)
# K and V share physical SMEM:
# - same dtype: V aliases K (V uses K's base pointer)
# - FP4 K + BF16 V: K aliases V (K uses V's base pointer, stride scaled by dtype ratio)
kv_stages = storage.sK.stages if const_expr(self.bf16_split) else storage.sK
bf16_tiles = (
storage.sK.bf16
if const_expr(self.bf16_split)
else (storage.sO.bf16 if const_expr(self.overlap_sO_bf16) else None)
)
if const_expr(self.v_dtype == self.k_dtype):
sK = kv_stages.get_tensor(sK_layout.outer, swizzle=sK_layout.inner)
sV = cute.make_tensor(cute.recast_ptr(sK.iterator, sV_layout.inner), sV_layout.outer)
elif const_expr(self.k_dtype.width < self.v_dtype.width):
# K aliases V's buffer — K is smaller, fits inside V's stage.
# K's stage stride must match V's stage stride in bytes so they align.
sV = storage.sV.get_tensor(sV_layout.outer, swizzle=sV_layout.inner)
stride_sV = const_expr(max(sV_layout.outer.stride[-1], 0))
stride_sK_aligned = const_expr(stride_sV * self.v_dtype.width // self.k_dtype.width)
sK_outer_aligned = cute.make_layout(
sK_layout.outer.shape,
stride=(*sK_layout.outer.stride[:-1], stride_sK_aligned),
)
sK = storage.sV.get_tensor(sK_outer_aligned, swizzle=sK_layout.inner, dtype=self.k_dtype)
else:
sK = storage.sK.get_tensor(sK_layout.outer, swizzle=sK_layout.inner)
sV = storage.sV.get_tensor(sV_layout.outer, swizzle=sV_layout.inner)
if const_expr(self.overlap_sO_bf16):
sO = storage.sO.o.get_tensor(sO_layout.outer, swizzle=sO_layout.inner)
elif const_expr(not self.overlap_sO_sQ):
sO = storage.sO.get_tensor(sO_layout.outer, swizzle=sO_layout.inner)
else:
sO = cute.make_tensor(cute.recast_ptr(sQ.iterator, sO_layout.inner, self.o_dtype), sO_layout.outer)
sScale = storage.sScale.get_tensor(
cute.make_layout(max(self.q_stage * self.m_block_size * 2, self.m_block_size * 4))
)
# Get scale factor shared memory tensors if they exist
sSFV = None
sSFP = None
sSFQ = None
sSFK = None
if const_expr(self.quant_qk):
if const_expr(self.overlap_sO_sQ):
sSFQ = storage.sQ.get_tensor(
sfq_smem_layout_staged, dtype=self.sf_dtype
)
q_terms_bytes = const_expr(
self.query_terms * cute.cosize(sQ_layout) * self.q_dtype.width // 8
)
sSFQ = cute.make_tensor(sSFQ.iterator + q_terms_bytes, sSFQ.layout)
else:
sSFQ = storage.sSFQ.get_tensor(sfq_smem_layout_staged)
sSFQ_terms = tuple(
cute.make_tensor(
sSFQ.iterator
+ term * cute.cosize(sfq_smem_layout_staged),
sSFQ.layout,
)
for term in range(self.query_terms)
)
sSFK = storage.sSFK.get_tensor(sfk_smem_layout_staged)
else:
sSFQ_terms = (None,)
if const_expr(sfp_smem_layout_staged is not None):
sSFP = storage.sSFP.get_tensor(sfp_smem_layout_staged)
if const_expr(sfv_smem_layout_staged is not None):
sSFV = storage.sSFV.get_tensor(sfv_smem_layout_staged)
sK_bf16 = None
sV_bf16 = None
sP_bf16 = None
sP_bf16_mk = None
if const_expr(self.fused_residual_first_block):
sK_bf16 = bf16_tiles.sK_bf16.get_tensor(sK_bf16_layout.outer, swizzle=sK_bf16_layout.inner)
sV_bf16 = bf16_tiles.sV_bf16.get_tensor(sV_bf16_layout.outer, swizzle=sV_bf16_layout.inner)
sQ_bf16 = bf16_tiles.sQ_bf16.get_tensor(sQ_bf16_layout.outer, swizzle=sQ_bf16_layout.inner)
if const_expr(self.transpose_s):
sP_bf16 = bf16_tiles.sQ_bf16.get_tensor(
tP_bf16_layout.outer, swizzle=tP_bf16_layout.inner
)
# The softmax writes one query row at a time down its own kv
# column, so it wants the plain (query row, kv) view of the
# operand layout rather than its atom decomposition.
sP_bf16_mk = cute.composition(
sP_bf16[None, None, None, 0],
cute.make_layout((self.m_block_size, self.n_block_size)),
)
thr_mma_qk = tiled_mma_qk.get_slice(0) # default 1SM
thr_mma_pv = tiled_mma_pv.get_slice(0) # default 1SM
qk_acc_shape = thr_mma_qk.partition_shape_C(self.mma_tiler_qk[:2])
tStS_fake = thr_mma_qk.make_fragment_C(qk_acc_shape)
tmem_ptr = cute.make_ptr(Float32, 0, mem_space=cute.AddressSpace.tmem, assumed_align=16)
tStS = cute.make_tensor(tmem_ptr, tStS_fake.layout)
pv_acc_shape = thr_mma_pv.partition_shape_C(self.mma_tiler_pv[:2])
tOtO = thr_mma_pv.make_fragment_C(pv_acc_shape)
tStSs = tuple(
cute.make_tensor(tStS.iterator + self.tmem_s_offset[stage], tStS.layout)
for stage in range(2)
)
tOtOs = tuple(
cute.make_tensor(tOtO.iterator + self.tmem_o_offset[stage], tOtO.layout)
for stage in range(self.q_stage)
)
# P in shared memory: the MMA A-operand tile, plus a byte view for the
# softmax warps, which own single nibbles and can only store whole bytes.
sP = None
sP_uint8 = None
if const_expr(self.transpose_s):
sP = storage.sP.get_tensor(tP_layout.outer, swizzle=tP_layout.inner)
sP_uint8 = cute.make_tensor(
cute.recast_ptr(sP.iterator, cute.make_swizzle(2, 4, 3), cutlass.Uint8),
cute.make_layout(
(
(self.m_block_size, self.n_block_size // 4),
1,
2,
self.acc_stage,
),
stride=(
(self.n_block_size // 2, 1),
0,
self.n_block_size // 4,
self.m_block_size * self.n_block_size // 2,
),
),
)
sSoftmaxRed = storage.sSoftmaxRed.get_tensor(
cute.make_layout(self.softmax_red_slots)
)
else:
sSoftmaxRed = None
if const_expr(self.transpose_s):
tOrP = thr_mma_pv.make_fragment_A(sP)[None, None, None, 0]
tOrPs = [tOrP, tOrP]
else:
tP = cute.make_tensor(tStS.iterator, tP_layout.outer)
tOrP = thr_mma_pv.make_fragment_A(tP)[None, None, None, 0]
tOrPs = [
cute.make_tensor(
tOrP.iterator
+ self.qk_acc_dtype.width // tOrP._dtype.width * self.tmem_p_offset[stage],
tOrP.layout,
)
for stage in range(2)
]
tOrPs_bf16 = (None, None)
if const_expr(self.fused_residual_first_block):
thr_mma_pv_bf16 = tiled_mma_pv_bf16.get_slice(0)
if const_expr(self.transpose_s):
tOrP_bf16 = thr_mma_pv_bf16.make_fragment_A(sP_bf16)[None, None, None, 0]
if const_expr(self.fused_sink_block):
# The sink occupies one 16-token BF16 MMA-K block. Keep the
# original operand layout and expose only that first block.
tOrP_sink_bf16 = cute.make_tensor(
tOrP_bf16.iterator,
cute.make_layout(
(
tOrP_bf16.layout.shape[0],
tOrP_bf16.layout.shape[1],
(1, 1),
),
stride=tOrP_bf16.layout.stride,
),
)
tOrPs_bf16 = (tOrP_bf16, tOrP_sink_bf16)
else:
tOrPs_bf16 = (tOrP_bf16, tOrP_bf16)
else:
tP_bf16 = cute.make_tensor(tStS.iterator, tP_bf16_layout.outer)
tOrP_bf16 = thr_mma_pv_bf16.make_fragment_A(tP_bf16)[None, None, None, 0]
tOrPs_bf16 = tuple(
cute.make_tensor(
tOrP_bf16.iterator
+ self.qk_acc_dtype.width // tOrP_bf16._dtype.width
* self.tmem_p_bf16_offset[stage],
tOrP_bf16.layout,
)
for stage in range(2)
)
# Setup scale factor TMEM tensors and S2T copy operations
# Use the TMEM region immediately following the accumulator (O tensor)
# sf_tmem_ptr = cute.make_ptr(self.sf_dtype, 0, mem_space=cute.AddressSpace.tmem, assumed_align=16)
align = 16 # Required for tcgen05.cp 4x32dp128bit.
# find_tmem_tensor_col_offset returns u32 columns; convert to sf_dtype elements
sf_dtype_per_u32 = 32 // self.sf_dtype.width
tCtSFQ_terms = [[None] * self.q_stage]
tCtSFKs = [None] * self.q_stage
if const_expr(self.quant_qk):
if const_expr(self.q_stage == 2):
sfq_tmem_col_offsets = [self.tmem_s_offset[self.q_stage - 1 - stage] for stage in range(self.q_stage)]
else:
sfq_tmem_col_offsets = [self.tmem_s_offset[1]] # the unused stage-1 S region
sfq_tmem_ptrs = [cute.recast_ptr(
cute.make_ptr(Float32, sfq_tmem_col_offsets[stage], # shuffle to minimize dependency
mem_space=cute.AddressSpace.tmem, assumed_align=align),
dtype=self.sf_dtype) for stage in range(self.q_stage)
]
# (MMA, MMA_M, MMA_K) — Q is the B operand once S is transposed.
make_tmem_layout_sfq = (
blockscaled_utils.make_tmem_layout_sfb
if const_expr(self.transpose_s)
else blockscaled_utils.make_tmem_layout_sfa
)
make_tmem_layout_sfk = (
blockscaled_utils.make_tmem_layout_sfa
if const_expr(self.transpose_s)
else blockscaled_utils.make_tmem_layout_sfb
)
tCtSFQ_layout = make_tmem_layout_sfq(
tiled_mma_qk,
self.mma_tiler_qk,
self.sf_vec_size,
cute.slice_(sfq_smem_layout_staged, (None, None, None, 0)),
)
tCtSFQ_probe = cute.make_tensor(
sfq_tmem_ptrs[0], tCtSFQ_layout
)
sfq_col_offset = tcgen05.find_tmem_tensor_col_offset(
tCtSFQ_probe
)
sfq_offset = math.ceil(sfq_col_offset * sf_dtype_per_u32 / align) * align
tCtSFQ_terms = [
[
cute.make_tensor(
sfq_tmem_ptrs[stage] + term * sfq_offset,
tCtSFQ_layout,
)
for stage in range(self.q_stage)
]
for term in range(self.query_terms)
]
# K scale factors follow every independently scaled query term.
sfk_tmem_ptrs = [
sfq_tmem_ptrs[stage] + self.query_terms * sfq_offset
for stage in range(self.q_stage)
]
# (MMA, MMA_N, MMA_K)
tCtSFK_layout = make_tmem_layout_sfk(
tiled_mma_qk,
self.mma_tiler_qk,
self.sf_vec_size,
cute.slice_(sfk_smem_layout_staged, (None, None, None, 0)),
)
tCtSFKs = [cute.make_tensor(sfk_tmem_ptrs[stage], tCtSFK_layout) for stage in range(self.q_stage)]
# Setup SFP and SFV TMEM tensors. These normally sit in the columns S
# was read out of, which costs the next QK a wait on this block's PV.
# Where tmem_sfpv_offset says there is room, they get their own columns
# instead and the two GEMMs stop contending.
tCtSFPs = [None] * self.q_stage
tCtSFVs = [None] * self.q_stage
if const_expr(self.quant_pv):
if const_expr(self.tmem_sfpv_offset is not None):
sfpv_col_offsets = [self.tmem_sfpv_offset] * self.q_stage
else:
sfpv_col_offsets = [self.tmem_s_offset[stage] for stage in range(self.q_stage)]
sfp_tmem_ptrs = [cute.recast_ptr(
cute.make_ptr(Float32, sfpv_col_offsets[stage],
mem_space=cute.AddressSpace.tmem, assumed_align=align),
dtype=self.sf_dtype) for stage in range(self.q_stage)]
# A narrow PV swaps the operands, so P's scales are the B operand's
# and V's the A operand's, as K's and Q's are on a transposed QK.
# The shared-memory layouts are the same either way round for a
# 128 x 128 tile; only the tensor-memory placement follows the role.
make_tmem_layout_sfp = (
blockscaled_utils.make_tmem_layout_sfb
if const_expr(self.narrow_pv)
else blockscaled_utils.make_tmem_layout_sfa
)
make_tmem_layout_sfv = (
blockscaled_utils.make_tmem_layout_sfa
if const_expr(self.narrow_pv)
else blockscaled_utils.make_tmem_layout_sfb
)
# (MMA, MMA_M, MMA_K)
tCtSFP_layout = make_tmem_layout_sfp(
tiled_mma_pv,
self.mma_tiler_pv,
self.sf_vec_size,
cute.slice_(sfp_smem_layout_staged, (None, None, None, 0)),
)
tCtSFPs = [cute.make_tensor(sfp_tmem_ptrs[stage], tCtSFP_layout) for stage in range(self.q_stage)]
# Make SFV tmem tensor
sfp_offset = math.ceil(tcgen05.find_tmem_tensor_col_offset(tCtSFPs[0]) * sf_dtype_per_u32 / align) * align
sfv_tmem_ptrs = [sfp_tmem_ptrs[stage] + sfp_offset for stage in range(self.q_stage)]
# (MMA, MMA_N, MMA_K) for P*V operation (V is the B matrix)
tCtSFV_layout = make_tmem_layout_sfv(
tiled_mma_pv,
self.mma_tiler_pv,
self.sf_vec_size,
cute.slice_(sfv_smem_layout_staged, (None, None, None, 0)),
)
tCtSFVs = [cute.make_tensor(sfv_tmem_ptrs[stage], tCtSFV_layout) for stage in range(self.q_stage)]
if const_expr(self.tmem_sfpv_offset is not None):
sfpv_cols = math.ceil(
(sfp_offset + tcgen05.find_tmem_tensor_col_offset(tCtSFVs[0]) * sf_dtype_per_u32)
/ sf_dtype_per_u32
)
assert self.tmem_sfpv_offset + sfpv_cols <= self.tmem_alloc_cols, (
f"SFP/SFV need {sfpv_cols} columns at {self.tmem_sfpv_offset}, "
f"past the {self.tmem_alloc_cols}-column capacity"
)
block_info = BlockInfo(
# This is cta_tiler, not mma_tiler_qk, since we move by block by (2 * mma_tiler[0], mma_tiler[1])
self.cta_tiler[0],
self.cta_tiler[1],
self.is_causal,
self.is_local,
self.is_split_kv,
window_size_left,
window_size_right,
qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1,
)
SeqlenInfoCls = partial(
SeqlenInfoQK.create,
seqlen_q_static=(
1
if const_expr(self.seqlen_q_static_one)
else mQ.shape[0] if const_expr(not self.pack_gqa) else mQ.shape[0][1]
),
seqlen_k_static=mK.shape[0]
if const_expr(mPageTable is None)
else mK.shape[0] * mPageTable.shape[1],
mCuSeqlensQ=mCuSeqlensQ,
mCuSeqlensK=mCuSeqlensK,
mSeqUsedQ=mSeqUsedQ,
mSeqUsedK=mSeqUsedK,
)
AttentionMaskCls = partial(
AttentionMask,
self.m_block_size,
self.n_block_size,
window_size_left=window_size_left,
window_size_right=window_size_right,
qhead_per_kvhead_packgqa=self.qhead_per_kvhead if const_expr(self.pack_gqa) else 1,
)
TileSchedulerCls = partial(self.tile_scheduler_cls.create, tile_sched_params)
is_bf16_cta = False
if const_expr(self.bf16_split):
is_bf16_cta = self.is_bf16_split(
TileSchedulerCls().initial_work_tile_info().tile_idx[3], num_splits
)
iket.range_pop()
# ///////////////////////////////////////////////////////////////////////////////
# EMPTY
# ///////////////////////////////////////////////////////////////////////////////
if const_expr(len(self.empty_warp_ids) > 0):
if warp_idx == self.empty_warp_ids[0]:
cute.arch.warpgroup_reg_dealloc(self.num_regs_empty)
if const_expr(len(self.empty_warp_ids) > 1):
if warp_idx == self.empty_warp_ids[1]:
cute.arch.warpgroup_reg_dealloc(self.num_regs_empty)
assert len(self.empty_warp_ids) <= 2
# ///////////////////////////////////////////////////////////////////////////////
# LOAD
# ///////////////////////////////////////////////////////////////////////////////
if warp_idx >= self.load_warp_ids[0] and warp_idx <= self.load_warp_ids[-1]:
role_lifetime = iket.range_start("load_life")
cute.arch.warpgroup_reg_dealloc(self.num_regs_other)
iket.range_push("load_main")
self.run_split_role(is_bf16_cta, partial(
self.load,
thr_mma_qk,
thr_mma_pv,
mQ,
mQSecond,
mK,
mV,
sQ_terms,
sK,
sV,
mPageTable,
tma_atom_Q,
tma_atom_Q_second,
tma_atom_K,
tma_atom_V,
tma_atom_sfq,
tma_tensor_sfq,
tma_atom_sfq_second,
tma_tensor_sfq_second,
tma_atom_sfk,
tma_tensor_sfk,
sSFQ_terms,
sSFK,
pipeline_kv,
mbar_ptr,
block_info,
num_splits,
SeqlenInfoCls,
TileSchedulerCls,
blocksparse_tensors,
tma_atom_sfv,
tma_tensor_sfv,
sfv_smem_layout_staged,
sSFV,
sQ_bf16=sQ_bf16,
sQ_uint8=sQ_uint8,
mResidualK_t=mResidualK,
mResidualV_t=mResidualV,
mResidualQ_t=mResidualQ,
tma_atom_K_bf16=tma_atom_K_bf16,
tma_atom_V_bf16=tma_atom_V_bf16,
tma_atom_Q_bf16=tma_atom_Q_bf16,
sK_bf16=sK_bf16,
sV_bf16=sV_bf16,
tiled_mma_qk_bf16=tiled_mma_qk_bf16,
tiled_mma_pv_bf16=tiled_mma_pv_bf16,
mResidualBlockIds=mResidualBlockIds,
))
iket.range_pop()
iket.range_end(role_lifetime)
# ///////////////////////////////////////////////////////////////////////////////
# MMA
# ///////////////////////////////////////////////////////////////////////////////
if warp_idx == self.mma_warp_id:
role_lifetime = iket.range_start("mma_life")
# if warp_idx == self.mma_warp_id or warp_idx == self.empty_warp_ids:
cute.arch.warpgroup_reg_dealloc(self.num_regs_other)
# Alloc tmem buffer
tmem_alloc_cols = Int32(self.tmem_alloc_cols)
if warp_idx == self.mma_warp_id:
cute.arch.alloc_tmem(tmem_alloc_cols, storage.tmem_holding_buf)
cute.arch.sync_warp()
self.run_split_role(is_bf16_cta, partial(
self.mma,
tiled_mma_qk,
tiled_mma_pv,
sQ_terms,
sK,
sV,
tStSs,
tOtOs,
tOrPs,
pipeline_kv,
mbar_ptr,
block_info,
num_splits,
SeqlenInfoCls,
TileSchedulerCls,
blocksparse_tensors,
sSFQ_terms,
sSFK,
sSFV,
sSFP,
tCtSFQ_terms,
tCtSFKs,
tCtSFPs,
tCtSFVs,
tiled_mma_qk_bf16=tiled_mma_qk_bf16,
tiled_mma_pv_bf16=tiled_mma_pv_bf16,
sK_bf16=sK_bf16,
sV_bf16=sV_bf16,
sQ_bf16=sQ_bf16,
mResidualSeqUsedK=mResidualSeqUsedK,
tOrPs_bf16=tOrPs_bf16,
sP=sP,
sP_bf16=sP_bf16,
))
# if warp_idx == self.mma_warp_id:
# dealloc tmem buffer
cute.arch.relinquish_tmem_alloc_permit()
cute.arch.mbarrier_wait(mbar_ptr + self.mbar_tmem_dealloc_offset, 0)
tmem_alloc_cols = Int32(self.tmem_alloc_cols)
# Retrieving tmem ptr and make acc
tmem_ptr = cute.arch.retrieve_tmem_ptr(
Float32,
alignment=16,
ptr_to_buffer_holding_addr=storage.tmem_holding_buf,
)
cute.arch.dealloc_tmem(tmem_ptr, tmem_alloc_cols)
iket.range_end(role_lifetime)
# ///////////////////////////////////////////////////////////////////////////////
# Epilogue
# ///////////////////////////////////////////////////////////////////////////////
if const_expr(not self.use_correction_warps_for_epi):
if warp_idx >= self.epilogue_warp_ids[0] and warp_idx <= self.epilogue_warp_ids[-1]:
role_lifetime = iket.range_start("epi_life")
cute.arch.warpgroup_reg_dealloc(self.num_regs_other)
iket.range_push("epi_main")
self.epilogue_s2g(
mO,
sO,
gmem_tiled_copy_O,
tma_atom_O,
mbar_ptr,
block_info,
num_splits,
SeqlenInfoCls,
TileSchedulerCls,
mOutIndices=mOutIndices,
)
iket.range_pop()
iket.range_end(role_lifetime)
# ///////////////////////////////////////////////////////////////////////////////
# Softmax
# ///////////////////////////////////////////////////////////////////////////////
if warp_idx < self.correction_warp_ids[0]:
role_lifetime = iket.range_start("softmax_life")
# increase register after decreasing
cute.arch.warpgroup_reg_alloc(self.num_regs_softmax)
softmax_loop = partial(
self.softmax_loop,
softmax_scale_log2=softmax_scale_log2,
softmax_scale=softmax_scale,
thr_mma_qk=thr_mma_qk,
sScale=sScale,
mLSE=mLSE,
learnable_sink=learnable_sink,
mbar_ptr=mbar_ptr,
block_info=block_info,
num_splits=num_splits,
SeqlenInfoCls=SeqlenInfoCls,
AttentionMaskCls=AttentionMaskCls,
TileSchedulerCls=TileSchedulerCls,
aux_tensors=aux_tensors,
fastdiv_mods=fastdiv_mods,
blocksparse_tensors=blocksparse_tensors,
sSFP=sSFP,
mResidualSeqUsedK=mResidualSeqUsedK,
mSinkSeqUsedK=mSinkSeqUsedK,
sP_uint8=sP_uint8,
sSoftmaxRed=sSoftmaxRed,
sP_bf16=sP_bf16,
sP_bf16_mk=sP_bf16_mk,
)
if const_expr(not self.s0_s1_barrier):
stage = Int32(0 if warp_idx < self.softmax1_warp_ids[0] else 1)
# Compute tCtSFP/tStSi unconditionally so they have stable types.
# When q_stage=1 we always read [0]; when q_stage=2 we pick by stage.
if const_expr(self.quant_pv):
if const_expr(self.q_stage == 2):
tCtSFP = tCtSFPs[0] if stage == 0 else tCtSFPs[1]
else:
tCtSFP = tCtSFPs[0]
else:
tCtSFP = None
if const_expr(self.q_stage == 2):
s_off = self.tmem_s_offset[0] if stage == 0 else self.tmem_s_offset[1]
else:
s_off = self.tmem_s_offset[0]
tStSi_local = cute.make_tensor(tStS.iterator + s_off, tStS.layout)
if const_expr(self.softmax_row_groups > 1):
# Every group drives the same single Q stage over its own
# query rows; specialising on the group keeps its row
# offset, its scratch base and its barrier id compile-time
# constants.
softmax_warpgroup = warp_idx // len(self.softmax0_warp_ids)
for group in cutlass.range_constexpr(self.softmax_row_groups):
if softmax_warpgroup == group:
self.run_split_role(is_bf16_cta, partial(
softmax_loop,
stage=0,
tStSi=tStSi_local,
tCtSFP=tCtSFP,
row_group=group,
))
elif const_expr(self.q_stage == 2) or stage == 0:
self.run_split_role(is_bf16_cta, partial(
softmax_loop,
stage=stage,
tStSi=tStSi_local,
tCtSFP=tCtSFP, # need to copy P sf to tmem after exp
))
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset)
else:
# If there's s0_s1_barrier, it's faster to have 2 WGs having different code
if warp_idx < self.softmax1_warp_ids[0]:
tStSi = cute.make_tensor(tStS.iterator + self.tmem_s_offset[0], tStS.layout)
softmax_loop(stage=0, tStSi=tStSi, tCtSFP=tCtSFPs[0])
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset)
if warp_idx < self.correction_warp_ids[0] and warp_idx >= self.softmax1_warp_ids[0]:
tStSi = cute.make_tensor(tStS.iterator + self.tmem_s_offset[1], tStS.layout)
softmax_loop(stage=1, tStSi=tStSi, tCtSFP=tCtSFPs[1])
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset)
iket.range_end(role_lifetime)
# ///////////////////////////////////////////////////////////////////////////////
# Correction
# ///////////////////////////////////////////////////////////////////////////////
if warp_idx >= self.correction_warp_ids[0] and warp_idx < self.mma_warp_id:
role_lifetime = iket.range_start("corr_life")
cute.arch.warpgroup_reg_dealloc(self.num_regs_correction)
iket.range_push("corr_main")
self.run_split_role(is_bf16_cta, partial(
self.correction_loop,
thr_mma_qk,
thr_mma_pv,
tStS,
tOtOs,
sScale,
mO,
mLSE,
sO,
learnable_sink,
gmem_tiled_copy_O,
tma_atom_O,
mbar_ptr,
softmax_scale_log2,
block_info,
num_splits,
SeqlenInfoCls,
TileSchedulerCls,
blocksparse_tensors,
))
iket.range_pop()
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_tmem_dealloc_offset)
iket.range_end(role_lifetime)
return
@cute.jit
def load(
self,
thr_mma_qk: cute.ThrMma,
thr_mma_pv: cute.ThrMma,
mQ: cute.Tensor,
mQSecond: cute.Tensor,
mK: cute.Tensor,
mV: cute.Tensor,
sQ_terms: Tuple[cute.Tensor, ...],
sK: cute.Tensor,
sV: cute.Tensor,
mPageTable: Optional[cute.Tensor],
tma_atom_Q: cute.CopyAtom,
tma_atom_Q_second: cute.CopyAtom,
tma_atom_K: Optional[cute.CopyAtom],
tma_atom_V: Optional[cute.CopyAtom],
tma_atom_sfq: Optional[cute.CopyAtom],
tma_tensor_sfq: Optional[cute.Tensor],
tma_atom_sfq_second: Optional[cute.CopyAtom],
tma_tensor_sfq_second: Optional[cute.Tensor],
tma_atom_sfk: Optional[cute.CopyAtom],
tma_tensor_sfk: Optional[cute.Tensor],
sSFQ_terms: Tuple[Optional[cute.Tensor], ...],
sSFK: Optional[cute.Tensor],
pipeline_kv: cutlass.pipeline.PipelineAsync,
mbar_ptr: cute.Pointer,
block_info: BlockInfo,
num_splits: Int32,
SeqlenInfoCls: Callable,
TileSchedulerCls: Callable,
blocksparse_tensors: Optional[BlockSparseTensors],
tma_atom_sfv: Optional[cute.CopyAtom] = None,
tma_tensor_sfv: Optional[cute.Tensor] = None,
sfv_smem_layout_staged: Optional[cute.ComposedLayout] = None,
sSFV: Optional[cute.Tensor] = None,
sQ_bf16: Optional[cute.Tensor] = None,
sQ_uint8: Optional[cute.Tensor] = None,
mResidualK_t: Optional[cute.Tensor] = None,
mResidualV_t: Optional[cute.Tensor] = None,
mResidualQ_t: Optional[cute.Tensor] = None,
tma_atom_K_bf16: Optional[cute.CopyAtom] = None,
tma_atom_V_bf16: Optional[cute.CopyAtom] = None,
tma_atom_Q_bf16: Optional[cute.CopyAtom] = None,
sK_bf16: Optional[cute.Tensor] = None,
sV_bf16: Optional[cute.Tensor] = None,
tiled_mma_qk_bf16: Optional[cute.TiledMma] = None,
tiled_mma_pv_bf16: Optional[cute.TiledMma] = None,
mResidualBlockIds: Optional[cute.Tensor] = None,
role: cutlass.Constexpr = SplitRole.ALL,
):
loads_pages = const_expr(role != SplitRole.BF16)
loads_bf16 = const_expr(
self.fused_residual_first_block and role != SplitRole.PAGES
)
num_load_threads = len(self.load_warp_ids) * cute.arch.WARP_SIZE
tidx = cute.arch.thread_idx()[0] % num_load_threads
q_producer_phase = Int32(1)
kv_producer_state = cutlass.pipeline.make_pipeline_state(
cutlass.pipeline.PipelineUserType.Producer, self.kv_stage
)
residual_kv_empty_producer_phase = Int32(1)
residual_k_empty_producer_phase = Int32(1)
residual_v_empty_producer_phase = Int32(1)
residual_q_empty_producer_phase = Int32(1)
tile_scheduler = TileSchedulerCls()
work_tile = tile_scheduler.initial_work_tile_info()
while work_tile.is_valid_tile:
m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx
seqlen = SeqlenInfoCls(batch_idx)
# mQ: [s, d, h, b]
mQ_cur = seqlen.offset_batch_Q(
mQ, batch_idx, dim=3
)[None, None, head_idx]
gQ = cute.local_tile(
mQ_cur,
cute.select(self.mma_tiler_qk, mode=[0, 2]),
(None, 0),
)
mQ_second_cur = seqlen.offset_batch_Q(
mQSecond, batch_idx, dim=3
)[None, None, head_idx]
gQ_second = cute.local_tile(
mQ_second_cur,
cute.select(self.mma_tiler_qk, mode=[0, 2]),
(None, 0),
)
head_idx_kv = (
head_idx // self.qhead_per_kvhead if const_expr(not self.pack_gqa) else head_idx
)
if const_expr(mPageTable is None):
if const_expr(not seqlen.has_cu_seqlens_k):
mK_cur, mV_cur = [t[None, None, head_idx_kv, batch_idx] for t in (mK, mV)]
else:
mK_cur = cute.domain_offset((seqlen.offset_k, 0), mK[None, None, head_idx_kv])
mV_cur = cute.domain_offset((0, seqlen.offset_k), mV[None, None, head_idx_kv])
gK = cute.local_tile(mK_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0))
gV = cute.local_tile(mV_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None))
else:
# Need to keep batch coord None since we'll index into it with page idx
mK_cur, mV_cur = [t[None, None, head_idx_kv, None] for t in (mK, mV)]
gK = cute.local_tile(
mK_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0, None)
)
gV = cute.local_tile(
mV_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None, None)
)
tSgQ = thr_mma_qk.partition_A(gQ)
tSgQ_second = thr_mma_qk.partition_A(gQ_second)
tSgK = thr_mma_qk.partition_B(gK)
tOgV = thr_mma_pv.partition_B(gV)
load_Q_fn, _, _ = copy_utils.tma_get_copy_fn(
tma_atom_Q,
0,
cute.make_layout(1),
tSgQ,
sQ_terms[0],
)
load_Q_second_fn = load_Q_fn
if const_expr(self.query_terms == 2):
(
load_Q_second_fn,
_,
_,
) = copy_utils.tma_get_copy_fn(
tma_atom_Q_second,
0,
cute.make_layout(1),
tSgQ_second,
sQ_terms[1],
)
if const_expr(self.use_tma_KV):
tKsK, tKgK = cpasync.tma_partition(
tma_atom_K,
0, # no multicast
cute.make_layout(1),
cute.group_modes(sK, 0, 3),
cute.group_modes(tSgK, 0, 3),
)
tVsV, tVgV = cpasync.tma_partition(
tma_atom_V,
0, # no multicast
cute.make_layout(1),
cute.group_modes(sV, 0, 3),
cute.group_modes(tOgV, 0, 3),
)
paged_kv_manager = None
else:
page_size = mK.shape[0]
paged_kv_manager = PagedKVManager.create(
mPageTable,
mK,
mV,
FastDivmod.create(page_size),
batch_idx,
head_idx_kv,
tidx,
seqlen.seqlen_k,
0, # leftpad_k
self.n_block_size,
self.head_dim_padded,
self.head_dim_v_padded,
num_load_threads,
mK.element_type,
)
tKsK, tKgK = None, None
tVsV, tVgV = None, None
load_SFQ_fn = None
load_SFQ_second_fn = None
if const_expr(self.quant_qk):
tma_tensor_sfq_cur = seqlen.offset_batch_Q(
tma_tensor_sfq, batch_idx, dim=3
)[None, None, head_idx]
gSFQ = cute.local_tile(
tma_tensor_sfq_cur,
cute.select(self.mma_tiler_qk, mode=[0, 2]),
(None, 0),
)
tSgSFQ = thr_mma_qk.partition_A(gSFQ)
load_SFQ_fn, _, _ = copy_utils.tma_get_copy_fn(
tma_atom_sfq,
0,
cute.make_layout(1),
tSgSFQ,
sSFQ_terms[0],
filter_zeros=True,
)
load_SFQ_second_fn = load_SFQ_fn
if const_expr(self.query_terms == 2):
tma_tensor_sfq_second_cur = seqlen.offset_batch_Q(
tma_tensor_sfq_second, batch_idx, dim=3
)[None, None, head_idx]
gSFQ_second = cute.local_tile(
tma_tensor_sfq_second_cur,
cute.select(self.mma_tiler_qk, mode=[0, 2]),
(None, 0),
)
tSgSFQ_second = thr_mma_qk.partition_A(
gSFQ_second
)
(
load_SFQ_second_fn,
_,
_,
) = copy_utils.tma_get_copy_fn(
tma_atom_sfq_second,
0,
cute.make_layout(1),
tSgSFQ_second,
sSFQ_terms[1],
filter_zeros=True,
)
# Partition SFK similar to K - index batch and head first like mK_cur
tKsSFK = None
tKgSFK = None
if const_expr(self.quant_qk):
if const_expr(mPageTable is None):
if const_expr(not seqlen.has_cu_seqlens_k):
tma_tensor_sfk_cur = tma_tensor_sfk[None, None, head_idx_kv, batch_idx]
gSFK = cute.local_tile(tma_tensor_sfk_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0))
else:
tma_tensor_sfk_cur = cute.domain_offset((seqlen.offset_k, 0), tma_tensor_sfk[None, None, head_idx_kv])
gSFK = cute.local_tile(tma_tensor_sfk_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0))
else:
tma_tensor_sfk_cur = tma_tensor_sfk[None, None, head_idx_kv, None]
gSFK = cute.local_tile(tma_tensor_sfk_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0, None))
tSgSFK = thr_mma_qk.partition_B(gSFK)
# Group only the first 3 modes (static MMA modes) to avoid grouping dynamic Rest modes
tKsSFK, tKgSFK = cpasync.tma_partition(
tma_atom_sfk,
0,
cute.make_layout(1),
cute.group_modes(sSFK, 0, 3),
cute.group_modes(tSgSFK, 0, 3),
)
tKsSFK = cute.filter_zeros(tKsSFK)
tKgSFK = cute.filter_zeros(tKgSFK)
# Partition SFV similar to V
tVsSFV, tVgSFV = None, None
if const_expr(self.quant_pv):
if const_expr(mPageTable is None):
if const_expr(not seqlen.has_cu_seqlens_k):
tma_tensor_sfv_cur = tma_tensor_sfv[None, None, head_idx_kv, batch_idx]
gSFV = cute.local_tile(tma_tensor_sfv_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None))
else:
tma_tensor_sfv_cur = cute.domain_offset((0, seqlen.offset_k), tma_tensor_sfv[None, None, head_idx_kv])
gSFV = cute.local_tile(tma_tensor_sfv_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None))
else:
tma_tensor_sfv_cur = tma_tensor_sfv[None, None, head_idx_kv, None]
gSFV = cute.local_tile(tma_tensor_sfv_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None, None))
tOgSFV = thr_mma_pv.partition_B(gSFV)
# Group only the first 3 modes (static MMA modes) to avoid grouping dynamic Rest modes
tVsSFV, tVgSFV = cpasync.tma_partition(
tma_atom_sfv,
0,
cute.make_layout(1),
cute.group_modes(sSFV, 0, 3),
cute.group_modes(tOgSFV, 0, 3),
)
tVsSFV = cute.filter_zeros(tVsSFV)
tVgSFV = cute.filter_zeros(tVgSFV)
load_Q = partial(
self.load_Q,
load_Q_fn,
mbar_ptr + self.mbar_load_q_full_offset,
mbar_ptr + self.mbar_load_q_empty_offset,
phase=q_producer_phase,
load_SFQ_fn=load_SFQ_fn,
load_Q_second_fn=load_Q_second_fn,
load_SFQ_second_fn=load_SFQ_second_fn,
)
replicate_Q = None
if const_expr(self.q_replicate > 1):
# Replication gets its own pass over the tile between the TMA
# and the first MMA, signalling mbar_q_fp4_ready when the tile
# is whole.
replicate_Q = partial(
self.replicate_Q_fp4_rows,
sQ_uint8,
sSFQ_terms[0],
mbar_ptr + self.mbar_load_q_full_offset,
mbar_ptr + self.mbar_q_fp4_ready_offset,
phase=q_producer_phase ^ 1, # waits for the just-issued TMA
tidx=tidx,
)
stack_Q = None
if const_expr(self.stacked_query_terms):
stack_Q = partial(
self.stack_Q_fp4_terms,
sQ_uint8,
sQ_terms[1],
sSFQ_terms[0],
sSFQ_terms[1],
mbar_ptr + self.mbar_load_q_full_offset,
mbar_ptr + self.mbar_q_fp4_ready_offset,
phase=q_producer_phase ^ 1,
tidx=tidx,
)
# We have to use mbarrier directly in the load for KV instead of replying on
# pipeline_kv, because we could have different number of TMA bytes for K and V
load_K = partial(
self.load_KV,
tma_atom_K,
tKgK,
tKsK,
paged_kv_manager,
sK,
mbar_ptr + self.mbar_load_kv_full_offset,
mbar_ptr + self.mbar_load_kv_empty_offset,
K_or_V="K",
tma_atom_sf=tma_atom_sfk,
tXgSF=tKgSFK,
tXsSF=tKsSFK,
)
load_V = partial(
self.load_KV,
tma_atom_V,
tVgV,
tVsV,
paged_kv_manager,
sV,
mbar_ptr + self.mbar_load_kv_full_offset,
mbar_ptr + self.mbar_load_kv_empty_offset,
K_or_V="V",
tma_atom_sf=tma_atom_sfv,
tXgSF=tVgSFV,
tXsSF=tVsSFV,
)
if const_expr(not self.use_block_sparsity):
iket.range_push("load_seqinfo")
n_block_min, n_block_max = self.split_block_range(
block_info, seqlen, m_block, split_idx, num_splits
)
iket.range_pop()
has_work = self.tile_has_work(
n_block_min, n_block_max, split_idx, num_splits
)
# A split with no kv block runs no MMA, so nothing would release
# the BF16 tiles the sink reload waits on: load only for work.
if has_work:
if const_expr(loads_bf16):
if const_expr(self.residual_source == "paged_bf16"):
# Graph padding keeps row_to_slot=-1 so it cannot own or
# write a tail. TMA still needs a legal source page before
# the zero residual/sink lengths mask the row away.
residual_kv_idx = cutlass.max(
mResidualBlockIds[batch_idx], Int32(0)
)
else:
residual_kv_idx = batch_idx
# BF16 residual K/V layout: (s=128, d, h_kv, b-or-num_blocks).
# Slice batch/block + head, local_tile to BLOCK_SIZE (=128),
# partition via the BF16 tiled MMAs, then tma_partition with
# the BF16 atoms.
thr_mma_qk_bf16_slc = tiled_mma_qk_bf16.get_slice(0)
thr_mma_pv_bf16_slc = tiled_mma_pv_bf16.get_slice(0)
mResK_cur = mResidualK_t[None, None, head_idx_kv, residual_kv_idx]
mResV_cur = mResidualV_t[None, None, head_idx_kv, residual_kv_idx]
gResK = cute.local_tile(
mResK_cur, cute.select(self.mma_tiler_qk, mode=[1, 2]), (None, 0)
)
gResV = cute.local_tile(
mResV_cur, cute.select(self.mma_tiler_pv, mode=[1, 2]), (0, None)
)
tSgResK = thr_mma_qk_bf16_slc.partition_B(gResK)
tOgResV = thr_mma_pv_bf16_slc.partition_B(gResV)
tKsK_bf16, tKgResK = cpasync.tma_partition(
tma_atom_K_bf16,
0, # no multicast
cute.make_layout(1),
cute.group_modes(sK_bf16, 0, 3),
cute.group_modes(tSgResK, 0, 3),
)
tVsV_bf16, tVgResV = cpasync.tma_partition(
tma_atom_V_bf16,
0, # no multicast
cute.make_layout(1),
cute.group_modes(sV_bf16, 0, 3),
cute.group_modes(tOgResV, 0, 3),
)
mResQ_cur = mResidualQ_t[None, None, head_idx, batch_idx]
gResQ = cute.local_tile(
mResQ_cur, cute.select(self.mma_tiler_qk, mode=[0, 2]), (None, 0)
)
tSgResQ = thr_mma_qk_bf16_slc.partition_A(gResQ)
tQsQ_bf16, tQgResQ = cpasync.tma_partition(
tma_atom_Q_bf16,
0, # no multicast
cute.make_layout(1),
cute.group_modes(sQ_bf16, 0, 3),
cute.group_modes(tSgResQ, 0, 3),
)
# BF16 K/V/Q are loaded into dedicated single-stage SMEM tiles
# (sK_bf16 / sV_bf16 / sQ_bf16) using dedicated mbarriers
# (mbar_residual_kv_full + 0 = K, +1 = V, +2 = Q). The FP4
# mainloop's kv pipeline is unchanged. For subsequent work
# tiles (M>1), wait on the empty barrier so the MMA warp
# has finished consuming the previous tile's data before
# the load warp overwrites the SMEM.
bf16_full_mbar = mbar_ptr + self.mbar_residual_kv_full_offset
bf16_empty_mbar = mbar_ptr + self.mbar_residual_kv_empty_offset
if const_expr(self.use_tma_KV) or tidx < cute.arch.WARP_SIZE:
iket.range_push("load_wait_res")
cute.arch.mbarrier_wait(
bf16_empty_mbar + 0,
residual_k_empty_producer_phase
if const_expr(self.fused_sink_block)
else residual_kv_empty_producer_phase,
)
cute.arch.mbarrier_wait(
bf16_empty_mbar + 1,
residual_v_empty_producer_phase
if const_expr(self.fused_sink_block)
else residual_kv_empty_producer_phase,
)
cute.arch.mbarrier_wait(
bf16_empty_mbar + 2,
residual_q_empty_producer_phase
if const_expr(self.fused_sink_block)
else residual_kv_empty_producer_phase,
)
iket.range_pop()
# Slot 0: BF16 K
with cute.arch.elect_one():
cute.arch.mbarrier_arrive_and_expect_tx(
bf16_full_mbar + 0,
cute.cosize(sK_bf16) * 2, # bf16 = 2 bytes
)
cute.copy(
tma_atom_K_bf16,
tKgResK[None, 0],
tKsK_bf16[None, 0],
tma_bar_ptr=bf16_full_mbar + 0,
)
# Slot 1: BF16 V
with cute.arch.elect_one():
cute.arch.mbarrier_arrive_and_expect_tx(
bf16_full_mbar + 1,
cute.cosize(sV_bf16) * 2,
)
cute.copy(
tma_atom_V_bf16,
tVgResV[None, 0],
tVsV_bf16[None, 0],
tma_bar_ptr=bf16_full_mbar + 1,
)
sQ_bf16_stage_elems = const_expr(cute.cosize(sQ_bf16))
with cute.arch.elect_one():
cute.arch.mbarrier_arrive_and_expect_tx(
bf16_full_mbar + 2,
sQ_bf16_stage_elems * 2,
)
cute.copy(
tma_atom_Q_bf16,
tQgResQ[None, m_block],
tQsQ_bf16[None, 0],
tma_bar_ptr=bf16_full_mbar + 2,
)
if const_expr(self.fused_sink_block):
residual_k_empty_producer_phase ^= 1
residual_v_empty_producer_phase ^= 1
residual_q_empty_producer_phase ^= 1
else:
residual_kv_empty_producer_phase ^= 1
if const_expr(loads_pages):
if const_expr(self.use_tma_KV) or tidx < cute.arch.WARP_SIZE:
load_Q(block=self.q_stage * m_block + 0, stage=0) # Q0 + SFQ0
n_block_first = n_block_max - 1 if n_block_max > 0 else 0
iket.range_push("load_pageidx")
page_idx = (
mPageTable[batch_idx, n_block_first]
if const_expr(mPageTable is not None and self.use_tma_KV)
else None
)
if const_expr(not self.use_tma_KV):
paged_kv_manager.load_page_table(n_block_first)
iket.range_pop()
iket.range_push("load_issue_k0")
load_K(block=n_block_max - 1, producer_state=kv_producer_state, page_idx=page_idx) # K0 + SFK0
iket.range_pop()
kv_producer_state.advance()
if const_expr(self.q_stage == 2) and (const_expr(self.use_tma_KV) or tidx < cute.arch.WARP_SIZE):
load_Q(block=self.q_stage * m_block + 1, stage=1) # Q1 + SFQ1
if const_expr(self.q_replicate > 1):
for qs in cutlass.range_constexpr(self.q_stage):
replicate_Q(stage=qs)
if const_expr(self.stacked_query_terms):
# One warp owns both the byte copy and its single
# ready-barrier arrival, including non-TMA KV loads.
if tidx < cute.arch.WARP_SIZE:
stack_Q(stage=0)
q_producer_phase ^= 1
load_V(block=n_block_max - 1, producer_state=kv_producer_state, page_idx=page_idx) # V0 + SFV0
kv_producer_state.advance()
for i in cutlass.range(n_block_max - 1 - n_block_min, unroll=1):
n_block = n_block_max - 2 - i
page_idx = (
mPageTable[batch_idx, n_block]
if const_expr(mPageTable is not None and self.use_tma_KV)
else None
)
if const_expr(not self.use_tma_KV):
paged_kv_manager.load_page_table(n_block)
load_K(block=n_block, producer_state=kv_producer_state, page_idx=page_idx) # Ki + SFKi
kv_producer_state.advance()
load_V(block=n_block, producer_state=kv_producer_state, page_idx=page_idx) # Vi + SFVi
kv_producer_state.advance()
# The sink is merged after the last page, so its operands
# are loaded behind the pages: they stream while the tail's
# GEMMs run, and the sink's own load hides under the last
# blocks. K can be replaced once the tail QK has issued; V
# and Q/P stay resident until the tail PV consumes them, so
# each goes through its own empty barrier.
if const_expr(loads_bf16 and self.fused_sink_block):
if const_expr(self.use_tma_KV) or tidx < cute.arch.WARP_SIZE:
cute.arch.mbarrier_wait(
bf16_empty_mbar + 0,
residual_k_empty_producer_phase,
)
with cute.arch.elect_one():
cute.arch.mbarrier_arrive_and_expect_tx(
bf16_full_mbar + 0,
cute.cosize(sK_bf16) * 2,
)
cute.copy(
tma_atom_K_bf16,
tKgResK[None, 1],
tKsK_bf16[None, 0],
tma_bar_ptr=bf16_full_mbar + 0,
)
residual_k_empty_producer_phase ^= 1
cute.arch.mbarrier_wait(
bf16_empty_mbar + 1,
residual_v_empty_producer_phase,
)
with cute.arch.elect_one():
cute.arch.mbarrier_arrive_and_expect_tx(
bf16_full_mbar + 1,
cute.cosize(sV_bf16) * 2,
)
cute.copy(
tma_atom_V_bf16,
tVgResV[None, 1],
tVsV_bf16[None, 0],
tma_bar_ptr=bf16_full_mbar + 1,
)
residual_v_empty_producer_phase ^= 1
cute.arch.mbarrier_wait(
bf16_empty_mbar + 2,
residual_q_empty_producer_phase,
)
with cute.arch.elect_one():
cute.arch.mbarrier_arrive_and_expect_tx(
bf16_full_mbar + 2,
cute.cosize(sQ_bf16) * 2,
)
cute.copy(
tma_atom_Q_bf16,
tQgResQ[None, m_block],
tQsQ_bf16[None, 0],
tma_bar_ptr=bf16_full_mbar + 2,
)
residual_q_empty_producer_phase ^= 1
else:
kv_producer_state, q_producer_phase = produce_block_sparse_loads_sm100(
blocksparse_tensors,
batch_idx,
head_idx,
m_block,
kv_producer_state,
load_Q,
load_K,
load_V,
pipeline_kv,
self.q_stage,
q_producer_phase,
)
tile_scheduler.prefetch_next_work()
tile_scheduler.advance_to_next_work()
work_tile = tile_scheduler.get_current_work()
# End of persistent scheduler loop
def qk_gemm(self, gemm, tCrK: cute.Tensor, sK_cur: cute.Tensor):
"""Issue one QK GEMM with the K tile in whichever operand slot it holds.
K is the B operand normally and the A operand once S is transposed. The
Q side is already bound into ``gemm``.
"""
if const_expr(self.transpose_s):
gemm(tCrA=tCrK, sA=sK_cur)
else:
gemm(tCrB=tCrK, sB=sK_cur)
def release_shared_p(self, mbar_ptr: cute.Pointer, stage: int):
"""Signal once the PV GEMM just issued has read P from shared memory.
Transposed, P and its scale factors sit in one shared-memory tile that
the softmax rewrites for every block, and the next block's QK is issued
ahead of this block's PV, so the S it signals says nothing about this
PV. Every FP4 PV signals here, which keeps one completion per block.
"""
if const_expr(self.transpose_s):
with cute.arch.elect_one():
tcgen05.commit(mbar_ptr + self.mbar_P_consumed_offset + stage)
def pv_gemm(
self,
gemm,
tOrVi: cute.Tensor,
sV_cur: cute.Tensor,
zero_init,
mbar_ptr: cute.Pointer,
stage: int,
phase: Int32,
):
"""Issue one PV GEMM, splitting the wait on P only where P is in TMEM.
With P in tensor memory the instruction can start on its first half and
wait for the rest mid-flight. A shared-memory A operand has no such
entry point, so the whole tile must already be published; the caller's
wait on ``mbar_P_full_O_rescaled`` covers it and the second-half barrier
is unused.
"""
if const_expr(self.transpose_s):
gemm(tCrB=tOrVi, sB=sV_cur, zero_init=zero_init)
else:
gemm(
tCrB=tOrVi,
sB=sV_cur,
zero_init=zero_init,
mbar_ptr=mbar_ptr + self.mbar_P_full_2_offset + stage,
mbar_phase=phase,
)
@cute.jit
def mma(
self,
tiled_mma_qk: cute.ThrMma,
tiled_mma_pv: cute.ThrMma,
sQ_terms: Tuple[cute.Tensor, ...],
sK: cute.Tensor,
sV: cute.Tensor,
tStSs: Tuple[cute.Tensor, cute.Tensor],
tOtOs: tuple[cute.Tensor],
tOrPs: Tuple[cute.Tensor, cute.Tensor],
pipeline_kv: cutlass.pipeline.PipelineAsync,
mbar_ptr: cute.Pointer,
block_info: BlockInfo,
num_splits: Int32,
SeqlenInfoCls: Callable,
TileSchedulerCls: Callable,
blocksparse_tensors: Optional[BlockSparseTensors],
# In smem
sSFQ_terms: Tuple[Optional[cute.Tensor], ...],
sSFK: Optional[cute.Tensor],
sSFV: Optional[cute.Tensor],
sSFP: Optional[cute.Tensor],
# In tmem - per-stage scale factors
tCtSFQ_terms: Tuple[Tuple[cute.Tensor, ...], ...],
tCtSFKs: Tuple[cute.Tensor, ...],
tCtSFPs: Tuple[cute.Tensor, ...],
tCtSFVs: Tuple[cute.Tensor, ...],
tiled_mma_qk_bf16: Optional[cute.TiledMma] = None,
tiled_mma_pv_bf16: Optional[cute.TiledMma] = None,
sK_bf16: Optional[cute.Tensor] = None,
sV_bf16: Optional[cute.Tensor] = None,
sQ_bf16: Optional[cute.Tensor] = None,
mResidualSeqUsedK: Optional[cute.Tensor] = None,
tOrPs_bf16: Tuple[Optional[cute.Tensor], Optional[cute.Tensor]] = (None, None),
sP: Optional[cute.Tensor] = None,
sP_bf16: Optional[cute.Tensor] = None,
role: cutlass.Constexpr = SplitRole.ALL,
):
runs_pages = const_expr(role != SplitRole.BF16)
runs_tail = const_expr(
self.fused_residual_first_block and role != SplitRole.PAGES
)
runs_sink = const_expr(self.fused_sink_block and role != SplitRole.PAGES)
if const_expr(self.fused_residual_first_block):
tOrV_bf16 = tiled_mma_pv_bf16.make_fragment_B(sV_bf16)
if const_expr(self.transpose_s):
tSrK_bf16 = tiled_mma_qk_bf16.make_fragment_A(sK_bf16)
tSrQ_bf16 = tiled_mma_qk_bf16.make_fragment_B(sQ_bf16)
else:
tSrK_bf16 = tiled_mma_qk_bf16.make_fragment_B(sK_bf16)
tSrQ_bf16 = tiled_mma_qk_bf16.make_fragment_A(sQ_bf16)
tSrQ_bf16_stages = (tSrQ_bf16[None, None, None, 0], tSrQ_bf16[None, None, None, 0])
else:
tSrK_bf16 = None
tOrV_bf16 = None
tSrQ_bf16 = None
tSrQ_bf16_stages = (None, None)
if const_expr(self.transpose_s):
tSrQ_terms = tuple(
tiled_mma_qk.make_fragment_B(sQ_term)
for sQ_term in sQ_terms
)
tSrK = tiled_mma_qk.make_fragment_A(sK)
else:
tSrQ_terms = tuple(
tiled_mma_qk.make_fragment_A(sQ_term)
for sQ_term in sQ_terms
)
tSrK = tiled_mma_qk.make_fragment_B(sK)
tOrV = tiled_mma_pv.make_fragment_B(sV)
tSrQ_term_stages = tuple(
(
(
tSrQ[None, None, None, 0],
tSrQ[None, None, None, 1],
)
if const_expr(self.q_stage == 2)
else (
tSrQ[None, None, None, 0],
tSrQ[None, None, None, 0],
)
)
for tSrQ in tSrQ_terms
)
qk_mma_op, pv_mma_op = tiled_mma_qk.op, tiled_mma_pv.op
if const_expr(self.stacked_query_terms):
# Retain the full-width operand layouts for TMA and BF16 paths.
# This op consumes their populated prefix and writes exactly the
# two score ranges that the FP4 softmax joins.
assert qk_mma_op.shape_mnk == (128, 128, 64)
narrow_shape = (128, 2 * self.transposed_query_rows, 64)
if const_expr(self.is_mxfp4):
qk_mma_op = MmaMXF4Op(
narrow_shape, tcgen05.CtaGroup.ONE, tcgen05.OperandSource.SMEM
)
else:
qk_mma_op = MmaMXF4NVF4Op(
self.sf_dtype,
narrow_shape,
tcgen05.CtaGroup.ONE,
tcgen05.OperandSource.SMEM,
)
# Q is bound here because it is fixed for the whole tile; K arrives per
# KV stage and is supplied at the call site through ``qk_gemm``. Which
# operand each one is depends on whether S is transposed, so bind Q by
# keyword and let the caller name the K side the same way either way.
def _qk_scales(term, stage):
if const_expr(self.transpose_s):
return dict(
tScaleA=tCtSFKs[stage],
tScaleB=tCtSFQ_terms[term][stage],
)
return dict(
tScaleA=tCtSFQ_terms[term][stage],
tScaleB=tCtSFKs[stage],
)
def _make_gemm_si(term, stage):
common = dict(zero_init=(term == 0))
if const_expr(self.transpose_s):
common.update(
tCrB=tSrQ_term_stages[term][stage],
sB=sQ_terms[term][None, None, None, stage],
)
positional = ()
else:
positional = (tSrQ_term_stages[term][stage],)
common.update(
sA=sQ_terms[term][None, None, None, stage]
)
if const_expr(self.quant_qk):
return partial(
sm100_utils.gemm_ptx_partial_fp4,
qk_mma_op,
self.tmem_s_offset[stage],
*positional,
**common,
**_qk_scales(term, stage),
)
return partial(
sm100_utils.gemm_ptx_partial,
qk_mma_op,
self.tmem_s_offset[stage],
*positional,
**common,
)
gemm_Si_terms = [
[
_make_gemm_si(term, stage)
for stage in range(self.q_stage)
]
for term in range(self.query_terms)
]
def _make_gemm_pi(stage):
acc_offset = self.tmem_o_offset[stage if self.q_stage == 2 else 0]
if const_expr(self.narrow_pv):
# O^T = V^T P^T. Both operands keep their shared-memory tiles
# and K-major layouts; the instruction reads the first pv_rows
# rows of P, the only ones softmax fills.
narrow_shape = (self.m_block_size, self.pv_rows, 64)
if const_expr(self.is_mxfp4):
narrow_pv_op = MmaMXF4Op(
narrow_shape, tcgen05.CtaGroup.ONE, tcgen05.OperandSource.SMEM
)
else:
narrow_pv_op = MmaMXF4NVF4Op(
self.sf_dtype,
narrow_shape,
tcgen05.CtaGroup.ONE,
tcgen05.OperandSource.SMEM,
)
def gemm_pv_narrow(tCrB, sB, zero_init):
sm100_utils.gemm_ptx_partial_fp4(
narrow_pv_op,
acc_offset,
tCrB,
tOrPs[stage],
sA=sB,
sB=sP[None, None, None, 0],
tScaleA=tCtSFVs[stage],
tScaleB=tCtSFPs[stage],
zero_init=zero_init,
)
return gemm_pv_narrow
if const_expr(self.transpose_s):
# A from shared memory: no TMEM address, and the split wait the
# TMEM path uses is not available here.
return partial(
sm100_utils.gemm_ptx_partial_fp4,
pv_mma_op,
acc_offset,
tOrPs[stage],
sA=sP[None, None, None, 0],
tScaleA=tCtSFPs[stage],
tScaleB=tCtSFVs[stage],
)
if const_expr(self.quant_pv):
return partial(
sm100_utils.gemm_ptx_partial_fp4,
pv_mma_op,
acc_offset,
tOrPs[stage],
sA=None,
tScaleA=tCtSFPs[stage],
tScaleB=tCtSFVs[stage],
pre_mbar_tiles=self.mbar_p_split(cute.size(tOrPs[stage].shape[2])),
tA_addr=self.tmem_p_offset[stage],
)
return partial(
sm100_utils.gemm_ptx_partial,
pv_mma_op,
acc_offset,
tOrPs[stage],
sA=None,
pre_mbar_tiles=self.mbar_p_split(cute.size(tOrPs[stage].shape[2])),
)
gemm_Pi = [_make_gemm_pi(stage) for stage in range(self.q_stage)]
if const_expr(self.fused_residual_first_block):
qk_mma_op_bf16 = tiled_mma_qk_bf16.op
pv_mma_op_bf16 = tiled_mma_pv_bf16.op
def _make_gemm_si_bf16(stage):
# Q is bound here and K supplied at the call site, mirroring the
# FP4 builders; transposed, Q is the B operand.
if const_expr(self.transpose_s):
return partial(
sm100_utils.gemm_ptx_partial,
qk_mma_op_bf16,
self.tmem_s_offset[stage],
tCrB=tSrQ_bf16_stages[stage],
sB=sQ_bf16[None, None, None, 0],
zero_init=True,
)
return partial(
sm100_utils.gemm_ptx_partial,
qk_mma_op_bf16,
self.tmem_s_offset[stage],
tSrQ_bf16_stages[stage],
sA=sQ_bf16[None, None, None, 0],
zero_init=True,
)
gemm_Si_bf16 = [_make_gemm_si_bf16(stage) for stage in range(self.q_stage)]
gemm_S_sink_bf16 = (
_make_gemm_si_bf16(1)
if const_expr(self.fused_sink_block)
else None
)
def _make_gemm_pi_bf16(stage, operand=None):
tOrP_bf16 = tOrPs_bf16[stage] if operand is None else operand
if const_expr(self.narrow_pv):
# The FP4 blocks accumulate O^T, so the residual's PV does
# too: V^T is an MN-major A operand over the head dimension.
narrow_pv_op_bf16 = tcgen05.MmaF16BF16Op(
pv_mma_op_bf16.a_dtype,
pv_mma_op_bf16.acc_dtype,
(self.m_block_size, self.pv_rows, 16),
tcgen05.CtaGroup.ONE,
tcgen05.OperandSource.SMEM,
pv_mma_op_bf16.b_major_mode,
pv_mma_op_bf16.a_major_mode,
)
acc_offset = self.tmem_o_offset[stage if self.q_stage == 2 else 0]
k_blocks = cute.size(tOrP_bf16.shape[2])
def gemm_pv_bf16_narrow(tCrB, sB, zero_init):
# The instruction count follows the A operand, so V is
# cut to as many K blocks as this P carries: one for
# the sink.
layout = tCrB.layout
tCrA = tCrB
if const_expr(k_blocks != cute.size(layout.shape[2])):
assert k_blocks == 1
tCrA = cute.make_tensor(
tCrB.iterator,
cute.make_layout(
(layout.shape[0], layout.shape[1], 1),
stride=(layout.stride[0], layout.stride[1], 0),
),
)
sm100_utils.gemm_ptx_partial(
narrow_pv_op_bf16,
acc_offset,
tCrA,
tOrP_bf16,
sA=sB,
sB=sP_bf16[None, None, None, 0],
zero_init=zero_init,
)
return gemm_pv_bf16_narrow
if const_expr(self.transpose_s):
# A from shared memory: no split wait, as on the FP4 side.
return partial(
sm100_utils.gemm_ptx_partial,
pv_mma_op_bf16,
self.tmem_o_offset[stage if self.q_stage == 2 else 0],
tOrP_bf16,
sA=sP_bf16[None, None, None, 0],
)
return partial(
sm100_utils.gemm_ptx_partial,
pv_mma_op_bf16,
self.tmem_o_offset[stage if self.q_stage == 2 else 0],
tOrP_bf16,
sA=None,
pre_mbar_tiles=self.mbar_p_split(cute.size(tOrP_bf16.shape[2])),
)
gemm_Pi_bf16 = [_make_gemm_pi_bf16(stage) for stage in range(self.q_stage)]
gemm_P_sink_bf16 = (
_make_gemm_pi_bf16(0, tOrPs_bf16[1])
if const_expr(self.fused_sink_block)
else None
)
else:
gemm_Si_bf16 = None
gemm_S_sink_bf16 = None
gemm_Pi_bf16 = None
gemm_P_sink_bf16 = None
mma_q_consumer_phase = Int32(0)
mma_kv_consumer_state = cutlass.pipeline.make_pipeline_state(
cutlass.pipeline.PipelineUserType.Consumer, self.kv_stage
)
P_full_O_rescaled_phase = Int32(0)
residual_kv_full_phase = Int32(0)
residual_k_full_phase = Int32(0)
residual_v_full_phase = Int32(0)
residual_q_full_phase = Int32(0)
bf16_p_full_phase = Int32(0)
residual_kv_empty_phase = Int32(1)
# Partition for S2T copy of SFQ/SFK - change addr per q_stage to avoid overwriting
if const_expr(self.quant_qk):
tiled_copy_s2t_sfq_terms_staged = [
[
self.mainloop_s2t_copy_and_partition(
sSFQ_terms[term],
tCtSFQ_terms[term][stage],
)
for stage in range(self.q_stage)
]
for term in range(self.query_terms)
]
tiled_copy_s2t_sfk_staged = [
self.mainloop_s2t_copy_and_partition(sSFK, tCtSFKs[stage])
for stage in range(self.q_stage)
]
(
tiled_copy_s2t_sfq_0,
tCsSFQ_compact_s2t_0,
_,
) = tiled_copy_s2t_sfq_terms_staged[0][0]
if const_expr(self.query_terms == 2 and not self.stacked_query_terms):
(
tiled_copy_s2t_sfq_1,
tCsSFQ_compact_s2t_1,
_,
) = tiled_copy_s2t_sfq_terms_staged[1][0]
else:
tiled_copy_s2t_sfq_1 = tiled_copy_s2t_sfq_0
tCsSFQ_compact_s2t_1 = tCsSFQ_compact_s2t_0
tiled_copy_s2t_sfk, tCsSFK_compact_s2t, _ = tiled_copy_s2t_sfk_staged[0]
else:
# Dummy values when quant_qk is False - these won't be used
tiled_copy_s2t_sfq_terms_staged = []
tiled_copy_s2t_sfk_staged = []
tiled_copy_s2t_sfq_0 = None
tCsSFQ_compact_s2t_0 = None
tiled_copy_s2t_sfq_1 = None
tCsSFQ_compact_s2t_1 = None
tiled_copy_s2t_sfk = None
tCsSFK_compact_s2t = None
if const_expr(self.quant_pv):
tiled_copy_s2t_sfv_staged = [
self.mainloop_s2t_copy_and_partition(sSFV, tCtSFVs[stage])
for stage in range(self.q_stage)
]
tiled_copy_s2t_sfv, tCsSFV_compact_s2t, _ = tiled_copy_s2t_sfv_staged[0]
# S2T copy setup for SFP (P scale factors, computed by softmax warp via R2S)
tiled_copy_s2t_sfp_staged = [
self.mainloop_s2t_copy_and_partition(sSFP, tCtSFPs[stage])
for stage in range(self.q_stage)
]
tiled_copy_s2t_sfp, tCsSFP_compact_s2t, _ = tiled_copy_s2t_sfp_staged[0]
else:
tiled_copy_s2t_sfv_staged = []
tiled_copy_s2t_sfv = None
tCsSFV_compact_s2t = None
tiled_copy_s2t_sfp_staged = []
tiled_copy_s2t_sfp = None
tCsSFP_compact_s2t = None
tile_scheduler = TileSchedulerCls()
work_tile = tile_scheduler.initial_work_tile_info()
mma_sfqk_producer_phase = Int32(0)
while work_tile.is_valid_tile:
m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx
seqlen = SeqlenInfoCls(batch_idx)
block_iter_count = Int32(0)
process_tile = False
if const_expr(self.use_block_sparsity):
block_iter_count = get_total_block_count(blocksparse_tensors, batch_idx, head_idx, m_block)
process_tile = block_iter_count > Int32(0)
else:
n_block_min, n_block_max = self.split_block_range(
block_info, seqlen, m_block, split_idx, num_splits
)
block_iter_count = n_block_max - n_block_min
process_tile = self.tile_has_work(
n_block_min, n_block_max, split_idx, num_splits
)
if process_tile:
if const_expr(runs_tail):
if const_expr(self.fused_sink_block):
# The rolling residual seeds O before the FP4 pages.
# The permanent sink is deliberately deferred until the
# page loop has accumulated O at page-local FP4-P scale.
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_residual_kv_full_offset + 0,
residual_k_full_phase,
)
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_residual_kv_full_offset + 2,
residual_q_full_phase,
)
self.qk_gemm(
gemm_Si_bf16[0],
tSrK_bf16[None, None, None, 0],
sK_bf16[None, None, None, 0],
)
with cute.arch.elect_one():
tcgen05.commit(
mbar_ptr + self.mbar_bf16_S_full_offset
)
cute.arch.mbarrier_arrive(
mbar_ptr
+ self.mbar_residual_kv_empty_offset
+ 0
)
residual_k_full_phase ^= 1
residual_q_full_phase ^= 1
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_residual_kv_full_offset + 1,
residual_v_full_phase,
)
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_bf16_P_full_offset,
bf16_p_full_phase,
)
gemm_Pi_bf16[0](
tCrB=tOrV_bf16[None, None, None, 0],
sB=sV_bf16[None, None, None, 0],
zero_init=True,
)
# PV consumes the aliased P/V tiles asynchronously; let
# the Tensor Core completion release the producer slots.
with cute.arch.elect_one():
tcgen05.commit(
mbar_ptr
+ self.mbar_residual_kv_empty_offset
+ 1
)
tcgen05.commit(
mbar_ptr
+ self.mbar_residual_kv_empty_offset
+ 2
)
residual_v_full_phase ^= 1
bf16_p_full_phase ^= 1
else:
# Wait for K and Q (V is consumed later in the PV phase, but
# the QK GEMM only needs K + Q here).
iket.range_push("mma_res_wait_k")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_residual_kv_full_offset + 0,
residual_kv_full_phase,
)
iket.range_pop()
iket.range_push("mma_res_wait_q")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_residual_kv_full_offset + 2,
residual_kv_full_phase,
)
iket.range_pop()
self.qk_gemm(
gemm_Si_bf16[0],
tSrK_bf16[None, None, None, 0],
sK_bf16[None, None, None, 0],
)
with cute.arch.elect_one():
tcgen05.commit(mbar_ptr + self.mbar_bf16_S_full_offset)
# Wait for V (consumed in BF16 GEMM_PV below).
iket.range_push("mma_res_wait_v")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_residual_kv_full_offset + 1,
residual_kv_full_phase,
)
iket.range_pop()
iket.range_push("mma_res_wait_p")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_bf16_P_full_offset,
Int32(0),
)
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_bf16_P_full_2_offset,
Int32(0),
)
iket.range_pop()
# BF16 PV: P_BF16 (TMEM) × V_BF16 (SMEM) → O TMEM (zero_init=True).
gemm_Pi_bf16[0](
tCrB=tOrV_bf16[None, None, None, 0],
sB=sV_bf16[None, None, None, 0],
zero_init=True,
)
with cute.arch.elect_one():
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_residual_kv_empty_offset + 0,
)
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_residual_kv_empty_offset + 1,
)
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_residual_kv_empty_offset + 2,
)
residual_kv_full_phase ^= 1
if const_expr(runs_pages):
for stage in cutlass.range_constexpr(self.q_stage):
iket.range_push("mma_wait_qk")
if const_expr(self.q_replicate > 1 or self.stacked_query_terms):
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_q_fp4_ready_offset + stage,
mma_q_consumer_phase,
)
else:
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_load_q_full_offset + stage,
mma_q_consumer_phase,
)
# 2. wait for K0
if const_expr(stage == 0):
pipeline_kv.consumer_wait(mma_kv_consumer_state)
iket.range_pop()
tSrKi = tSrK[None, None, None, mma_kv_consumer_state.index]
# We don't need to acquire empty S0 / S1.
# For the first iteration, we don't need to wait as we're guaranteed S0 / S1
# are empty. For subsequent iterations, the wait happened at the end
# of the while loop.
# Copy SFQ
# only tmem changes per q_stage.
if const_expr(self.quant_qk):
sm100_utils.tcgen05_after_thread_sync()
iket.range_push("mma_wait_sf")
cute.arch.mbarrier_wait(mbar_ptr + self.mbar_sfqk_load_offset + stage, mma_sfqk_producer_phase)
iket.range_pop()
_, _, tCtSFQ_compact_s2t_0 = (
tiled_copy_s2t_sfq_terms_staged[0][stage]
)
cute.copy(
tiled_copy_s2t_sfq_0,
tCsSFQ_compact_s2t_0[
None, None, None, None, stage
],
tCtSFQ_compact_s2t_0,
)
if const_expr(self.query_terms == 2 and not self.stacked_query_terms):
_, _, tCtSFQ_compact_s2t_1 = (
tiled_copy_s2t_sfq_terms_staged[1][stage]
)
cute.copy(
tiled_copy_s2t_sfq_1,
tCsSFQ_compact_s2t_1[
None, None, None, None, stage
],
tCtSFQ_compact_s2t_1,
)
_, _, tCtSFK_compact_s2t = tiled_copy_s2t_sfk_staged[stage]
tCsSFK_compact_s2t_staged = tCsSFK_compact_s2t[None, None, None, None, mma_kv_consumer_state.index]
cute.copy(
tiled_copy_s2t_sfk,
tCsSFK_compact_s2t_staged,
tCtSFK_compact_s2t,
)
# 3. gemm
# tiled_mma_qk = sm100_utils.gemm(tiled_mma_qk, tStSs[stage], tSrQs[stage], tSrKi, zero_init=True)
sK_cur = sK[None, None, None, mma_kv_consumer_state.index]
if const_expr(self.uneven_kv_smem):
sK_cur = self.offset_kv_smem(
sK_cur, mma_kv_consumer_state.index, mma_kv_consumer_state.phase
)
self.qk_gemm(
gemm_Si_terms[0][stage],
tSrKi,
sK_cur,
)
if const_expr(self.query_terms == 2 and not self.stacked_query_terms):
self.qk_gemm(
gemm_Si_terms[1][stage],
tSrKi,
sK_cur,
)
# 4. release S0 / S1
with cute.arch.elect_one():
tcgen05.commit(mbar_ptr + self.mbar_S_full_offset + stage)
mma_q_consumer_phase ^= 1
if const_expr(self.quant_qk):
mma_sfqk_producer_phase ^= 1
# 5. release K0
pipeline_kv.consumer_release(mma_kv_consumer_state)
mma_kv_consumer_state.advance()
# End of GEMM (Q1 * K0 -> S1)
# Note: Q0 & Q1 are still needed in the seqlen_kv loop
# so we need to release them after the seqlen_kv loop
block_loop_count = block_iter_count - 1
# A tail ahead of the pages has already written O.
O_should_accumulate = runs_tail
for i in cutlass.range(block_loop_count, unroll=1):
# GEMM_PV00 (P0 * V0 -> O0_partial), O0 needs to be accumulated in the seqlen_kv loop
# 1. wait for V0
iket.range_push("mma_wait_v")
pipeline_kv.consumer_wait(mma_kv_consumer_state)
iket.range_pop()
mma_kv_release_state = mma_kv_consumer_state.clone()
Vi_index, Vi_phase = mma_kv_consumer_state.index, mma_kv_consumer_state.phase
tOrVi = tOrV[None, None, None, Vi_index]
for stage in cutlass.range_constexpr(self.q_stage):
def issue_pv():
# 2. acquire corrected O0/O1_partial and P0 / P1
# For the first iteration in this work tile, waiting for O0/O1_partial
# means that the correction warps has finished reading tO during
# the last iteration of the previous work tile has finished.
iket.range_push("mma_wait_p")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage,
P_full_O_rescaled_phase,
)
iket.range_pop()
# No need for mbar.wait because it depends on the same Si as the prev qk mma
if const_expr(self.quant_pv):
# S2T copy SFP: sSFP (smem) -> tCtSFPs (tmem)
_, _, tCtSFP_compact_s2t = tiled_copy_s2t_sfp_staged[stage]
tCsSFP_compact_s2t_cur = tCsSFP_compact_s2t[None, None, None, None, stage]
cute.copy(
tiled_copy_s2t_sfp,
tCsSFP_compact_s2t_cur,
tCtSFP_compact_s2t,
)
# S2T copy SFV: sSFV (smem) -> tCtSFVs (tmem)
_, _, tCtSFV_compact_s2t = tiled_copy_s2t_sfv_staged[stage]
tCsSFV_compact_s2t_staged = tCsSFV_compact_s2t[None, None, None, None, Vi_index]
cute.copy(
tiled_copy_s2t_sfv,
tCsSFV_compact_s2t_staged,
tCtSFV_compact_s2t,
)
sV_cur = sV[None, None, None, Vi_index]
if const_expr(self.uneven_kv_smem):
sV_cur = self.offset_kv_smem(sV_cur, Vi_index, Vi_phase)
self.pv_gemm(
gemm_Pi[stage],
tOrVi,
sV_cur,
self.p_scale_two_level or not O_should_accumulate,
mbar_ptr,
stage,
P_full_O_rescaled_phase,
)
self.release_shared_p(mbar_ptr, stage)
if const_expr(stage == self.q_stage - 1):
pipeline_kv.consumer_release(mma_kv_release_state)
mma_kv_release_state.advance()
# End of GEMM_PV00 (P0 * V0 -> O0_partial)
def issue_qk():
# GEMM_QK0i (Q0 * Ki -> S0)
# 1. wait for Ki
if const_expr(stage == 0):
mma_kv_consumer_state.advance()
iket.range_push("mma_wait_k")
pipeline_kv.consumer_wait(mma_kv_consumer_state)
iket.range_pop()
Ki_index, Ki_phase = mma_kv_consumer_state.index, mma_kv_consumer_state.phase
# The wait below is what makes overwriting Si safe: the
# softmax arrives on it once its tensor-memory load has
# retired, which also implies the previous QK finished.
if const_expr(self.quant_qk):
sm100_utils.tcgen05_after_thread_sync()
iket.range_push("mma_wait_sf")
cute.arch.mbarrier_wait(mbar_ptr + self.mbar_sfqk_load_offset + stage, mma_sfqk_producer_phase)
iket.range_pop()
# One Q stage keeps its scales in the unused
# S stage throughout the page loop. The first
# page loads them again for every new tile;
# the sink reuses those columns only after the loop.
# Two Q stages overwrite each other's scales.
if const_expr(self.q_stage == 2):
_, _, tCtSFQ_compact_s2t_0 = (
tiled_copy_s2t_sfq_terms_staged[0][stage]
)
cute.copy(
tiled_copy_s2t_sfq_0,
tCsSFQ_compact_s2t_0[
None, None, None, None, stage
],
tCtSFQ_compact_s2t_0,
)
if const_expr(self.query_terms == 2 and not self.stacked_query_terms):
_, _, tCtSFQ_compact_s2t_1 = (
tiled_copy_s2t_sfq_terms_staged[1][stage]
)
cute.copy(
tiled_copy_s2t_sfq_1,
tCsSFQ_compact_s2t_1[
None, None, None, None, stage
],
tCtSFQ_compact_s2t_1,
)
_, _, tCtSFK_compact_s2t = tiled_copy_s2t_sfk_staged[stage]
tCsSFK_compact_s2t_staged = tCsSFK_compact_s2t[None, None, None, None, mma_kv_consumer_state.index]
cute.copy(
tiled_copy_s2t_sfk,
tCsSFK_compact_s2t_staged,
tCtSFK_compact_s2t,
)
sK_cur = sK[None, None, None, Ki_index]
if const_expr(self.uneven_kv_smem):
sK_cur = self.offset_kv_smem(sK_cur, Ki_index, Ki_phase)
self.qk_gemm(
gemm_Si_terms[0][stage],
tSrK[None, None, None, Ki_index],
sK_cur,
)
if const_expr(self.query_terms == 2 and not self.stacked_query_terms):
self.qk_gemm(
gemm_Si_terms[1][stage],
tSrK[None, None, None, Ki_index],
sK_cur,
)
# Release Si. The commit fires once every MMA this
# warp issued before it has retired, so it belongs
# next to the QK: leaving it after the PV would make
# the softmax wait for a GEMM it does not read.
with cute.arch.elect_one():
tcgen05.commit(mbar_ptr + self.mbar_S_full_offset + stage)
if const_expr(self.tmem_sfpv_offset is not None):
# Nothing the next QK overwrites is still live. P is in
# shared memory here and its scale factors have tensor
# memory of their own, so S is the only thing left in
# these columns and the softmax released it at the top
# of its step. Issuing the QK before waiting on P is
# what lets it run underneath the softmax instead of
# after it, which is the whole of the block's exposed
# GEMM latency.
issue_qk()
issue_pv()
else:
# P and its scale factors share S's columns, so the
# next QK cannot be issued until this PV has consumed
# them.
issue_pv()
issue_qk()
# 4. release Ki
pipeline_kv.consumer_release(mma_kv_consumer_state)
mma_kv_consumer_state.advance()
P_full_O_rescaled_phase ^= 1
if const_expr(self.quant_qk):
mma_sfqk_producer_phase ^= 1
O_should_accumulate = True
# End of seqlen_kv loop
# release Q0 & Q1
with cute.arch.elect_one():
for stage in cutlass.range_constexpr(self.q_stage):
tcgen05.commit(mbar_ptr + self.mbar_load_q_empty_offset + stage)
# GEMM_PV00 (P0 * V0 -> O0_partial), O0 needs to be accumulated in the seqlen_kv loop
# 1. wait for V0
iket.range_push("mma_wait_v")
pipeline_kv.consumer_wait(mma_kv_consumer_state)
iket.range_pop()
Vi_index, Vi_phase = mma_kv_consumer_state.index, mma_kv_consumer_state.phase
tOrVi = tOrV[None, None, None, Vi_index]
for stage in cutlass.range_constexpr(self.q_stage):
# 2. acquire corrected Oi_partial and Pi
iket.range_push("mma_wait_p")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage, P_full_O_rescaled_phase
)
iket.range_pop()
# 3. gemm
# sm100_utils.gemm(tiled_mma_pv, tOtO0, tOrP0, tOrVi, zero_init=True)
# gemm_Pi[stage](tCrB=tOrVi, sB=sV[None, None, None, Vi_index], zero_init=not O_should_accumulate)
if const_expr(self.quant_pv):
# S2T copy SFP: sSFP (smem) -> tCtSFPs (tmem)
_, _, tCtSFP_compact_s2t = tiled_copy_s2t_sfp_staged[stage]
tCsSFP_compact_s2t_cur = tCsSFP_compact_s2t[None, None, None, None, stage]
cute.copy(
tiled_copy_s2t_sfp,
tCsSFP_compact_s2t_cur,
tCtSFP_compact_s2t,
)
# S2T copy SFV: sSFV (smem) -> tCtSFVs (tmem)
_, _, tCtSFV_compact_s2t = tiled_copy_s2t_sfv_staged[stage]
tCsSFV_compact_s2t_staged = tCsSFV_compact_s2t[None, None, None, None, Vi_index]
cute.copy(
tiled_copy_s2t_sfv,
tCsSFV_compact_s2t_staged,
tCtSFV_compact_s2t,
)
sV_cur = sV[None, None, None, Vi_index]
if const_expr(self.uneven_kv_smem):
sV_cur = self.offset_kv_smem(sV_cur, Vi_index, Vi_phase)
_zi_post = self.p_scale_two_level or not O_should_accumulate
self.pv_gemm(
gemm_Pi[stage],
tOrVi,
sV_cur,
_zi_post,
mbar_ptr,
stage,
P_full_O_rescaled_phase,
)
self.release_shared_p(mbar_ptr, stage)
# 4. release accumulated O0_partial
# We do need O_full here since for the last tile, by the time the softmax warp
# has signaled to the correction warps, the softmax warp has just finished compute
# the row sum of the current tile. It does not guarantee that the 1st tile
# of the next work tile has been computed yet.
if const_expr(not runs_sink):
with cute.arch.elect_one():
tcgen05.commit(mbar_ptr + self.mbar_O_full_offset + stage)
if const_expr(self.transpose_s and self.is_persistent):
# No later page observes this tile's final P phase.
# Close it before another persistent tile can reuse
# the barrier and overwrite the shared P tile.
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_P_consumed_offset + stage,
P_full_O_rescaled_phase,
)
# End of GEMM_PV00 (P0 * V0 -> O0_partial)
P_full_O_rescaled_phase ^= 1
# 5. release Vi_end
pipeline_kv.consumer_release(mma_kv_consumer_state)
mma_kv_consumer_state.advance()
# End of GEMM_PV1(i_end) (P1 * Vi_end -> O1)
elif const_expr(not runs_sink):
# The tail's PV is the last GEMM of a split with no page
# and no sink.
with cute.arch.elect_one():
tcgen05.commit(mbar_ptr + self.mbar_O_full_offset)
if const_expr(runs_sink):
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_residual_kv_full_offset + 0,
residual_k_full_phase,
)
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_residual_kv_full_offset + 2,
residual_q_full_phase,
)
self.qk_gemm(
gemm_S_sink_bf16,
tSrK_bf16[None, None, None, 0],
sK_bf16[None, None, None, 0],
)
with cute.arch.elect_one():
tcgen05.commit(mbar_ptr + self.mbar_S_full_offset)
cute.arch.mbarrier_arrive(
mbar_ptr
+ self.mbar_residual_kv_empty_offset
+ 0
)
residual_k_full_phase ^= 1
residual_q_full_phase ^= 1
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_residual_kv_full_offset + 1,
residual_v_full_phase,
)
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_bf16_P_full_offset,
bf16_p_full_phase,
)
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_P_full_O_rescaled_offset,
P_full_O_rescaled_phase,
)
gemm_P_sink_bf16(
tCrB=tOrV_bf16[None, None, None, 0],
sB=sV_bf16[None, None, None, 0],
zero_init=self.p_scale_two_level and runs_pages,
)
with cute.arch.elect_one():
tcgen05.commit(
mbar_ptr + self.mbar_O_full_offset
)
cute.arch.mbarrier_arrive(
mbar_ptr
+ self.mbar_residual_kv_empty_offset
+ 1
)
cute.arch.mbarrier_arrive(
mbar_ptr
+ self.mbar_residual_kv_empty_offset
+ 2
)
residual_v_full_phase ^= 1
bf16_p_full_phase ^= 1
P_full_O_rescaled_phase ^= 1
# Advance to next tile
tile_scheduler.advance_to_next_work()
work_tile = tile_scheduler.get_current_work()
# End of persistent scheduler loop
@cute.jit
def softmax_warpgroup_reduce(
self,
vals: cute.Tensor,
sSoftmaxRed: cute.Tensor,
tidx: Int32,
red_buf: Int32,
op: Callable,
intra_warp: cutlass.Constexpr[bool] = True,
row_group: cutlass.Constexpr[int] = 0,
) -> Int32:
"""Reduce one value per query row across all softmax threads, in place.
Transposed, a row of S runs along the thread axis, so a row reduction is
a warp butterfly followed by an exchange between the four softmax warps.
The scratch is double buffered by ``red_buf`` so a single named barrier
per call is enough: a warp can only overwrite the buffer it read two
calls ago, and the intervening barrier already ordered that read.
``intra_warp`` is for callers that already hold the warp-wide result
because they needed one of the butterfly's intermediate stages anyway.
"""
num_warps = const_expr(len(self.softmax0_warp_ids))
rows = const_expr(self.softmax_rows_per_group)
if const_expr(intra_warp):
for j in cutlass.range_constexpr(rows):
vals[j] = utils.warp_reduce(vals[j], op)
base = const_expr(row_group * 2 * num_warps * rows) + red_buf * (num_warps * rows)
if tidx % cute.arch.WARP_SIZE == 0:
warp = tidx // cute.arch.WARP_SIZE
for j in cutlass.range_constexpr(rows):
sSoftmaxRed[base + warp * rows + j] = vals[j]
cute.arch.barrier(
barrier_id=int(NamedBarrierFwd.SoftmaxReduce) + row_group,
number_of_threads=num_warps * cute.arch.WARP_SIZE,
)
for j in cutlass.range_constexpr(rows):
acc = sSoftmaxRed[base + j]
for w in cutlass.range_constexpr(1, num_warps):
acc = op(acc, sSoftmaxRed[base + w * rows + j])
vals[j] = acc
return red_buf ^ 1
@cute.jit
def softmax_wait_corr(
self, mbar_ptr: cute.Pointer, stage: int | Int32, si_corr_producer_phase: Int32
) -> Int32:
"""Wait until correction has taken the factors last published in
sScale, and return the producer phase for the next wait."""
iket.range_push("sm_wait_corr")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_softmax_corr_empty_offset + stage, si_corr_producer_phase
)
iket.range_pop()
return si_corr_producer_phase ^ 1
def scatter_lane(self, row: int) -> int:
"""The lane within a cluster of ``softmax_rows_per_group`` that ends a
reduce-scatter holding ``row``: the row's bits in reverse order, since
the first butterfly step splits the rows in halves."""
steps = self.softmax_rows_per_group.bit_length() - 1
return int(format(row, f"0{steps}b")[::-1], 2) if steps else 0
@cute.jit
def scatter_row(self, tidx: Int32) -> Int32:
"""The row a lane holds after the reduce-scatter, ``scatter_lane``
inverted."""
rows = const_expr(self.softmax_rows_per_group)
row = Int32(0)
for i in cutlass.range_constexpr(rows.bit_length() - 1):
row += ((tidx >> i) & 1) * (rows >> (i + 1))
return row
@cute.jit
def softmax_warpgroup_reduce_scattered(
self,
val: Float32,
own_row: Int32,
block_max: cute.Tensor,
sSoftmaxRed: cute.Tensor,
tidx: Int32,
red_buf: Int32,
row_group: cutlass.Constexpr[int] = 0,
) -> Int32:
"""``softmax_warpgroup_reduce`` for a warp maximum reduce-scattered over
lanes: the first lanes of each warp hold one row each, and every thread
reads every row back, so ``block_max`` ends as the unscattered reduce
leaves it. Same scratch, slots and barrier."""
num_warps = const_expr(len(self.softmax0_warp_ids))
rows = const_expr(self.softmax_rows_per_group)
base = const_expr(row_group * 2 * num_warps * rows) + red_buf * (num_warps * rows)
if tidx % cute.arch.WARP_SIZE < rows:
warp = tidx // cute.arch.WARP_SIZE
sSoftmaxRed[base + warp * rows + own_row] = val
cute.arch.barrier(
barrier_id=int(NamedBarrierFwd.SoftmaxReduce) + row_group,
number_of_threads=num_warps * cute.arch.WARP_SIZE,
)
for j in cutlass.range_constexpr(rows):
acc = sSoftmaxRed[base + j]
for w in cutlass.range_constexpr(1, num_warps):
acc = cute.arch.fmax(acc, sSoftmaxRed[base + w * rows + j])
block_max[j] = acc
return red_buf ^ 1
@cute.jit
def split_block_range(
self,
block_info: BlockInfo,
seqlen: SeqlenInfoQK,
m_block: Int32,
split_idx: Int32,
num_splits: Int32,
) -> Tuple[Int32, Int32]:
"""The FP4 page blocks a split walks, ``[n_block_min, n_block_max)``.
With a split of its own for the BF16 blocks, ``num_splits`` counts it
too: the pages are divided among the others exactly as a launch with
no BF16 block would divide them, and the last split is handed none.
Every warp role must ask here, since a role that saw a different range
would wait on barriers the others never arrive on.
"""
if const_expr(not self.bf16_split):
return block_info.get_n_block_min_max(
seqlen, m_block, split_idx, num_splits
)
n_block_min, n_block_max = block_info.get_n_block_min_max(
seqlen, m_block, split_idx, num_splits - 1
)
if split_idx == num_splits - 1:
n_block_max = n_block_min
return n_block_min, n_block_max
@cute.jit
def is_bf16_split(self, split_idx: Int32, num_splits: Int32):
"""Whether this is the split that merges the tail and the sink."""
if const_expr(self.bf16_split):
return split_idx == num_splits - 1
return split_idx == self.residual_split_idx
@cute.jit
def tile_has_work(
self,
n_block_min: Int32,
n_block_max: Int32,
split_idx: Int32,
num_splits: Int32,
):
"""Whether a work tile has anything for the pipeline to do.
Without split-k every tile runs, including one whose fp4 length is
zero: its single mainloop step is fully masked and the epilogue still
owes an output row. Split-k drops empty splits instead, except for the
one that owns the residual block, which has work whether or not any fp4
block landed in it.
Every warp asks this question separately and they have to agree, so
they all ask it here.
"""
if const_expr(not self.is_split_kv):
return True
has_work = n_block_min < n_block_max
if const_expr(self.fused_residual_first_block):
if self.is_bf16_split(split_idx, num_splits):
has_work = cutlass.Boolean(True)
return has_work
@cute.jit
def run_split_role(self, is_bf16_cta, role_body: Callable):
"""Run one warp role's body, specialised for what this CTA merges.
``role_body`` takes the :class:`SplitRole` as a compile-time keyword,
so each CTA runs one of two separately compiled bodies when the BF16
blocks have a split of their own, and the one combined body otherwise.
A CTA owns a single tile, which is what keeps every barrier phase of
one body independent of the other's.
"""
if const_expr(self.bf16_split):
if is_bf16_cta:
role_body(role=SplitRole.BF16)
else:
role_body(role=SplitRole.PAGES)
else:
role_body(role=SplitRole.ALL)
@cute.jit
def zero_transposed_p(self, sP_uint8: cute.Tensor, sSFP: cute.Tensor, tidx: Int32):
"""Zero the shared P tile and its scale factors once per CTA.
Only the first ``transposed_query_rows`` rows are ever written, but the
MMA reads all 128 and an undefined E4M3 scale factor can decode to NaN.
Zeroing the whole region is a permutation-invariant write, so the
swizzle can be ignored and the buffers filled as flat words.
"""
num_threads = const_expr(len(self.softmax0_warp_ids) * cute.arch.WARP_SIZE)
p_words = const_expr(self.m_block_size * self.n_block_size // 2 // 4)
sfp_words = const_expr(
self.m_block_size * self.head_dim_v_padded // self.sf_vec_size * self.q_stage // 4
)
p_flat = cute.make_tensor(
cute.recast_ptr(sP_uint8.iterator, dtype=Int32), cute.make_layout(p_words)
)
sfp_flat = cute.make_tensor(
cute.recast_ptr(sSFP.iterator, dtype=Int32), cute.make_layout(sfp_words)
)
for i in cutlass.range_constexpr(p_words // num_threads):
p_flat[tidx + i * num_threads] = Int32(0)
for i in cutlass.range_constexpr(max(1, sfp_words // num_threads)):
if tidx + i * num_threads < sfp_words:
sfp_flat[tidx + i * num_threads] = Int32(0)
@cute.jit
def zero_transposed_p_bf16(
self,
sP_bf16: cute.Tensor,
sP_bf16_mk: cute.Tensor,
tidx: Int32,
row_group: cutlass.Constexpr[int] = 0,
):
"""Zero the shared BF16 P tile of the residual block.
The FP4 tile is cleared once per CTA, but this one shares its bytes
with the residual Q tile and so cannot be cleared until the QK GEMM
has consumed it. As above the write is permutation invariant, so the
swizzle can be ignored and the buffer filled as flat words.
Several row groups wake on the same S and nothing orders one group's
clear against another's stores, so then each clears only its share of
the rows past ``transposed_query_rows``: a group stores every kv
position of its own rows, and no group stores into those.
"""
groups = const_expr(self.softmax_row_groups)
if const_expr(groups == 1):
num_threads = const_expr(len(self.softmax0_warp_ids) * cute.arch.WARP_SIZE)
words = const_expr(self.m_block_size * self.n_block_size * 2 // 4)
flat = cute.make_tensor(
cute.recast_ptr(sP_bf16.iterator, dtype=Int32), cute.make_layout(words)
)
for i in cutlass.range_constexpr(words // num_threads):
flat[tidx + i * num_threads] = Int32(0)
else:
share = const_expr(
(self.m_block_size - self.transposed_query_rows) // groups
)
first = const_expr(self.transposed_query_rows + row_group * share)
for i in cutlass.range_constexpr(share):
sP_bf16_mk[first + i, tidx] = cutlass.BFloat16(0.0)
@cute.jit
def softmax_residual_step_transposed(
self,
softmax_scale_log2: Float32,
mbar_ptr: cute.Pointer,
thr_tmem_load: cute.CopyAtom,
tStS_t2r: cute.Tensor,
frg_shape,
row_max: cute.Tensor,
row_sum: cute.Tensor,
sP_bf16: cute.Tensor,
sP_bf16_mk: cute.Tensor,
sSoftmaxRed: cute.Tensor,
red_buf: Int32,
tidx: Int32,
batch_idx: Int32,
split_idx: Int32,
mResidualSeqUsedK: Optional[cute.Tensor],
mSinkSeqUsedK: Optional[cute.Tensor] = None,
seqused_fp4: Int32 = Int32(1),
row_group: cutlass.Constexpr[int] = 0,
pages_follow: cutlass.Constexpr[bool] = True,
) -> Int32:
"""The residual block in the same orientation as the FP4 blocks.
Structurally ``softmax_step_transposed`` with the quantization replaced
by a plain bf16 store, since P feeds a bf16 MMA and carries no scale
factors. It runs before any FP4 block and seeds the online softmax, so
``row_max`` and ``row_sum`` leave here holding the residual's
contribution and the FP4 blocks merge onto them with is_first=False.
``pages_follow`` is whether this CTA has FP4 blocks to run after it.
"""
rows = const_expr(self.softmax_rows_per_group)
row_base = const_expr(row_group * rows)
iket.range_push("sm_res_wait_s")
cute.arch.mbarrier_wait(mbar_ptr + self.mbar_bf16_S_full_offset, Int32(0))
iket.range_pop()
iket.range_push("sm_res_comp")
# Safe here and only here: the QK GEMM has just released the Q tile
# these bytes belong to. With one row group the row-maximum exchange
# below carries the barrier that separates this clear from the stores
# at the end; with more the groups clear disjoint rows that none of
# them stores into.
self.zero_transposed_p_bf16(sP_bf16, sP_bf16_mk, tidx, row_group=row_group)
tSrS = cute.make_rmem_tensor(frg_shape, self.qk_acc_dtype)
cute.copy(thr_tmem_load, tStS_t2r, tSrS)
cute.arch.fence_view_async_tmem_load()
if const_expr(mSinkSeqUsedK is not None and pages_follow):
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset)
# Transposed, thread ``tidx`` owns kv position ``tidx``, so the residual
# length is one predicate. A split that does not own the residual sees a
# length of zero, masks every position, and contributes nothing.
seqused_k = Int32(self.n_block_size)
residual_start = Int32(0)
if const_expr(mResidualSeqUsedK is not None):
seqused_k = mResidualSeqUsedK[batch_idx]
if const_expr(self.is_split_kv and not self.bf16_split):
if split_idx != self.residual_split_idx:
seqused_k = Int32(0)
if const_expr(mSinkSeqUsedK is not None):
sink_length = mSinkSeqUsedK[batch_idx]
if seqused_fp4 == Int32(0):
residual_start = sink_length
if (tidx < residual_start) or (tidx >= seqused_k):
for j in cutlass.range_constexpr(rows):
tSrS[j] = -Float32.inf
block_max = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
for j in cutlass.range_constexpr(rows):
block_max[j] = utils.warp_reduce(tSrS[j], cute.arch.fmax)
red_buf = self.softmax_warpgroup_reduce(
block_max,
sSoftmaxRed,
tidx,
red_buf,
cute.arch.fmax,
intra_warp=False,
row_group=row_group,
)
max_scaled = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
for j in cutlass.range_constexpr(rows):
row_max[j] = block_max[j]
safe = block_max[j] if block_max[j] != -Float32.inf else 0.0
max_scaled[j] = safe * softmax_scale_log2
if const_expr(self.p_scale_rescale_log2):
max_scaled[j] -= Float32(self.p_scale_rescale_log2)
for j in cutlass.range_constexpr(rows):
tSrS[j] = cute.arch.exp2(tSrS[j] * softmax_scale_log2 - max_scaled[j])
row_sum[j] = tSrS[j]
tSrP = cute.make_rmem_tensor(frg_shape, cutlass.BFloat16)
tSrP.store(tSrS.load().to(cutlass.BFloat16))
for j in cutlass.range_constexpr(rows):
sP_bf16_mk[row_base + j, tidx] = tSrP[j]
# The MMA reads P through the async proxy while these are generic-proxy
# stores, so the mbarrier below needs the fence to make them visible.
cute.arch.fence_view_async_shared()
iket.range_pop()
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_bf16_P_full_offset)
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_bf16_P_full_2_offset)
return red_buf
@cute.jit
def softmax_sink_step_transposed(
self,
mma_si_consumer_phase: Int32,
si_corr_producer_phase: Int32,
red_buf: Int32,
softmax_scale_log2: Float32,
mbar_ptr: cute.Pointer,
thr_tmem_load: cute.CopyAtom,
tStS_t2r: cute.Tensor,
tStS_sink_t2r: cute.Tensor,
frg_shape,
row_max: cute.Tensor,
row_sum: cute.Tensor,
sScale: cute.Tensor,
sP_bf16: cute.Tensor,
sP_bf16_mk: cute.Tensor,
sSoftmaxRed: cute.Tensor,
stage: int | Int32,
tidx: Int32,
batch_idx: Int32,
mSinkSeqUsedK: cute.Tensor,
split_idx: Int32,
row_group: cutlass.Constexpr[int] = 0,
waits_for_previous: cutlass.Constexpr[bool] = True,
) -> Tuple[Int32, Int32, Int32]:
"""Merge the permanent BF16 sink after every FP4 page.
``waits_for_previous`` as for ``softmax_step_transposed``: false only
when nothing in the tile published before it.
The sink's probabilities are kept in BF16, so its large score can
rescale the FP32 output accumulator without forcing the FP4 page P
tiles to quantize against the sink maximum.
"""
rows = const_expr(self.softmax_rows_per_group)
row_base = const_expr(row_group * rows)
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_S_full_offset + stage,
mma_si_consumer_phase,
)
tSrS = cute.make_rmem_tensor(frg_shape, self.qk_acc_dtype)
cute.copy(thr_tmem_load, tStS_sink_t2r, tSrS)
cute.arch.fence_view_async_tmem_load()
sink_length = mSinkSeqUsedK[batch_idx]
if const_expr(self.is_split_kv and not self.bf16_split):
# One sink, one owner, as with the tail: every other split merges
# a length of zero, so the combine sees the sink exactly once.
if split_idx != self.residual_split_idx:
sink_length = Int32(0)
if tidx >= sink_length:
for j in cutlass.range_constexpr(rows):
tSrS[j] = -Float32.inf
block_max = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
for j in cutlass.range_constexpr(rows):
block_max[j] = utils.warp_reduce(tSrS[j], cute.arch.fmax)
red_buf = self.softmax_warpgroup_reduce(
block_max,
sSoftmaxRed,
tidx,
red_buf,
cute.arch.fmax,
intra_warp=False,
row_group=row_group,
)
acc_scale = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
row_max_safe = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
for j in cutlass.range_constexpr(rows):
row_max_old = row_max[j]
row_max_new = cute.arch.fmax(row_max_old, block_max[j])
safe = row_max_new if row_max_new != -Float32.inf else 0.0
acc_scale_ = (row_max_old - safe) * softmax_scale_log2
scale = utils.exp2f(acc_scale_)
if const_expr(RESCALE_THRESHOLD > 0.0):
if acc_scale_ >= -RESCALE_THRESHOLD:
row_max_new = row_max_old
safe = row_max_old
scale = Float32(1.0)
row_max[j] = row_max_new
row_max_safe[j] = safe
acc_scale[j] = scale
if const_expr(waits_for_previous):
si_corr_producer_phase = self.softmax_wait_corr(
mbar_ptr, stage, si_corr_producer_phase
)
if tidx == 0:
for j in cutlass.range_constexpr(rows):
sScale[row_base + j + stage * self.m_block_size] = acc_scale[j]
if const_expr(self.p_scale_two_level):
sScale[row_base + j + stage * self.m_block_size + self.m_block_size * 2] = 1.0
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_softmax_corr_full_offset + stage
)
max_scaled = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
for j in cutlass.range_constexpr(rows):
max_scaled[j] = row_max_safe[j] * softmax_scale_log2
if const_expr(self.p_scale_rescale_log2):
max_scaled[j] -= Float32(self.p_scale_rescale_log2)
tSrS[j] = cute.arch.exp2(tSrS[j] * softmax_scale_log2 - max_scaled[j])
row_sum[j] = row_sum[j] * acc_scale[j] + tSrS[j]
tSrP = cute.make_rmem_tensor(frg_shape, cutlass.BFloat16)
tSrP.store(tSrS.load().to(cutlass.BFloat16))
for j in cutlass.range_constexpr(rows):
sP_bf16_mk[row_base + j, tidx] = tSrP[j]
cute.arch.fence_view_async_shared()
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_bf16_P_full_offset
)
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage
)
return mma_si_consumer_phase ^ 1, si_corr_producer_phase, red_buf
@cute.jit
def softmax_loop_transposed(
self,
stage: int | Int32,
softmax_scale_log2: Float32,
thr_mma_qk: cute.ThrMma,
tStSi: cute.Tensor,
sScale: cute.Tensor,
mLSE: Optional[cute.Tensor],
learnable_sink: Optional[cute.Tensor],
mbar_ptr: cute.Pointer,
block_info: BlockInfo,
num_splits: Int32,
SeqlenInfoCls: Callable,
TileSchedulerCls: Callable,
sSFP: cute.Tensor,
sP_uint8: cute.Tensor,
sSoftmaxRed: cute.Tensor,
tidx: Int32,
row_group: cutlass.Constexpr[int] = 0,
mResidualSeqUsedK: Optional[cute.Tensor] = None,
mSinkSeqUsedK: Optional[cute.Tensor] = None,
sP_bf16: Optional[cute.Tensor] = None,
sP_bf16_mk: Optional[cute.Tensor] = None,
role: cutlass.Constexpr = SplitRole.ALL,
):
"""Softmax over an S tile held as (kv position, query row).
Thread ``tidx`` owns kv position ``tidx`` of the block and the
``transposed_query_rows`` columns that carry a query, so it runs that
many exponentials instead of a whole row of keys. The row maximum is
along kv and therefore crosses threads; the row sum is deferred to the
end of the tile because the running correction factor is the same in
every thread and factors out of the accumulation.
"""
runs_pages = const_expr(role != SplitRole.BF16)
runs_tail = const_expr(
self.fused_residual_first_block and role != SplitRole.PAGES
)
runs_sink = const_expr(self.fused_sink_block and role != SplitRole.PAGES)
rows = const_expr(self.softmax_rows_per_group)
row_base = const_expr(row_group * rows)
# Only the leading columns of S carry a query; the rest of the MMA tile
# is padding that the tensor-memory load simply never reads. A column of
# S is a tensor-memory column, so a warpgroup reaches its own rows by
# advancing the iterator past the group before it.
tStS_group = cute.make_tensor(tStSi.iterator + row_base, tStSi.layout)
tStS_live = cute.composition(tStS_group, cute.make_layout((self.m_block_size, rows)))
tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2]))
tScS_live = cute.composition(tScS, cute.make_layout((self.m_block_size, rows)))
tmem_load_atom = cute.make_copy_atom(
tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(rows)),
Float32,
)
thr_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tStS_live).get_slice(tidx)
tStS_t2r = thr_tmem_load.partition_S(tStS_live)
tStS_second_t2r = None
if const_expr(self.stacked_query_terms):
tStS_second_group = cute.make_tensor(
tStSi.iterator + self.transposed_query_rows + row_base,
tStSi.layout,
)
tStS_second_live = cute.composition(
tStS_second_group, cute.make_layout((self.m_block_size, rows))
)
tStS_second_t2r = thr_tmem_load.partition_S(tStS_second_live)
tStS_sink_t2r = None
if const_expr(self.fused_sink_block):
sink_offset = self.tmem_s_offset[1] - self.tmem_s_offset[0]
tStS_sink_group = cute.make_tensor(
tStSi.iterator + sink_offset + row_base, tStSi.layout
)
tStS_sink_live = cute.composition(
tStS_sink_group,
cute.make_layout((self.m_block_size, rows)),
)
tStS_sink_t2r = thr_tmem_load.partition_S(tStS_sink_live)
frg_shape = thr_tmem_load.partition_D(tScS_live).shape
if const_expr(runs_pages):
self.zero_transposed_p(sP_uint8, sSFP, tidx)
mma_si_consumer_phase = Int32(0)
si_corr_producer_phase = Int32(1)
red_buf = Int32(0)
# The first FP4 QK may overwrite S once this arrives. Behind a tail
# with a sink that is only after the tail's S has been read, so the
# residual step arrives instead.
if const_expr(runs_pages and not (runs_tail and self.fused_sink_block)):
if stage == 1:
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset + 0)
if const_expr(self.q_stage == 1) and stage == 0:
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset + 0)
row_max = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
row_sum = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
softmax_step = partial(
self.softmax_step_transposed,
softmax_scale_log2=softmax_scale_log2,
mbar_ptr=mbar_ptr,
thr_tmem_load=thr_tmem_load,
tStS_t2r=tStS_t2r,
tStS_second_t2r=tStS_second_t2r,
frg_shape=frg_shape,
row_max=row_max,
row_sum=row_sum,
sScale=sScale,
sSFP=sSFP,
sP_uint8=sP_uint8,
sSoftmaxRed=sSoftmaxRed,
stage=stage,
tidx=tidx,
row_group=row_group,
)
tile_scheduler = TileSchedulerCls()
work_tile = tile_scheduler.initial_work_tile_info()
while work_tile.is_valid_tile:
m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx
iket.range_push("sm_seqinfo")
seqlen = SeqlenInfoCls(batch_idx)
n_block_min, n_block_max = self.split_block_range(
block_info, seqlen, m_block, split_idx, num_splits
)
iket.range_pop()
# A split can be handed no kv block at all. Such a tile must stay out
# of the correction handshake entirely, as on the untransposed path,
# or the two warp groups fall out of step.
has_work = self.tile_has_work(
n_block_min, n_block_max, split_idx, num_splits
)
if has_work:
row_max.fill(-Float32.inf)
row_sum.fill(0.0)
iket.range_push("sm_wait_corr")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_softmax_corr_empty_offset + stage,
si_corr_producer_phase,
)
iket.range_pop()
si_corr_producer_phase ^= 1
seeded = runs_tail
if const_expr(runs_tail):
if const_expr(self.fused_sink_block):
red_buf = self.softmax_residual_step_transposed(
softmax_scale_log2=softmax_scale_log2,
mbar_ptr=mbar_ptr,
thr_tmem_load=thr_tmem_load,
tStS_t2r=tStS_t2r,
frg_shape=frg_shape,
row_max=row_max,
row_sum=row_sum,
sP_bf16=sP_bf16,
sP_bf16_mk=sP_bf16_mk,
sSoftmaxRed=sSoftmaxRed,
red_buf=red_buf,
tidx=tidx,
batch_idx=batch_idx,
mSinkSeqUsedK=mSinkSeqUsedK,
split_idx=split_idx,
mResidualSeqUsedK=mResidualSeqUsedK,
seqused_fp4=seqlen.seqlen_k,
row_group=row_group,
pages_follow=runs_pages,
)
else:
red_buf = self.softmax_residual_step_transposed(
softmax_scale_log2=softmax_scale_log2,
mbar_ptr=mbar_ptr,
thr_tmem_load=thr_tmem_load,
tStS_t2r=tStS_t2r,
frg_shape=frg_shape,
row_max=row_max,
row_sum=row_sum,
sP_bf16=sP_bf16,
sP_bf16_mk=sP_bf16_mk,
sSoftmaxRed=sSoftmaxRed,
red_buf=red_buf,
tidx=tidx,
batch_idx=batch_idx,
split_idx=split_idx,
mResidualSeqUsedK=mResidualSeqUsedK,
row_group=row_group,
)
if const_expr(runs_pages):
sink_length = Int32(0)
if const_expr(self.fused_sink_block):
sink_length = mSinkSeqUsedK[batch_idx]
mma_si_consumer_phase, si_corr_producer_phase, red_buf = softmax_step(
mma_si_consumer_phase,
si_corr_producer_phase,
red_buf,
n_block_max - 1,
seqlen_k=seqlen.seqlen_k,
is_first=not seeded,
mask_seqlen=True,
sink_length=sink_length,
mask_sink=True,
waits_for_previous=False,
)
# Blocks run from the last page down, so page zero, whose
# sink duplicate is masked, can only be the first block or
# the last; the steps between carry no sink test.
peel_last = const_expr(self.fused_sink_block)
for n_tile in cutlass.range(
n_block_max - 1 - n_block_min - int(peel_last), unroll=1
):
mma_si_consumer_phase, si_corr_producer_phase, red_buf = softmax_step(
mma_si_consumer_phase,
si_corr_producer_phase,
red_buf,
n_block_max - 2 - n_tile,
seqlen_k=seqlen.seqlen_k,
is_first=False,
mask_seqlen=False,
sink_length=sink_length,
follows_pv=True,
)
if const_expr(peel_last):
if n_block_max - 1 > n_block_min:
mma_si_consumer_phase, si_corr_producer_phase, red_buf = softmax_step(
mma_si_consumer_phase,
si_corr_producer_phase,
red_buf,
n_block_min,
seqlen_k=seqlen.seqlen_k,
is_first=False,
mask_seqlen=False,
sink_length=sink_length,
follows_pv=True,
mask_sink=True,
)
if const_expr(runs_sink):
(
mma_si_consumer_phase,
si_corr_producer_phase,
red_buf,
) = self.softmax_sink_step_transposed(
mma_si_consumer_phase=mma_si_consumer_phase,
si_corr_producer_phase=si_corr_producer_phase,
red_buf=red_buf,
softmax_scale_log2=softmax_scale_log2,
mbar_ptr=mbar_ptr,
thr_tmem_load=thr_tmem_load,
tStS_t2r=tStS_t2r,
tStS_sink_t2r=tStS_sink_t2r,
frg_shape=frg_shape,
row_max=row_max,
row_sum=row_sum,
sScale=sScale,
sP_bf16=sP_bf16,
sP_bf16_mk=sP_bf16_mk,
sSoftmaxRed=sSoftmaxRed,
stage=stage,
tidx=tidx,
batch_idx=batch_idx,
mSinkSeqUsedK=mSinkSeqUsedK,
split_idx=split_idx,
row_group=row_group,
waits_for_previous=runs_pages,
)
red_buf = self.softmax_warpgroup_reduce(
row_sum,
sSoftmaxRed,
tidx,
red_buf,
lambda a, b: a + b,
row_group=row_group,
)
if const_expr(runs_pages or runs_sink):
si_corr_producer_phase = self.softmax_wait_corr(
mbar_ptr, stage, si_corr_producer_phase
)
if tidx == 0:
for j in cutlass.range_constexpr(rows):
sScale[row_base + j + stage * self.m_block_size] = row_sum[j]
if const_expr(mLSE is not None or learnable_sink is not None):
for j in cutlass.range_constexpr(rows):
sScale[
row_base + j + stage * self.m_block_size + self.m_block_size * 2
] = row_max[j]
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_softmax_corr_full_offset + stage
)
tile_scheduler.advance_to_next_work()
work_tile = tile_scheduler.get_current_work()
@cute.jit
def softmax_step_transposed(
self,
mma_si_consumer_phase: Int32,
si_corr_producer_phase: Int32,
red_buf: Int32,
n_block: Int32,
softmax_scale_log2: Float32,
mbar_ptr: cute.Pointer,
thr_tmem_load: cute.CopyAtom,
tStS_t2r: cute.Tensor,
tStS_second_t2r: Optional[cute.Tensor],
frg_shape,
row_max: cute.Tensor,
row_sum: cute.Tensor,
sScale: cute.Tensor,
sSFP: cute.Tensor,
sP_uint8: cute.Tensor,
sSoftmaxRed: cute.Tensor,
stage: int | Int32,
tidx: Int32,
seqlen_k: Int32,
is_first: cutlass.Constexpr[bool],
mask_seqlen: cutlass.Constexpr[bool],
sink_length: Int32,
row_group: cutlass.Constexpr[int] = 0,
follows_pv: cutlass.Constexpr[bool] = False,
mask_sink: cutlass.Constexpr[bool] = False,
waits_for_previous: cutlass.Constexpr[bool] = True,
) -> Tuple[Int32, Int32, Int32]:
"""One KV block of the transposed softmax.
The tensor-memory load gives thread ``tidx`` kv position ``tidx``, which
is the invariant the whole step rests on: masking is one predicate, a
scale group of 16 kv positions is 16 lanes of one warp, and the two FP4
nibbles that share a byte sit in neighbouring lanes.
``follows_pv`` is every block of a tile but its first: the previous
block's PV may still be reading the P tile this step rewrites.
``mask_sink`` is a block that may be page zero. The test sits on the
softmax's critical path, which paces the whole loop, so the steps that
cannot reach page zero leave it out.
``waits_for_previous`` is every step but the tile's first to publish:
correction must have taken the previous step's factors from sScale
before this step overwrites them. The wait sits just before the
overwrite rather than at the end of the step that published, so
correction has this step's row maximum to wake in.
"""
rows = const_expr(self.softmax_rows_per_group)
row_base = const_expr(row_group * rows)
bytes_per_row = const_expr(self.n_block_size // 2)
atom_k_bytes = const_expr(self.n_block_size // 4)
iket.range_push("sm_wait_s")
cute.arch.mbarrier_wait(mbar_ptr + self.mbar_S_full_offset + stage, mma_si_consumer_phase)
iket.range_pop()
tSrS = cute.make_rmem_tensor(frg_shape, self.qk_acc_dtype)
cute.copy(thr_tmem_load, tStS_t2r, tSrS)
if const_expr(self.stacked_query_terms):
tSrS_second = cute.make_rmem_tensor(frg_shape, self.qk_acc_dtype)
cute.copy(thr_tmem_load, tStS_second_t2r, tSrS_second)
cute.arch.fence_view_async_tmem_load()
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_sfqk_load_offset + (self.q_stage - 1 - stage)
)
if const_expr(self.stacked_query_terms):
for j in cutlass.range_constexpr(rows):
tSrS[j] = tSrS[j] + tSrS_second[j]
if const_expr(mask_seqlen):
# A row whose fp4 length is zero still runs one step, at block -1.
# Folding that back onto block 0 puts the limit at the start of the
# sequence, so every position is masked, as the untransposed mask
# does for the same edge.
n_block_masked = n_block
if n_block_masked < 0:
n_block_masked = Int32(0)
if n_block_masked * self.n_block_size + tidx >= seqlen_k:
for j in cutlass.range_constexpr(rows):
tSrS[j] = -Float32.inf
if const_expr(self.fused_sink_block and mask_sink):
# Page zero retains a quantized duplicate of the permanent BF16
# sink so that every later page keeps its original vLLM boundary.
# Suppress only that prefix; all other FP4 positions are unchanged.
if (n_block == Int32(0)) and (tidx < sink_length):
for j in cutlass.range_constexpr(rows):
tSrS[j] = -Float32.inf
iket.range_push("sm_rowmax")
# A scale group is sf_vec_size consecutive kv positions, hence that many
# consecutive lanes, so the butterfly passes through the group maximum
# on its way to the warp maximum. That intermediate is what the FP4
# packing below needs, because exp2 is monotonic: the largest
# exponential of a group is the exponential of the group's largest
# score. Carrying it out costs one exp2 per row and removes a second
# butterfly of the same depth.
group_steps = const_expr(self.sf_vec_size.bit_length() - 1)
warp_steps = const_expr(cute.arch.WARP_SIZE.bit_length() - 1)
block_max = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
group_max_score = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
scatter = const_expr(self.softmax_scatters_rows)
# Past the first block a row's maximum only moves when it grows by
# more than the rescale threshold, which once it settles it almost
# never does. Each thread applies the skip test below to its own
# scores and one barrier ORs the verdicts. The test is monotonic in
# the score, so it passes on the row maximum exactly when it passes on
# every score of the row, and the cross-warp maximum is reduced only
# when some row will take it.
votes = const_expr(not is_first and RESCALE_THRESHOLD > 0.0 and not self.p_scale_two_level)
grows = cutlass.Boolean(True)
if const_expr(votes):
own_grows = Int32(0)
for j in cutlass.range_constexpr(rows):
grown = (
row_max[j] - cute.arch.fmax(row_max[j], tSrS[j])
) * softmax_scale_log2
own_grows = Int32(
cutlass.select_(grown >= -RESCALE_THRESHOLD, own_grows, Int32(1))
)
# Every thread gets the same verdict back; voting on it tells the
# compiler so, which keeps divergence handling out of the branches
# it decides.
grows = cute.arch.vote_any_sync(
utils.barrier_or(
own_grows,
barrier_id=int(NamedBarrierFwd.SoftmaxReduce) + row_group,
number_of_threads=len(self.softmax0_warp_ids) * cute.arch.WARP_SIZE,
)
!= Int32(0)
)
if const_expr(scatter):
# Reduce-scatter: each of the first log2(rows) butterfly steps hands
# half of the rows still held to the partner lane and keeps the
# other half, so a lane ends with one row, the one ``scatter_row``
# names, and the steps cost rows - 1 shuffles in all rather than
# one per row each. The rest of the butterfly then runs on that
# row alone, through its group maximum to its warp maximum.
scatter_steps = const_expr(rows.bit_length() - 1)
held = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
for j in cutlass.range_constexpr(rows):
held[j] = tSrS[j]
for i in cutlass.range_constexpr(scatter_steps):
half = const_expr(rows >> (i + 1))
upper = ((tidx >> i) & 1) == 1
for k in cutlass.range_constexpr(half):
send = Float32(cutlass.select_(upper, held[k], held[k + half]))
keep = Float32(cutlass.select_(upper, held[k + half], held[k]))
held[k] = cute.arch.fmax(
keep, cute.arch.shuffle_sync_bfly(send, offset=1 << i)
)
val = held[0]
for i in cutlass.range_constexpr(scatter_steps, group_steps):
val = cute.arch.fmax(val, cute.arch.shuffle_sync_bfly(val, offset=1 << i))
own_group_max_score = val
own_row = self.scatter_row(tidx)
if grows:
for i in cutlass.range_constexpr(group_steps, warp_steps):
val = cute.arch.fmax(
val, cute.arch.shuffle_sync_bfly(val, offset=1 << i)
)
red_buf = self.softmax_warpgroup_reduce_scattered(
val,
own_row,
block_max,
sSoftmaxRed,
tidx,
red_buf,
row_group=row_group,
)
else:
for j in cutlass.range_constexpr(rows):
val = tSrS[j]
for i in cutlass.range_constexpr(group_steps):
val = cute.arch.fmax(val, cute.arch.shuffle_sync_bfly(val, offset=1 << i))
group_max_score[j] = val
block_max[j] = val
if grows:
for j in cutlass.range_constexpr(rows):
val = block_max[j]
for i in cutlass.range_constexpr(group_steps, warp_steps):
val = cute.arch.fmax(
val, cute.arch.shuffle_sync_bfly(val, offset=1 << i)
)
block_max[j] = val
red_buf = self.softmax_warpgroup_reduce(
block_max,
sSoftmaxRed,
tidx,
red_buf,
cute.arch.fmax,
intra_warp=False,
row_group=row_group,
)
acc_scale = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
row_max_safe = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
# Once the maximum settles every row skips its rescale, and the skip
# needs only the vote: the factor is 1 and the maximum stays. A block
# where some row grows takes the full update below, which applies the
# same test row by row against the reduced maximum.
if not grows:
for j in cutlass.range_constexpr(rows):
row_max_safe[j] = row_max[j]
acc_scale[j] = Float32(1.0)
else:
for j in cutlass.range_constexpr(rows):
if const_expr(is_first):
row_max_new = block_max[j]
row_max_safe[j] = row_max_new if row_max_new != -Float32.inf else 0.0
acc_scale[j] = 0.0
else:
row_max_old = row_max[j]
row_max_new = cute.arch.fmax(row_max_old, block_max[j])
safe = row_max_new if row_max_new != -Float32.inf else 0.0
acc_scale_ = (row_max_old - safe) * softmax_scale_log2
# The skip decision is made on a maximum every thread agrees
# on, so it stays uniform across the warpgroup, and a
# skipped rescale costs no exponential.
scale = Float32(1.0)
if const_expr(RESCALE_THRESHOLD > 0.0):
if acc_scale_ >= -RESCALE_THRESHOLD:
row_max_new = row_max_old
safe = row_max_old
else:
scale = utils.exp2f(acc_scale_)
else:
scale = utils.exp2f(acc_scale_)
row_max_safe[j] = safe
acc_scale[j] = scale
row_max[j] = row_max_new
# The shift is the same for every score of a row, so scaling it once
# leaves each exponential a single fused multiply-add. A reduce-
# scattered tile holds an even number of rows, so its arithmetic runs
# on pairs of rows in the packed FP32x2 units; one rounding per lane,
# as the scalar instructions round.
max_scaled = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
neg_max_scaled = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
if const_expr(scatter):
for j in cutlass.range_constexpr(0, rows, 2):
max_scaled[j], max_scaled[j + 1] = utils.mul_packed_f32x2(
(row_max_safe[j], row_max_safe[j + 1]),
(softmax_scale_log2, softmax_scale_log2),
)
neg_max_scaled[j], neg_max_scaled[j + 1] = utils.mul_packed_f32x2(
(row_max_safe[j], row_max_safe[j + 1]),
(-softmax_scale_log2, -softmax_scale_log2),
)
else:
for j in cutlass.range_constexpr(rows):
max_scaled[j] = row_max_safe[j] * softmax_scale_log2
if const_expr(self.p_scale_rescale_log2):
for j in cutlass.range_constexpr(rows):
max_scaled[j] -= Float32(self.p_scale_rescale_log2)
if const_expr(scatter):
neg_max_scaled[j] += Float32(self.p_scale_rescale_log2)
iket.range_pop()
if const_expr(waits_for_previous):
si_corr_producer_phase = self.softmax_wait_corr(
mbar_ptr, stage, si_corr_producer_phase
)
# Every thread holds the same correction factors, so one publishes them.
if const_expr(not is_first):
if tidx == 0:
for j in cutlass.range_constexpr(rows):
sScale[row_base + j + stage * self.m_block_size] = acc_scale[j]
if const_expr(self.p_scale_two_level):
if tidx == 0:
for j in cutlass.range_constexpr(rows):
outer = cute.arch.exp2(
block_max[j] * softmax_scale_log2 - max_scaled[j]
- Float32(math.log2(448.0 * 6.0))
)
sScale[row_base + j + stage * self.m_block_size + self.m_block_size * 2] = outer
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_full_offset + stage)
amplified = None
if const_expr(self.p_scale_two_level):
amplified = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
for j in cutlass.range_constexpr(rows):
peak = Float32(cutlass.select_(block_max[j] == -Float32.inf, 0.0, block_max[j]))
amplified[j] = cute.arch.exp2(
(tSrS[j] - peak) * softmax_scale_log2
+ Float32(math.log2(448.0 * 6.0))
)
iket.range_push("sm_exp")
if const_expr(scatter):
for j in cutlass.range_constexpr(0, rows, 2):
arg0, arg1 = utils.fma_packed_f32x2(
(tSrS[j], tSrS[j + 1]),
(softmax_scale_log2, softmax_scale_log2),
(neg_max_scaled[j], neg_max_scaled[j + 1]),
)
tSrS[j] = cute.arch.exp2(arg0)
tSrS[j + 1] = cute.arch.exp2(arg1)
if const_expr(is_first):
row_sum[j] = tSrS[j]
row_sum[j + 1] = tSrS[j + 1]
else:
row_sum[j], row_sum[j + 1] = utils.fma_packed_f32x2(
(row_sum[j], row_sum[j + 1]),
(acc_scale[j], acc_scale[j + 1]),
(tSrS[j], tSrS[j + 1]),
)
else:
for j in cutlass.range_constexpr(rows):
tSrS[j] = cute.arch.exp2(tSrS[j] * softmax_scale_log2 - max_scaled[j])
if const_expr(is_first):
row_sum[j] = tSrS[j]
else:
row_sum[j] = row_sum[j] * acc_scale[j] + tSrS[j]
iket.range_pop()
if const_expr(self.p_scale_two_level):
for j in cutlass.range_constexpr(rows):
peak = Float32(cutlass.select_(block_max[j] == -Float32.inf, 0.0, block_max[j]))
max_scaled[j] = peak * softmax_scale_log2 - Float32(math.log2(448.0 * 6.0))
tSrS[j] = amplified[j]
iket.range_push("sm_pquant")
group_max = cute.make_rmem_tensor(cute.make_layout(rows), Float32)
group_exponent = cute.make_rmem_tensor(cute.make_layout(rows), Int32)
own_scale = Float32(0.0)
own_exponent = Int32(0)
if const_expr(scatter):
# Every lane of a scale group would compute the same scale for a
# row, so the lane that holds the row's group maximum computes it
# once and the group's lanes fetch its reciprocal from there.
own_max_scaled = max_scaled[0]
for j in cutlass.range_constexpr(1, rows):
own_max_scaled = Float32(
cutlass.select_(own_row == j, max_scaled[j], own_max_scaled)
)
maximum = cute.arch.exp2(
own_group_max_score * softmax_scale_log2 - own_max_scaled
)
if const_expr(self.is_mxfp4):
own_exponent = round_up_exponent(maximum)
else:
if const_expr(self.p_scale_amax2_or_amax6):
own_scale = _amax2_or_amax6_probability_scale(maximum)
else:
own_scale = maximum * Float32(1.0 / self.p_scale_divisor)
own_inverse = cute.arch.rcp_approx(cute.arch.fmax(own_scale, 1e-20))
cluster = tidx & (cute.arch.WARP_SIZE - rows)
if const_expr(self.is_mxfp4):
# Each row still has one P value in every lane. Preserve the
# scattered maximum/exponential arithmetic, but reduce each
# candidate's error over all 32 values of that row's group.
for j in cutlass.range_constexpr(rows):
exponent = Int32(cute.arch.shuffle_sync(
own_exponent, cluster | self.scatter_lane(j)
))
group_exponent[j] = _mxfp4_probability_exponent(tSrS[j], exponent)
tSrS[j] = tSrS[j] * inverse_power_of_two(group_exponent[j])
# Do not overwrite any scatter owner until all original
# exponents have been fetched by every row above.
for j in cutlass.range_constexpr(rows):
own_exponent = Int32(cutlass.select_(
own_row == j, group_exponent[j], own_exponent
))
else:
for j in cutlass.range_constexpr(0, rows, 2):
tSrS[j], tSrS[j + 1] = utils.mul_packed_f32x2(
(tSrS[j], tSrS[j + 1]),
(
cute.arch.shuffle_sync(own_inverse, cluster | self.scatter_lane(j)),
cute.arch.shuffle_sync(
own_inverse, cluster | self.scatter_lane(j + 1)
),
),
)
for j in cutlass.range_constexpr(0 if scatter else rows):
# Same expression as the exponentials above, so this is the value
# the largest lane of the group holds, to the bit.
maximum = cute.arch.exp2(
group_max_score[j] * softmax_scale_log2 - max_scaled[j]
)
if const_expr(self.is_mxfp4):
group_exponent[j] = _mxfp4_probability_exponent(
tSrS[j], round_up_exponent(maximum)
)
elif const_expr(self.p_scale_amax2_or_amax6):
group_max[j] = _amax2_or_amax6_probability_scale(maximum)
elif const_expr(self.p_scale_grid_search):
best_scale = maximum * Float32(1.0 / 6.0)
best_stored = _round_probability_scale(best_scale)
best_code = _round_probability_code(
tSrS[j]
* cute.arch.rcp_approx(
cute.arch.fmax(best_scale, 1e-20)
)
)
difference = best_code * best_stored - tSrS[j]
best_error = difference * difference
for i in cutlass.range_constexpr(group_steps):
best_error += cute.arch.shuffle_sync_bfly(
best_error, offset=1 << i
)
rival_divisors = (4.0, 3.0, 2.0)
for candidate in cutlass.range_constexpr(
len(rival_divisors)
):
divisor = rival_divisors[candidate]
rival_scale = maximum * Float32(1.0 / divisor)
rival_stored = _round_probability_scale(rival_scale)
rival_code = _round_probability_code(
tSrS[j]
* cute.arch.rcp_approx(
cute.arch.fmax(rival_scale, 1e-20)
)
)
difference = rival_code * rival_stored - tSrS[j]
rival_error = difference * difference
for i in cutlass.range_constexpr(group_steps):
rival_error += cute.arch.shuffle_sync_bfly(
rival_error, offset=1 << i
)
better = rival_error < best_error
best_scale = Float32(
cutlass.select_(better, rival_scale, best_scale)
)
best_error = Float32(
cutlass.select_(better, rival_error, best_error)
)
group_max[j] = best_scale
else:
group_max[j] = maximum * Float32(
1.0 / self.p_scale_divisor
)
for j in cutlass.range_constexpr(0 if scatter else rows):
# An approximate reciprocal is one instruction where the exact
# division is a sequence, and P carries three mantissa bits.
if const_expr(self.is_mxfp4):
tSrS[j] = tSrS[j] * inverse_power_of_two(group_exponent[j])
else:
tSrS[j] = tSrS[j] * cute.arch.rcp_approx(cute.arch.fmax(group_max[j], 1e-20))
# The S this step read was signalled before the previous block's PV
# was even issued, so that PV is waited for here, as late as possible.
# It completes once per block and runs at most one block behind, so
# the S phase of the block before names it.
if const_expr(follows_pv):
iket.range_push("sm_wait_pv")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_P_consumed_offset + stage,
mma_si_consumer_phase ^ 1,
)
iket.range_pop()
# K-major P wants kv contiguous, so the nibbles for kv = tidx and
# tidx + 1 share a byte. One butterfly per query row brings the odd
# lane's value over and the even lane stores the merged byte.
k_byte = tidx // 2
for j in cutlass.range_constexpr(rows):
row = row_base + j
partner = cute.arch.shuffle_sync_bfly(tSrS[j], offset=1)
if tidx % 2 == 0:
packed = sm100_utils.packed_float_to_e2m1x2(partner, tSrS[j])
sP_uint8[(row, k_byte % atom_k_bytes), 0, k_byte // atom_k_bytes, 0] = (
cutlass.Uint8(packed & 0xFF)
)
sSFP_u8 = cute.recast_tensor(sSFP[None, None, None, stage], cutlass.Uint8)
if const_expr(scatter):
# The first lanes of a group hold one row each, so each stores the
# scale of its own row.
if tidx % self.sf_vec_size < rows:
k_group = tidx // self.sf_vec_size
row = row_base + own_row
if const_expr(self.is_mxfp4):
sf_byte = own_exponent + Int32(127)
else:
sf_byte = packed_float_to_ue4m3(
own_scale, Float32(0.0), Float32(0.0), Float32(0.0)
)
sSFP_u8[
(((row % 32, row // 32), 0),
(0, k_group % self.sf_groups_per_mma_k)),
0, k_group // self.sf_groups_per_mma_k
] = cutlass.Uint8(sf_byte & 0xFF)
elif tidx % self.sf_vec_size == 0:
k_group = tidx // self.sf_vec_size
for j in cutlass.range_constexpr(rows):
row = row_base + j
if const_expr(self.is_mxfp4):
sf_byte = group_exponent[j] + Int32(127)
else:
sf_byte = packed_float_to_ue4m3(
group_max[j], Float32(0.0), Float32(0.0), Float32(0.0)
)
sSFP_u8[
(((row % 32, row // 32), 0),
(0, k_group % self.sf_groups_per_mma_k)),
0, k_group // self.sf_groups_per_mma_k
] = cutlass.Uint8(sf_byte & 0xFF)
# The MMA reads P and its scale factors through the async proxy while
# these are generic-proxy stores; the mbarrier below only orders generic
# traffic, so the fence is what makes them visible.
cute.arch.fence_view_async_shared()
iket.range_pop()
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage)
return mma_si_consumer_phase ^ 1, si_corr_producer_phase, red_buf
# for both softmax0 and softmax1 warp group
@cute.jit
def softmax_loop(
self,
stage: int | Int32,
softmax_scale_log2: Float32,
softmax_scale: Float32,
thr_mma_qk: cute.ThrMma,
tStSi: cute.Tensor,
sScale: cute.Tensor,
mLSE: Optional[cute.Tensor],
learnable_sink: Optional[cute.Tensor],
mbar_ptr: cute.Pointer,
block_info: BlockInfo,
num_splits: Int32,
SeqlenInfoCls: Callable,
AttentionMaskCls: Callable,
TileSchedulerCls: Callable,
aux_tensors: Optional[list] = None,
fastdiv_mods=(None, None),
blocksparse_tensors: Optional[BlockSparseTensors] = None,
tCtSFP: Optional[Tuple[cute.Tensor, ...]] = None,
sSFP: Optional[cute.Tensor] = None,
mResidualSeqUsedK: Optional[cute.Tensor] = None,
mSinkSeqUsedK: Optional[cute.Tensor] = None,
sP_uint8: Optional[cute.Tensor] = None,
sSoftmaxRed: Optional[cute.Tensor] = None,
sP_bf16: Optional[cute.Tensor] = None,
sP_bf16_mk: Optional[cute.Tensor] = None,
row_group: cutlass.Constexpr[int] = 0,
role: cutlass.Constexpr = SplitRole.ALL,
):
"""Compute softmax on attention scores from QK matrix multiplication.
This method handles the softmax computation for either the first or second half of the
attention matrix, depending on the 'stage' parameter. It calculates row-wise maximum
and sum values needed for stable softmax computation, applies optional masking, and
transforms raw attention scores into probability distributions.
The implementation uses specialized memory access patterns and efficient math operations
for computing exp(x) using exp2 functions. It also coordinates pipeline
synchronization between MMA, correction, and sequence processing stages.
"""
tidx = cute.arch.thread_idx()[0] % (
cute.arch.WARP_SIZE
# * (len(self.softmax0_warp_ids) if stage == 0 else len(self.softmax1_warp_ids)
* (len(self.softmax0_warp_ids))
)
tStScale = cute.composition(tStSi, cute.make_layout((self.m_block_size, 1)))
tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2]))
tScScale = cute.composition(tScS, cute.make_layout((self.m_block_size, 1)))
tilePlikeFP32 = self.mma_tiler_qk[1] // 32 * self.v_dtype.width
tStP_layout = cute.composition(
tStSi.layout, cute.make_layout((self.m_block_size, tilePlikeFP32))
)
tStP = cute.make_tensor(tStSi.iterator + self.tmem_s_to_p_offset, tStP_layout)
tmem_load_atom = cute.make_copy_atom(
tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)),
Float32,
)
thr_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tStSi).get_slice(tidx)
tStS_t2r = thr_tmem_load.partition_S(tStSi)
tmem_store_scale_atom = cute.make_copy_atom(
tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(1)),
Float32,
)
thr_tmem_store_scale = tcgen05.make_tmem_copy(tmem_store_scale_atom, tStScale).get_slice(tidx)
tStScale_r2t = thr_tmem_store_scale.partition_D(tStScale)
tmem_store_atom = cute.make_copy_atom(
tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(16 if const_expr(not self.quant_pv) else 8)),
Float32,
)
thr_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tStP).get_slice(tidx)
tStP_r2t = thr_tmem_store.partition_D(tStP)
if const_expr(self.transpose_s):
self.softmax_loop_transposed(
stage=stage,
softmax_scale_log2=softmax_scale_log2,
thr_mma_qk=thr_mma_qk,
tStSi=tStSi,
sScale=sScale,
mLSE=mLSE,
learnable_sink=learnable_sink,
mbar_ptr=mbar_ptr,
block_info=block_info,
num_splits=num_splits,
SeqlenInfoCls=SeqlenInfoCls,
TileSchedulerCls=TileSchedulerCls,
sSFP=sSFP,
sP_uint8=sP_uint8,
sSoftmaxRed=sSoftmaxRed,
tidx=tidx,
row_group=row_group,
mResidualSeqUsedK=mResidualSeqUsedK,
mSinkSeqUsedK=mSinkSeqUsedK,
sP_bf16=sP_bf16,
sP_bf16_mk=sP_bf16_mk,
role=role,
)
return
thr_tmem_store_bf16 = thr_tmem_store
tStP_bf16_r2t = tStP_r2t
if const_expr(self.fused_residual_first_block):
tilePlikeFP32_bf16 = self.mma_tiler_qk[1] // 32 * 16 # BF16=16 bits
tStP_bf16_layout = cute.composition(
tStSi.layout, cute.make_layout((self.m_block_size, tilePlikeFP32_bf16))
)
tStP_bf16 = cute.make_tensor(
tStSi.iterator + self.tmem_s_to_p_offset, tStP_bf16_layout
)
tmem_store_bf16_atom = cute.make_copy_atom(
tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(16)),
Float32,
)
thr_tmem_store_bf16 = tcgen05.make_tmem_copy(
tmem_store_bf16_atom, tStP_bf16
).get_slice(tidx)
tStP_bf16_r2t = thr_tmem_store_bf16.partition_D(tStP_bf16)
mma_si_consumer_phase = Int32(0)
si_corr_producer_phase = Int32(1)
s0_s1_sequence_phase = Int32(1 if stage == 0 else 0)
if stage == 1 and const_expr(self.quant_qk):
sfqk_stage = 0
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset + sfqk_stage)
if const_expr(self.q_stage == 1) and stage == 0 and const_expr(self.quant_qk):
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset + 0)
# self.warp_scheduler_barrier_init()
warp_idx_in_wg = cute.arch.make_warp_uniform(cute.arch.warp_idx()) % 4
mbar_s0_s1_sequence_offset = self.mbar_s0_s1_sequence_offset + warp_idx_in_wg
tile_scheduler = TileSchedulerCls()
work_tile = tile_scheduler.initial_work_tile_info()
while work_tile.is_valid_tile:
m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx
seqlen = SeqlenInfoCls(batch_idx)
n_block_min, n_block_max = self.split_block_range(
block_info, seqlen, m_block, split_idx, num_splits
)
mask = AttentionMaskCls(seqlen.seqlen_q, seqlen.seqlen_k)
shared_mask_kwargs = dict(
m_block=self.q_stage * m_block + stage,
thr_mma=thr_mma_qk,
thr_tmem_load=thr_tmem_load,
mask_causal=self.is_causal,
mask_local=self.is_local,
batch_idx=batch_idx,
head_idx=head_idx,
aux_tensors=aux_tensors,
)
block_mask_mod = self.mask_mod if const_expr(self.use_block_sparsity) else None
mask_fn = partial(
mask.apply_mask_sm100,
mask_mod=block_mask_mod,
**shared_mask_kwargs,
)
if const_expr(self.use_block_sparsity):
# Full blocks dont need mask_mod
mask_fn_none = partial(
mask.apply_mask_sm100,
mask_mod=None,
**shared_mask_kwargs,
)
else:
mask_fn_none = None
softmax = SoftmaxSm100.create(
softmax_scale_log2,
rescale_threshold=RESCALE_THRESHOLD,
# rescale_threshold=8.0 if const_expr(self.q_dtype.width == 16) else 0.0, # (Wenxuan) disable skipping rescale until FP4 precision is verified
softmax_scale=softmax_scale,
quant_pv=self.quant_pv,
compute_sp1=self.compute_sp1,
kahan=self.fused_residual_first_block,
p_scale_divisor=self.p_scale_divisor,
p_scale_grid_search=self.p_scale_grid_search,
p_scale_amax2_or_amax6=self.p_scale_amax2_or_amax6,
)
softmax.reset()
if const_expr(self.use_block_sparsity):
tile_block_count = get_total_block_count(blocksparse_tensors, batch_idx, head_idx, m_block)
has_work = tile_block_count > Int32(0)
else:
tile_block_count = n_block_max - n_block_min
has_work = self.tile_has_work(
n_block_min, n_block_max, split_idx, num_splits
)
softmax_step = partial(
self.softmax_step,
softmax=softmax,
mbar_ptr=mbar_ptr,
mbar_s0_s1_sequence_offset=mbar_s0_s1_sequence_offset,
thr_mma_qk=thr_mma_qk,
thr_tmem_load=thr_tmem_load,
thr_tmem_store=thr_tmem_store,
thr_tmem_store_scale=thr_tmem_store_scale,
tStS_t2r=tStS_t2r,
tStScale_r2t=tStScale_r2t,
tStP_r2t=tStP_r2t,
sScale=sScale,
stage=stage,
batch_idx=batch_idx,
head_idx=head_idx,
m_block=self.q_stage * m_block + stage,
seqlen=seqlen,
aux_tensors=aux_tensors,
fastdiv_mods=fastdiv_mods,
mask_fn=partial(mask_fn, mask_seqlen=False),
tCtSFP=tCtSFP,
sSFP=sSFP,
)
if has_work:
# Softmax acts as the producer: wait until correction signals the stage is empty
iket.range_push("sm_wait_corr")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_softmax_corr_empty_offset + stage, si_corr_producer_phase
)
iket.range_pop()
si_corr_producer_phase ^= 1
if const_expr(self.fused_residual_first_block) and stage == 0 and has_work:
mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = (
self.softmax_residual_step(
mma_si_consumer_phase,
si_corr_producer_phase,
s0_s1_sequence_phase,
softmax=softmax,
mbar_ptr=mbar_ptr,
thr_mma_qk=thr_mma_qk,
thr_tmem_load=thr_tmem_load,
thr_tmem_store_bf16=thr_tmem_store_bf16,
thr_tmem_store_scale=thr_tmem_store_scale,
tStS_t2r=tStS_t2r,
tStP_bf16_r2t=tStP_bf16_r2t,
sScale=sScale,
stage=stage,
batch_idx=batch_idx,
m_block=self.q_stage * m_block + stage,
mResidualSeqUsedK=mResidualSeqUsedK,
thread_idx=tidx,
split_idx=split_idx,
)
)
# Block sparse or dense iteration
if const_expr(self.use_block_sparsity):
(
mma_si_consumer_phase,
si_corr_producer_phase,
s0_s1_sequence_phase,
empty_tile,
) = softmax_block_sparse_sm100(
blocksparse_tensors,
batch_idx,
head_idx,
m_block,
softmax_step,
mask_fn,
mask_fn_none,
mma_si_consumer_phase,
si_corr_producer_phase,
s0_s1_sequence_phase,
mbar_ptr,
self.mbar_softmax_corr_full_offset,
self.mbar_softmax_corr_empty_offset,
self.mbar_P_full_O_rescaled_offset,
self.mbar_P_full_2_offset,
self.q_stage,
Int32(stage),
)
if not empty_tile:
if const_expr(self.fused_residual_first_block):
sScale[tidx + stage * self.m_block_size] = (
softmax.row_sum[0] + softmax.row_sum_comp[0]
)
else:
sScale[tidx + stage * self.m_block_size] = softmax.row_sum[0]
if const_expr(mLSE is not None or learnable_sink is not None):
sScale[
tidx + stage * self.m_block_size + self.m_block_size * 2
] = softmax.row_max[0]
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_full_offset + stage)
else:
if has_work:
if const_expr(self.fused_residual_first_block) and stage == 0:
mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = softmax_step(
mma_si_consumer_phase,
si_corr_producer_phase,
s0_s1_sequence_phase,
n_block_max - 1,
is_first=False,
mask_fn=partial(mask_fn, mask_seqlen=True),
)
else:
mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = softmax_step(
mma_si_consumer_phase,
si_corr_producer_phase,
s0_s1_sequence_phase,
n_block_max - 1,
is_first=True,
mask_fn=partial(mask_fn, mask_seqlen=True),
)
n_block_max -= 1
# Next couple of iterations with causal masking
if const_expr(self.is_causal or self.is_local):
n_block_min_causal_local_mask = block_info.get_n_block_min_causal_local_mask(
seqlen, m_block, n_block_min
)
for n_tile in cutlass.range(n_block_max - n_block_min_causal_local_mask, unroll=1):
n_block = n_block_max - 1 - n_tile
mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = (
softmax_step(
mma_si_consumer_phase,
si_corr_producer_phase,
s0_s1_sequence_phase,
n_block,
mask_fn=partial(mask_fn, mask_seqlen=False),
)
)
n_block_max = cutlass.min(n_block_max, n_block_min_causal_local_mask)
# The remaining iterations have no masking
n_block_min_before_local_mask = block_info.get_n_block_min_before_local_mask(
seqlen, m_block, n_block_min
)
for n_tile in cutlass.range(n_block_max - n_block_min_before_local_mask, unroll=1):
n_block = n_block_max - n_tile - 1
mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = softmax_step(
mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase, n_block
)
# Separate iterations with local masking on the left
if const_expr(self.is_local and block_info.window_size_left is not None):
n_block_max = cutlass.min(n_block_max, n_block_min_before_local_mask)
for n_tile in cutlass.range(0, n_block_max - n_block_min, unroll=1):
n_block = n_block_max - 1 - n_tile
mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase = (
softmax_step(
mma_si_consumer_phase,
si_corr_producer_phase,
s0_s1_sequence_phase,
n_block,
mask_fn=partial(mask_fn, mask_seqlen=False),
)
)
# Now that we no longer already have the 1st iteration, need mask_seqlen=True here
if const_expr(self.fused_residual_first_block):
sScale[tidx + stage * self.m_block_size] = (
softmax.row_sum[0] + softmax.row_sum_comp[0]
)
else:
sScale[tidx + stage * self.m_block_size] = softmax.row_sum[0]
if const_expr(mLSE is not None or learnable_sink is not None):
sScale[
tidx + stage * self.m_block_size + self.m_block_size * 2
] = softmax.row_max[0]
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_full_offset + stage)
# # Write LSE to gmem
# if const_expr(mLSE is not None):
# acc_O_mn_row_is_zero_or_nan = softmax.row_sum[0] == 0.0 or softmax.row_sum[0] != softmax.row_sum[0]
# scale = (
# cute.arch.rcp_approx(softmax.row_sum[0] if not acc_O_mn_row_is_zero_or_nan else 1.0)
# )
# LN2 = math.log(2.0)
# lse = (
# (softmax.row_max[0] * softmax.scale_log2 + utils.log2f(softmax.row_sum[0])) * LN2
# if not acc_O_mn_row_is_zero_or_nan else -Float32.inf
# )
# if const_expr(not seqlen.has_cu_seqlens_q):
# mLSE_cur = mLSE[None, head_idx, batch_idx]
# else:
# mLSE_cur = cute.domain_offset((seqlen.offset_q,), mLSE[None, head_idx])
# gLSE = cute.local_tile(mLSE_cur, (self.m_block_size,), (m_block * 2 + stage,))
# if tidx < seqlen.seqlen_q - (m_block * 2 + stage) * self.m_block_size:
# gLSE[tidx] = lse
# Advance to next tile
tile_scheduler.advance_to_next_work()
work_tile = tile_scheduler.get_current_work()
# End of persistent scheduler loop
@cute.jit
def _quant_fp4(self,
# src
tSrP_f32: cute.Tensor,
tSrPSF_f32: cute.Tensor,
# dst
tSrP: cute.Tensor,
tSrPSF: cute.Tensor,
):
tSrP_f32_frag = cute.logical_divide(tSrP_f32, cute.make_layout(self.sf_vec_size))
assert cute.size(tSrP_f32_frag, mode=[1]) == cute.size(tSrPSF_f32)
tSrP_frag = cute.logical_divide(tSrP, cute.make_layout(self.sf_vec_size))
tSrPSF_u32_view = cute.recast_tensor(tSrPSF, cute.Int32)
# Process in groups of 4 for UE4M3 conversion
assert cute.size(tSrPSF_f32) % 4 == 0
for i in cutlass.range_constexpr(0, cute.size(tSrPSF_f32) // 4, unroll=1):
# for i in cutlass.range_constexpr(0, 2):
# Pack 4 FP32 values into UE4M3 format
packed_ue4m3 = packed_float_to_ue4m3(
tSrPSF_f32[i * 4],
tSrPSF_f32[i * 4 + 1],
tSrPSF_f32[i * 4 + 2],
tSrPSF_f32[i * 4 + 3]
)
tSrPSF_u32_view[i] = packed_ue4m3
# Quantize main tensor to E2M1 format (8 values per uint32_t)
# Process in groups of 8 for E2M1 conversion
for i in cutlass.range_constexpr(0, cute.size(tSrP_frag, mode=[1])):
tSrP_u32_view = cute.recast_tensor(tSrP_frag[None, i], cute.Int32)
for k in cutlass.range_constexpr(0, cute.size(tSrP_u32_view, mode=[0])):
packed_e2m1 = packed_float_to_e2m1(
tSrP_f32_frag[k * 8, i],
tSrP_f32_frag[k * 8 + 1, i],
tSrP_f32_frag[k * 8 + 2, i],
tSrP_f32_frag[k * 8 + 3, i],
tSrP_f32_frag[k * 8 + 4, i],
tSrP_f32_frag[k * 8 + 5, i],
tSrP_f32_frag[k * 8 + 6, i],
tSrP_f32_frag[k * 8 + 7, i]
)
tSrP_u32_view[k] = packed_e2m1
@cute.jit
def softmax_step(
self,
mma_si_consumer_phase: Int32,
si_corr_producer_phase: Int32,
s0_s1_sequence_phase: Int32,
n_block: Int32,
softmax: SoftmaxSm100,
mbar_ptr: cute.Pointer,
mbar_s0_s1_sequence_offset: Int32,
thr_mma_qk: cute.ThrMma,
thr_tmem_load: cute.CopyAtom,
thr_tmem_store: cute.CopyAtom,
thr_tmem_store_scale: cute.CopyAtom,
tStS_t2r: cute.Tensor,
tStScale_r2t: cute.Tensor,
tStP_r2t: cute.Tensor,
sScale: cute.Tensor,
stage: int | Int32,
batch_idx: Int32,
head_idx: Int32,
m_block: Int32,
seqlen,
aux_tensors: Optional[list] = None,
fastdiv_mods=(None, None),
mask_fn: Optional[Callable] = None,
is_first: bool = False,
tCtSFP: Optional[cute.Tensor] = None,
sSFP: Optional[cute.Tensor] = None,
) -> Tuple[cute.Int32, cute.Int32, cute.Int32]:
"""Perform a single step of the softmax computation on a block of attention scores.
This method processes one block of the attention matrix, computing numerically stable
softmax by first finding the row maximum, subtracting it from all elements, applying
exponential function, and then normalizing by the sum of exponentials. It also handles
optional masking of attention scores.
The method involves several key operations:
1. Loading attention scores from tensor memory
2. Applying optional masking based on position
3. Computing row-wise maximum values for numerical stability
4. Transforming scores using exp2(x*scale - max*scale)
5. Computing row sums for normalization
6. Coordinating pipeline synchronization between different processing stages
"""
tilePlikeFP32 = self.mma_tiler_qk[1] // Float32.width * self.v_dtype.width
tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2]))
tScScale = cute.composition(tScS, cute.make_layout((self.m_block_size, 1)))
# P size when in FP32
tScP = cute.composition(tScS, cute.make_layout((self.m_block_size, tilePlikeFP32)))
# Wait for Si
iket.range_push("sm_wait_s")
cute.arch.mbarrier_wait(mbar_ptr + self.mbar_S_full_offset + stage, mma_si_consumer_phase)
iket.range_pop()
tSrS_t2r = cute.make_rmem_tensor(thr_tmem_load.partition_D(tScS).shape, self.qk_acc_dtype)
cute.copy(thr_tmem_load, tStS_t2r, tSrS_t2r)
# unblock sfqk load
cute.arch.fence_view_async_tmem_load()
sfqk_stage = self.q_stage - 1 - stage
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_sfqk_load_offset + sfqk_stage)
if cutlass.const_expr(self.score_mod is not None):
self.apply_score_mod(
tSrS_t2r,
thr_tmem_load,
thr_mma_qk,
batch_idx,
head_idx,
m_block,
n_block,
softmax,
aux_tensors,
fastdiv_mods,
)
if const_expr(mask_fn is not None):
mask_fn(tSrS_t2r, n_block=n_block)
iket.range_push("sm_rowmax")
row_max, acc_scale = softmax.update_row_max(tSrS_t2r.load(), is_first)
iket.range_pop()
tSrPSF_f32 = None
tSrPSF = None
if const_expr(not is_first):
# tSrScale_r2t = cute.make_rmem_tensor(thr_tmem_store_scale.partition_S(tScScale).shape, Float32)
# tSrScale_r2t[0] = acc_scale
# cute.copy(thr_tmem_store_scale, tSrScale_r2t, tStScale_r2t)
# cute.arch.fence_view_async_tmem_store()
thread_idx = thr_tmem_load.thr_idx
sScale[thread_idx + stage * self.m_block_size] = acc_scale
# Notify correction wg that row_max is ready
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_full_offset + stage)
softmax.scale_subtract_rowmax(tSrS_t2r, row_max)
# Sequence barrier wait
if const_expr(self.s0_s1_barrier):
cute.arch.mbarrier_wait(
mbar_ptr + mbar_s0_s1_sequence_offset + stage * 4, s0_s1_sequence_phase
)
tSrP_r2t_f32 = cute.make_rmem_tensor(thr_tmem_store.partition_S(tScP).shape, Float32)
tSrP_r2t = cute.make_tensor(
cute.recast_ptr(tSrP_r2t_f32.iterator, dtype=self.v_dtype),
tSrS_t2r.layout, # shape of S owned by this thread
)
if const_expr(self.quant_pv):
iket.range_push("sm_exp")
# Exp2 with softmax scale and sp1 scaling
softmax.apply_exp2_convert(
tSrS_t2r,
e2e=mask_fn is None and self.head_dim_padded <= 128,
e2e_freq=self.e2e_freq,
)
# update_row_sum BEFORE scale_groupwise so it uses original P values
softmax.update_row_sum(tSrS_t2r.load(), acc_scale, is_first)
iket.range_pop()
# Everything below is the FP4-only P re-quantization: a segmented
# group max, a groupwise rescale, the FP4 convert, and the scale
# factor R2S. The BF16 path reaches the TMEM store directly from
# apply_exp2_convert, so this range is the extra cost FP4 pays on
# the softmax critical path.
iket.range_push("sm_pquant")
tSrPSF_f32 = softmax.compute_group_max(tSrS_t2r, sf_size=self.sf_vec_size)
tSrPSF = cute.make_rmem_tensor(tSrPSF_f32.layout, cute.Float8E4M3FN)
softmax.scale_groupwise(tSrS_t2r, tSrPSF_f32, sf_size=self.sf_vec_size)
self._quant_fp4(tSrS_t2r, tSrPSF_f32, tSrP_r2t, tSrPSF)
# R2S: Copy tSrPSF (registers) to sSFP (shared memory).
# The SFP smem layout is BlockScaledBasicChunk(16) tile_to_shape((M=128, K=128)),
# giving byte offsets:
# byte = (m%32)*16 + ((m//32)%4)*4 + (k_block%4) + (k_block//4)*512
# Each softmax thread holds 1 M row (lane_id within warp = row in [0,32),
# warp_id within softmax warpgroup = row block in [0,4)) and 8 K groups.
# The first 4 K groups land at +0,+1,+2,+3 within the row's atom; the
# next 4 land at +512,+513,+514,+515 (rest_k stride).
if const_expr(sSFP is not None):
thread_idx = thr_tmem_load.thr_idx
lane_id = thread_idx % 32
warp_id = thread_idx // 32
base_offset = lane_id * 16 + (warp_id % 4) * 4
sfp_thread_layout = cute.make_layout((4, 2), stride=(1, 512))
sSFP_stage_ptr = sSFP[None, None, None, stage].iterator
sSFP_thread = cute.make_tensor(sSFP_stage_ptr + base_offset, sfp_thread_layout)
tSrPSF_2d = cute.logical_divide(tSrPSF, cute.make_layout(4))
cute.autovec_copy(tSrPSF_2d, sSFP_thread)
# These stores go through the generic proxy, but the MMA warp
# reads sSFP with tcgen05.cp, an async-proxy access. The
# mbarrier arrive below orders generic-proxy traffic only, so
# without this fence the S2T copy can still observe the
# previous n_block's scale factors.
cute.arch.fence_view_async_shared()
iket.range_pop()
else:
# softmax.scale_apply_exp2_convert(tSrS_t2r, row_max, tSrP_r2t)
softmax.apply_exp2_convert(
tSrS_t2r,
tSrP_r2t,
e2e=mask_fn is None and self.head_dim_padded <= 128,
e2e_freq=self.e2e_freq,
)
# Sequence barrier arrive
if const_expr(self.s0_s1_barrier):
cute.arch.mbarrier_arrive(mbar_ptr + mbar_s0_s1_sequence_offset + (1 - stage) * 4)
for i in cutlass.range_constexpr(self.mbar_p_split(cute.size(tStP_r2t.shape[2]))):
cute.copy(thr_tmem_store, tSrP_r2t_f32[None, None, i], tStP_r2t[None, None, i])
cute.arch.fence_view_async_tmem_store()
# Notify mma warp that P is ready (and SFP is in SMEM)
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage)
for i in cutlass.range_constexpr(self.mbar_p_split(cute.size(tStP_r2t.shape[2])), cute.size(tStP_r2t.shape[2])):
cute.copy(thr_tmem_store, tSrP_r2t_f32[None, None, i], tStP_r2t[None, None, i])
cute.arch.fence_view_async_tmem_store()
# Notify mma warp that the 2nd half of P is ready
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_2_offset + stage)
iket.range_push("sm_wait_corr")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_softmax_corr_empty_offset + stage, si_corr_producer_phase
)
iket.range_pop()
if const_expr(not self.quant_pv):
softmax.update_row_sum(tSrS_t2r.load(), acc_scale, is_first)
# acc_scale = cute.arch.exp2(acc_scale_)
return mma_si_consumer_phase ^ 1, si_corr_producer_phase ^ 1, s0_s1_sequence_phase ^ 1
@cute.jit
def softmax_residual_step(
self,
mma_si_consumer_phase: Int32,
si_corr_producer_phase: Int32,
s0_s1_sequence_phase: Int32,
softmax: SoftmaxSm100,
mbar_ptr: cute.Pointer,
thr_mma_qk: cute.ThrMma,
thr_tmem_load: cute.CopyAtom,
thr_tmem_store_bf16: cute.CopyAtom,
thr_tmem_store_scale: cute.CopyAtom,
tStS_t2r: cute.Tensor,
tStP_bf16_r2t: cute.Tensor,
sScale: cute.Tensor,
stage: int | Int32,
batch_idx: Int32,
m_block: Int32,
mResidualSeqUsedK: cute.Tensor,
thread_idx: Int32,
split_idx: Int32 = Int32(0),
) -> Tuple[cute.Int32, cute.Int32, cute.Int32]:
"""Compute the BF16 residual softmax step.
BF16 variant of softmax_step for the residual block. Reads BF16-derived
FP32 scores from TMEM (written by BF16 GEMM_QK in MMA warp), applies
mResidualSeqUsedK mask, computes row_max, exp2, row_sum (is_first=True
so initializes the softmax state from this block alone), casts P→BF16,
stores P_BF16 to TMEM at tmem_p_bf16_offset (separate from FP4 P).
After this step, softmax.row_max / softmax.row_sum hold the BF16-block
contribution. The FP4 mainloop's first softmax_step then runs with
is_first=False so the online-softmax merge folds in the FP4 blocks on
top of the BF16 seed.
"""
tilePlikeFP32_bf16 = self.mma_tiler_qk[1] // Float32.width * 16 # BF16=16 bits
tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2]))
tScP_bf16 = cute.composition(tScS, cute.make_layout((self.m_block_size, tilePlikeFP32_bf16)))
iket.range_push("sm_res_wait_s")
cute.arch.mbarrier_wait(mbar_ptr + self.mbar_bf16_S_full_offset, Int32(0))
iket.range_pop()
iket.range_push("sm_res_comp")
tSrS_t2r = cute.make_rmem_tensor(thr_tmem_load.partition_D(tScS).shape, self.qk_acc_dtype)
cute.copy(thr_tmem_load, tStS_t2r, tSrS_t2r)
cute.arch.fence_view_async_tmem_load()
# Apply mResidualSeqUsedK mask: tokens at j >= seqused_k[batch_idx] → -inf.
# Values are in [0, BLOCK_SIZE]. A zero-length row still participates in
# the residual pipeline barriers, but below it writes P=0 and leaves the
# reset online-softmax state (-inf max, zero sum) untouched. The first
# FP4 tile can therefore merge with is_first=False without NaNs.
seqused_k = Int32(self.n_block_size)
if const_expr(mResidualSeqUsedK is not None):
seqused_k = mResidualSeqUsedK[batch_idx]
if const_expr(self.is_split_kv and not self.bf16_split):
# One block, one owner: the other splits run the same pipeline
# over a length of zero and contribute nothing, so the combine
# sees the residual exactly once.
if split_idx != self.residual_split_idx:
seqused_k = Int32(0)
tScS_t2r = thr_tmem_load.partition_D(tScS)
for i in cutlass.range_constexpr(cute.size(tSrS_t2r)):
n_coord = tScS_t2r[i][1] # N coordinate
if n_coord >= seqused_k:
tSrS_t2r[i] = -Float32.inf
# Build BF16 P fragment in registers (cast destination).
tSrP_bf16_r2t_view = cute.make_rmem_tensor(
thr_tmem_store_bf16.partition_S(tScP_bf16).shape, Float32
)
# The BF16 P fragment is the same logical shape as tSrS_t2r but typed
# as BFloat16. We use the f32 register and cast inside apply_exp2_convert.
tSrP_bf16 = cute.make_tensor(
cute.recast_ptr(tSrP_bf16_r2t_view.iterator, dtype=cutlass.BFloat16),
tSrS_t2r.layout,
)
if seqused_k > Int32(0):
# Compute row_max with is_first=True (initializes softmax state).
row_max, acc_scale = softmax.update_row_max(
tSrS_t2r.load(), is_first=True
)
# In-place compute (S - row_max) * scale_log2.
softmax.scale_subtract_rowmax(tSrS_t2r, row_max)
# exp2 in FP32 + cast to BF16 destination.
softmax.apply_exp2_convert(
tSrS_t2r,
tSrP_bf16,
e2e=False,
)
softmax.update_row_sum(
tSrS_t2r.load(), acc_scale, is_first=True
)
else:
# The MMA/PV pipeline must still observe its normal producer
# arrivals, so publish an all-zero P tile instead of skipping it.
tSrP_bf16_r2t_view.fill(Float32(0.0))
# Store BF16 P to TMEM (split into two halves to overlap with MMA).
for i in cutlass.range_constexpr(self.mbar_p_split(cute.size(tStP_bf16_r2t.shape[2]))):
cute.copy(thr_tmem_store_bf16, tSrP_bf16_r2t_view[None, None, i], tStP_bf16_r2t[None, None, i])
cute.arch.fence_view_async_tmem_store()
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_bf16_P_full_offset)
for i in cutlass.range_constexpr(
self.mbar_p_split(cute.size(tStP_bf16_r2t.shape[2])),
cute.size(tStP_bf16_r2t.shape[2]),
):
cute.copy(thr_tmem_store_bf16, tSrP_bf16_r2t_view[None, None, i], tStP_bf16_r2t[None, None, i])
cute.arch.fence_view_async_tmem_store()
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_bf16_P_full_2_offset)
iket.range_pop()
# Phase tracking: mma_si_consumer_phase is unchanged because we
# didn't consume mbar_S_full; si_corr_producer_phase is unchanged
# because we didn't consume mbar_softmax_corr_empty.
return mma_si_consumer_phase, si_corr_producer_phase, s0_s1_sequence_phase
@cute.jit
def correction_loop(
self,
thr_mma_qk: cute.ThrMma,
thr_mma_pv: cute.ThrMma,
tStS: cute.Tensor,
tOtOs: tuple[cute.Tensor],
sScale: cute.Tensor,
mO: cute.Tensor,
mLSE: cute.Tensor,
sO: cute.Tensor,
learnable_sink: Optional[cute.Tensor],
gmem_tiled_copy_O: cute.TiledCopy,
tma_atom_O: cute.CopyAtom,
mbar_ptr: cute.Pointer,
softmax_scale_log2: Float32,
block_info: BlockInfo,
num_splits: Int32,
SeqlenInfoCls: Callable,
TileSchedulerCls: Callable,
blocksparse_tensors: Optional[BlockSparseTensors] = None,
role: cutlass.Constexpr = SplitRole.ALL,
):
runs_pages = const_expr(role != SplitRole.BF16)
runs_tail = const_expr(
self.fused_residual_first_block and role != SplitRole.PAGES
)
runs_sink = const_expr(self.fused_sink_block and role != SplitRole.PAGES)
two_level = const_expr(self.p_scale_two_level and runs_pages)
accumulated = None
previous_outer = Float32(1.0)
if const_expr(two_level):
assert self.q_stage == 1
accumulated = cute.make_rmem_tensor(cute.make_layout(self.head_dim_v_padded), Float32)
tidx = cute.arch.thread_idx()[0] % (cute.arch.WARP_SIZE * len(self.correction_warp_ids))
tScS = thr_mma_qk.partition_C(cute.make_identity_tensor(self.mma_tiler_qk[:2]))
# O^T is a 128-lane FP32 accumulator like S, so S's tensor-memory
# layout at O's columns addresses it; column r is query row r.
tOtOs_corr = tOtOs
if const_expr(self.narrow_pv):
tOtOs_corr = (
cute.make_tensor(tStS.iterator + self.tmem_o_offset[0], tStS.layout),
)
tStScale_layout = cute.composition(tStS.layout, cute.make_layout((self.m_block_size, 1)))
tStScales = tuple(
cute.make_tensor(tStS.iterator + self.tmem_vec_offset[stage], tStScale_layout)
for stage in range(self.q_stage)
)
tScScale = cute.composition(tScS, cute.make_layout((self.m_block_size, 1)))
tmem_load_v_atom = cute.make_copy_atom(
tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(1)),
self.qk_acc_dtype,
)
thr_tmem_load_vec = tcgen05.make_tmem_copy(tmem_load_v_atom, tStScales[0]).get_slice(tidx)
tStScales_t2r = [thr_tmem_load_vec.partition_S(tStScales[stage]) for stage in range(self.q_stage)]
tSrScale_t2r_shape = thr_tmem_load_vec.partition_D(tScScale).shape
# The free arrival that lets the mma warp past its first PV. Only a tile
# starting on an fp4 block gets it; see correction_consume_scale.
if const_expr(not runs_tail):
for stage_init in cutlass.range_constexpr(self.q_stage):
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage_init
)
softmax_corr_consumer_phase = Int32(0)
o_corr_consumer_phase = Int32(0)
pv_consumer_phase = Int32(0)
corr_epi_producer_phase = Int32(1)
tile_scheduler = TileSchedulerCls()
work_tile = tile_scheduler.initial_work_tile_info()
while work_tile.is_valid_tile:
m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx
seqlen = SeqlenInfoCls(batch_idx)
n_block_min, n_block_max = self.split_block_range(
block_info, seqlen, m_block, split_idx, num_splits
)
if const_expr(self.is_split_kv):
mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[None, None, head_idx, split_idx]
else:
mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[None, None, head_idx]
gO = cute.local_tile(mO_cur, (self.m_block_size, self.head_dim_v_padded), (None, 0))
# Default LSE to -inf for invalid split_idx tiles
stats = [(0.0, -Float32.inf if const_expr(mLSE is not None or learnable_sink is not None) else None, True)] * self.q_stage
if const_expr(self.use_block_sparsity):
total_block_count = get_total_block_count(blocksparse_tensors, batch_idx, head_idx, m_block)
has_work = total_block_count > Int32(0)
else:
total_block_count = n_block_max - n_block_min
has_work = self.tile_has_work(
n_block_min, n_block_max, split_idx, num_splits
)
if has_work:
if const_expr(two_level):
accumulated.fill(0.0)
previous_outer = Float32(1.0)
if const_expr(runs_pages):
# The first signal from softmax needs no correction unless a
# residual seeded the online softmax, in which case it carries
# the factor that takes O from the residual's row max to the
# first fp4 block's.
iket.range_push("corr_wait_sm")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_softmax_corr_full_offset + 0, softmax_corr_consumer_phase
)
if const_expr(runs_tail):
previous_outer = self.correction_consume_scale(
thr_mma_pv, tOtOs_corr, sScale, tidx, 0, mbar_ptr,
accumulated=accumulated, previous_outer=previous_outer,
tScS=tScS,
)
elif const_expr(two_level):
previous_outer = sScale[tidx + self.m_block_size * 2] if tidx < self.transposed_query_rows else Float32(0.0)
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_empty_offset + 0)
if const_expr(self.q_stage == 2):
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_softmax_corr_full_offset + 1, softmax_corr_consumer_phase
)
if const_expr(runs_tail):
self.correction_consume_scale(
thr_mma_pv, tOtOs_corr, sScale, tidx, 1, mbar_ptr,
tScS=tScS,
)
iket.range_pop()
softmax_corr_consumer_phase ^= 1
tSrScale_t2r = cute.make_rmem_tensor(tSrScale_t2r_shape, Float32)
# The sink's factor comes over the same handshake as a
# page's, so it takes one more step of this loop. A second
# copy of the rescale after the loop, never mind whether it
# runs, compiles every block of the loop longer: 2.5%
# slower at batch 32 on Qwen3-4B. A tile whose fp4 length is
# zero still runs its first step.
correction_steps = total_block_count - 1
if const_expr(runs_sink):
correction_steps = cutlass.max(correction_steps, Int32(0)) + 1
for i in cutlass.range(correction_steps, unroll=1):
for stage in cutlass.range_constexpr(self.q_stage):
# wait for S0 / S1
iket.range_push("corr_wait_sm")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_softmax_corr_full_offset + stage,
softmax_corr_consumer_phase,
)
iket.range_pop()
previous_outer = self.correction_consume_scale(
thr_mma_pv, tOtOs_corr, sScale, tidx, stage, mbar_ptr,
previous_pv_phase=pv_consumer_phase ^ (i & 1),
accumulated=accumulated, previous_outer=previous_outer,
tScS=tScS,
)
if const_expr(self.q_stage == 2):
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_softmax_corr_empty_offset + (1 - stage)
)
else:
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_softmax_corr_empty_offset + stage
)
softmax_corr_consumer_phase ^= 1
if const_expr(self.transpose_s):
# P has one completion per FP4 page, including the
# zero-page placeholder. The correction handshake also
# signals the final row sum, so its phase cannot track
# P across tiles of a persistent CTA.
pv_consumer_phase ^= cutlass.max(total_block_count, Int32(1)) & 1
if const_expr(runs_sink and not runs_pages):
iket.range_push("corr_wait_sm")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_softmax_corr_full_offset,
softmax_corr_consumer_phase,
)
iket.range_pop()
self.correction_consume_scale(
thr_mma_pv, tOtOs_corr, sScale, tidx, 0, mbar_ptr,
tScS=tScS,
)
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_softmax_corr_empty_offset
)
softmax_corr_consumer_phase ^= 1
if const_expr(self.q_stage == 2):
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_softmax_corr_empty_offset + 1
)
# End of seqlen_corr_loop_steps
# Even in the case of self.overlap_sO_sQ, we can write to stage 0 of sO without
# additional sync because the MMA in the top half must have been done.
# Similarly we can write to stage 1 of sO without additional sync.
# Over the BF16 tiles the O_full wait below suffices: it follows
# the tile's last MMA, and that MMA is their last reader.
learnable_sink_val = [None] * self.q_stage
if const_expr(learnable_sink is not None):
if const_expr(not self.pack_gqa):
sink_val = Float32(learnable_sink[head_idx])
learnable_sink_val = [sink_val] * self.q_stage
else: # Each thread might have a different sink value due to different q_head
for stage in cutlass.range_constexpr(self.q_stage):
q_head_idx = (
(self.q_stage * m_block + stage) * self.m_block_size + tidx
) % self.qhead_per_kvhead + head_idx * self.qhead_per_kvhead
learnable_sink_val[stage] = Float32(learnable_sink[q_head_idx])
for stage in cutlass.range_constexpr(self.q_stage):
iket.range_push("corr_wait_sm")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_softmax_corr_full_offset + stage,
softmax_corr_consumer_phase,
)
iket.range_pop()
# cute.copy(tiled_tmem_load_vec, tStScales_t2r[stage], tSrScale_t2r)
# cute.arch.fence_view_async_tmem_load()
# scale = tSrScale_t2r[0]
if const_expr(self.transpose_s):
# A zero sum marks the row as empty further down, which
# is what the rows past the live query rows are.
row_sum = (
sScale[tidx + stage * self.m_block_size]
if tidx < self.transposed_query_rows
else Float32(0.0)
)
else:
row_sum = sScale[tidx + stage * self.m_block_size]
if const_expr(mLSE is not None or learnable_sink is not None):
row_max = sScale[tidx + stage * self.m_block_size + self.m_block_size * 2]
if const_expr(self.transpose_s):
row_max = (
row_max
if tidx < self.transposed_query_rows
else -Float32.inf
)
else:
row_max = None
# A narrow PV leaves every live row's lane of O^T in every
# thread, so each one needs all the row sums, and has to
# read them before softmax is free to overwrite sScale.
row_scales = None
if const_expr(self.narrow_pv):
row_scales = cute.make_rmem_tensor(
cute.make_layout(self.transposed_query_rows), Float32
)
for row in cutlass.range_constexpr(self.transposed_query_rows):
live_sum = sScale[row + stage * self.m_block_size]
row_scales[row] = cute.arch.rcp_approx(
live_sum
if not (live_sum == 0.0 or live_sum != live_sum)
else 1.0
)
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_softmax_corr_empty_offset + stage)
if const_expr(learnable_sink is not None):
LOG2_E = math.log2(math.e)
sink_val = learnable_sink_val[stage]
if const_expr(not self.is_split_kv) or split_idx == 0:
if row_max == -Float32.inf:
# It's possible to have an empty row with splitKV.
row_max = sink_val * (LOG2_E / softmax_scale_log2)
row_sum = Float32(2.0 ** self.p_scale_rescale_log2)
else:
row_sum += utils.exp2f(
sink_val * LOG2_E - row_max * softmax_scale_log2
+ Float32(self.p_scale_rescale_log2)
)
acc_O_mn_row_is_zero_or_nan = row_sum == 0.0 or row_sum != row_sum
stats[stage] = (row_sum, row_max, acc_O_mn_row_is_zero_or_nan)
scale = cute.arch.rcp_approx(row_sum if not acc_O_mn_row_is_zero_or_nan else 1.0)
iket.range_push("corr_wait_o")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_O_full_offset + stage, o_corr_consumer_phase
)
iket.range_pop()
if const_expr(not self.use_correction_warps_for_epi):
iket.range_push("corr_wait_epi")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_corr_epi_empty_offset + stage, corr_epi_producer_phase
)
iket.range_pop()
if const_expr(two_level):
self.correction_accumulate_tile(
thr_mma_pv, tOtOs[stage], tidx, accumulated,
previous_outer, Float32(1.0), store=True,
)
if const_expr(self.narrow_pv):
self.correction_epilogue_rows(
tOtOs_corr[0], tScS, tidx, row_scales, sO[None, None, stage]
)
else:
self.correction_epilogue(
thr_mma_pv,
tOtOs[stage],
tidx,
stage,
m_block,
seqlen.seqlen_q,
scale,
sO[None, None, stage],
mO_cur,
gO,
gmem_tiled_copy_O,
)
if const_expr(not self.use_correction_warps_for_epi):
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_corr_epi_full_offset + stage)
# Signal for the next work tile that O buffers in tmem are already read, so
# mma warp can write to them. Withheld on a residual-seeded
# tile for the same reason as the pre-loop arrival; see
# correction_consume_scale.
if const_expr(not runs_tail):
cute.arch.mbarrier_arrive(
mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage
)
o_corr_consumer_phase ^= 1
softmax_corr_consumer_phase ^= 1
corr_epi_producer_phase ^= 1
else:
# WARNING: we need some code before the const_expr, see https://github.com/NVIDIA/cutlass/issues/2781
if const_expr(self.use_correction_warps_for_epi):
gmem_tiled_copy_O_for_empty_tile = gmem_tiled_copy_O
else:
gmem_tiled_copy_O_for_empty_tile = None
if const_expr(self.use_block_sparsity):
(
softmax_corr_consumer_phase,
o_corr_consumer_phase,
corr_epi_producer_phase,
) = handle_block_sparse_empty_tile_correction_sm100(
tidx,
self.q_stage,
self.m_block_size,
self.qhead_per_kvhead,
self.pack_gqa,
self.is_split_kv,
learnable_sink,
mLSE,
seqlen,
m_block,
head_idx,
batch_idx,
split_idx,
sScale,
stats,
self.correction_epilogue,
thr_mma_pv,
tOtOs,
sO,
mbar_ptr,
self.mbar_softmax_corr_full_offset,
self.mbar_softmax_corr_empty_offset,
self.mbar_P_full_O_rescaled_offset,
self.mbar_P_full_2_offset,
self.mbar_corr_epi_full_offset,
self.mbar_corr_epi_empty_offset,
softmax_corr_consumer_phase,
o_corr_consumer_phase,
corr_epi_producer_phase,
softmax_scale_log2,
mO_cur,
gO,
gmem_tiled_copy_O_for_empty_tile,
)
if const_expr(mLSE is not None):
if const_expr(not seqlen.has_cu_seqlens_q):
if const_expr(self.is_split_kv):
mLSE_cur = mLSE[None, head_idx, batch_idx, split_idx]
else:
mLSE_cur = mLSE[None, head_idx, batch_idx]
else:
offset = (
seqlen.offset_q if const_expr(not self.pack_gqa) else (0, seqlen.offset_q)
)
if const_expr(self.is_split_kv):
mLSE_cur = cute.domain_offset((offset,), mLSE[None, head_idx, split_idx])
else:
mLSE_cur = cute.domain_offset((offset,), mLSE[None, head_idx])
for stage in cutlass.range_constexpr(self.q_stage):
gLSE = cute.local_tile(
mLSE_cur, (self.m_block_size,), (self.q_stage * m_block + stage,)
)
row_sum, row_max, acc_O_mn_row_is_zero_or_nan = stats[stage]
LN2 = math.log(2.0)
lse = (
(row_max * softmax_scale_log2 + utils.log2f(row_sum)) * LN2
if not acc_O_mn_row_is_zero_or_nan
else -Float32.inf
)
if const_expr(self.p_scale_rescale_log2):
lse -= Float32(self.p_scale_rescale_log2 * LN2)
effective_seqlen_q_lse = (
Int32(1)
if const_expr(self.seqlen_q_static_one)
else seqlen.seqlen_q
)
seqlen_q = (
effective_seqlen_q_lse
if const_expr(not self.pack_gqa)
else effective_seqlen_q_lse * self.qhead_per_kvhead
)
if tidx < seqlen_q - (self.q_stage * m_block + stage) * self.m_block_size:
# This actually just works with PackGQA too
gLSE[tidx] = lse
# Advance to next tile
tile_scheduler.advance_to_next_work()
work_tile = tile_scheduler.get_current_work()
# End of persistent scheduler loop
@cute.jit
def correction_rescale(
self,
thr_mma: cute.ThrMma,
tOtO: cute.Tensor,
tidx: Int32,
scale: Float32,
):
"""Rescale intermediate attention results based on softmax normalization factor.
This method performs a crucial correction step in the attention computation pipeline.
When processing attention in blocks, the softmax normalization factors may change
as new blocks are processed. This method rescales previously computed partial
output values to account for updated normalization factors.
The implementation uses efficient tensor memory operations to:
1. Load existing partial attention output from tensor memory
2. Apply the scaling factor to all elements
3. Store the rescaled results back to tensor memory
"""
tOcO = thr_mma.partition_C(cute.make_identity_tensor(self.mma_tiler_pv[:2]))
corr_tile_size = 16 # tuneable parameter
tmem_load_atom = cute.make_copy_atom(
tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(corr_tile_size)),
self.pv_acc_dtype,
)
tmem_store_atom = cute.make_copy_atom(
tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(corr_tile_size)),
self.pv_acc_dtype,
)
tOtO_i = cute.composition(tOtO, cute.make_layout((self.m_block_size, corr_tile_size)))
tOcO_i = cute.composition(tOcO, cute.make_layout((self.m_block_size, corr_tile_size)))
thr_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tOtO_i).get_slice(tidx)
thr_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tOtO_i).get_slice(tidx)
tOtO_t2r = thr_tmem_load.partition_S(tOtO_i)
tOrO_t2r_shape = thr_tmem_load.partition_D(tOcO_i).shape
tOtO_r2t = thr_tmem_store.partition_D(tOtO_i)
frg_count = self.head_dim_v_padded // corr_tile_size
tOrO_frg = cute.make_rmem_tensor((tOrO_t2r_shape, frg_count), self.pv_acc_dtype)
for i in cutlass.range_constexpr(frg_count):
tOrO_frg = cute.make_rmem_tensor(tOrO_t2r_shape, self.pv_acc_dtype)
tOtO_t2r_i = cute.make_tensor(tOtO_t2r.iterator + i * corr_tile_size, tOtO_t2r.layout)
cute.copy(thr_tmem_load, tOtO_t2r_i, tOrO_frg)
for j in cutlass.range(0, cute.size(tOrO_frg), 2, unroll_full=True):
tOrO_frg[j], tOrO_frg[j + 1] = utils.mul_packed_f32x2(
(tOrO_frg[j], tOrO_frg[j + 1]),
(scale, scale),
)
tOtO_r2t_i = cute.make_tensor(tOtO_r2t.iterator + i * corr_tile_size, tOtO_r2t.layout)
cute.copy(thr_tmem_store, tOrO_frg, tOtO_r2t_i)
cute.arch.fence_view_async_tmem_store()
def _o_transposed_copies(self, tOtT: cute.Tensor, tScS: cute.Tensor, tidx: Int32):
"""Tensor-memory copies over eight rows of O^T at a time.
A thread gets one head-dimension lane and eight consecutive columns,
which are query rows; fragment element j is column j.
"""
chunk = 8
tile = cute.composition(tOtT, cute.make_layout((self.m_block_size, chunk)))
coords = cute.composition(tScS, cute.make_layout((self.m_block_size, chunk)))
load = tcgen05.make_tmem_copy(
cute.make_copy_atom(
tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(chunk)), self.pv_acc_dtype
),
tile,
).get_slice(tidx)
store = tcgen05.make_tmem_copy(
cute.make_copy_atom(
tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(chunk)), self.pv_acc_dtype
),
tile,
).get_slice(tidx)
return (
chunk,
load,
store,
load.partition_S(tile),
store.partition_D(tile),
load.partition_D(coords).shape,
)
@cute.jit
def correction_rescale_rows(
self,
tOtT: cute.Tensor,
tScS: cute.Tensor,
tidx: Int32,
sScale: cute.Tensor,
stage: int,
):
"""Rescale O^T by each live row's factor; see ``narrow_pv``.
The columns past the live rows hold the product of P's zero rows and
are left alone.
"""
chunk, load, store, source, destination, frg_shape = self._o_transposed_copies(
tOtT, tScS, tidx
)
for c in cutlass.range_constexpr(self.pv_rows // chunk):
fragment = cute.make_rmem_tensor(frg_shape, self.pv_acc_dtype)
cute.copy(load, cute.make_tensor(source.iterator + c * chunk, source.layout), fragment)
for j in cutlass.range_constexpr(chunk):
if const_expr(c * chunk + j < self.transposed_query_rows):
fragment[j] = fragment[j] * sScale[c * chunk + j + stage * self.m_block_size]
cute.copy(
store, fragment, cute.make_tensor(destination.iterator + c * chunk, destination.layout)
)
cute.arch.fence_view_async_tmem_store()
@cute.jit
def correction_epilogue_rows(
self,
tOtT: cute.Tensor,
tScS: cute.Tensor,
tidx: Int32,
row_scales: cute.Tensor,
sO: cute.Tensor,
):
"""Normalize O^T and store it into the row-major O tile.
Each thread writes its head-dimension lane of every live row, so the
epilogue warp copies sO out exactly as it does the wide tile.
"""
chunk, load, _, source, _, frg_shape = self._o_transposed_copies(tOtT, tScS, tidx)
for c in cutlass.range_constexpr(self.pv_rows // chunk):
fragment = cute.make_rmem_tensor(frg_shape, self.pv_acc_dtype)
cute.copy(load, cute.make_tensor(source.iterator + c * chunk, source.layout), fragment)
for j in cutlass.range_constexpr(chunk):
if const_expr(c * chunk + j < self.transposed_query_rows):
fragment[j] = fragment[j] * row_scales[c * chunk + j]
converted = cute.make_rmem_tensor(frg_shape, self.o_dtype)
converted.store(fragment.load().to(self.o_dtype))
for j in cutlass.range_constexpr(chunk):
if const_expr(c * chunk + j < self.transposed_query_rows):
sO[c * chunk + j, tidx] = converted[j]
cute.arch.fence_view_async_tmem_load()
cute.arch.fence_proxy("async.shared", space="cta")
@cute.jit
def correction_accumulate_tile(
self, thr_mma: cute.ThrMma, tOtO: cute.Tensor, tidx: Int32,
accumulated: cute.Tensor, outer: Float32, rescale: Float32,
store: cutlass.Constexpr[bool] = False,
):
"""Merge one PV tile after undoing its FP32 outer scale exactly once.
The running numerator remains in the ordinary online-softmax basis.
Each FP4 MMA zero-initializes TMEM; the correction handshake prevents
its reuse until this load completes. BF16 tail and sink use outer=1.
"""
width = const_expr(16)
identity = thr_mma.partition_C(cute.make_identity_tensor(self.mma_tiler_pv[:2]))
tile = cute.composition(tOtO, cute.make_layout((self.m_block_size, width)))
coordinates = cute.composition(identity, cute.make_layout((self.m_block_size, width)))
loader = tcgen05.make_tmem_copy(cute.make_copy_atom(
tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(width)), Float32,
), tile).get_slice(tidx)
source = loader.partition_S(tile)
fragment = cute.make_rmem_tensor(loader.partition_D(coordinates).shape, Float32)
writer = tcgen05.make_tmem_copy(cute.make_copy_atom(
tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(width)), Float32,
), tile).get_slice(tidx)
destination = writer.partition_D(tile)
for i in cutlass.range_constexpr(self.head_dim_v_padded // width):
src = cute.make_tensor(source.iterator + i * width, source.layout)
cute.copy(loader, src, fragment)
for j in cutlass.range_constexpr(cute.size(fragment)):
contribution = Float32(0.0)
if outer > 0.0:
contribution = fragment[j] * outer
accumulated[i * width + j] = (accumulated[i * width + j] + contribution) * rescale
if const_expr(store):
fragment[j] = accumulated[i * width + j]
if const_expr(store):
dst = cute.make_tensor(destination.iterator + i * width, destination.layout)
cute.copy(writer, fragment, dst)
cute.arch.fence_view_async_tmem_load()
if const_expr(store):
cute.arch.fence_view_async_tmem_store()
@cute.jit
def correction_consume_scale(
self,
thr_mma_pv: cute.ThrMma,
tOtOs: tuple[cute.Tensor],
sScale: cute.Tensor,
tidx: Int32,
stage: int,
mbar_ptr: cute.Pointer,
previous_pv_phase: Optional[Int32] = None,
accumulated: Optional[cute.Tensor] = None,
previous_outer: Float32 = Float32(1.0),
tScS: Optional[cute.Tensor] = None,
):
"""Take one correction factor from softmax and release the mma warp.
Transposed, softmax publishes one factor per live query row rather than
one per thread, so the O rows past that bound have no factor and no
consumer.
Every step of the loop runs this. The first step of a tile does too,
but only when a fused residual seeded the online softmax: the
residual's PV has already written O by then, which makes that first
factor a real one rather than the no-op it is on a tile whose first
block zero-initializes O. The arrival matters as much as the rescale —
it is what holds the mma warp's first fp4 PV until O has been scaled,
and a residual-seeded tile has no free pre-loop arrival standing in for
it.
With a narrow PV ``tOtOs`` holds the O^T view instead, every thread
holds a lane of every live row, and all of them read every factor.
"""
if const_expr(self.narrow_pv):
min_scale = sScale[stage * self.m_block_size]
for row in cutlass.range_constexpr(1, self.transposed_query_rows):
min_scale = cutlass.min(min_scale, sScale[row + stage * self.m_block_size])
if min_scale < 1.0:
if const_expr(previous_pv_phase is not None):
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_P_consumed_offset + stage,
previous_pv_phase,
)
self.correction_rescale_rows(tOtOs[0], tScS, tidx, sScale, stage)
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage)
return Float32(1.0)
if const_expr(self.transpose_s):
scale = (
sScale[tidx + stage * self.m_block_size]
if tidx < self.transposed_query_rows
else Float32(1.0)
)
else:
scale = sScale[tidx + stage * self.m_block_size]
next_outer = Float32(1.0)
if const_expr(accumulated is not None):
if const_expr(previous_pv_phase is not None):
cute.arch.mbarrier_wait(mbar_ptr + self.mbar_P_consumed_offset + stage, previous_pv_phase)
self.correction_accumulate_tile(
thr_mma_pv, tOtOs[0], tidx, accumulated, previous_outer, scale,
)
next_outer = sScale[tidx + self.m_block_size * 2] if tidx < self.transposed_query_rows else Float32(0.0)
should_rescale = cute.arch.vote_ballot_sync(scale < 1.0) != 0
if const_expr(accumulated is None) and should_rescale:
# Transposed QK can finish before the preceding PV is issued.
# Its softmax signal therefore does not make O safe to rescale.
# Wait only when touching O; a unit factor needs no PV dependency.
if const_expr(self.transpose_s and previous_pv_phase is not None):
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_P_consumed_offset + stage,
previous_pv_phase,
)
self.correction_rescale(
thr_mma_pv, tOtOs[stage if self.q_stage == 2 else 0], tidx, scale
)
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_P_full_O_rescaled_offset + stage)
return next_outer
@cute.jit
def correction_epilogue(
self,
thr_mma: cute.ThrMma,
tOtO: cute.Tensor, # tmem
tidx: Int32,
stage: Int32,
m_block: Int32,
seqlen_q: Int32,
scale: Float32,
sO: cute.Tensor,
mO_cur: Optional[cute.Tensor] = None,
gO: Optional[cute.Tensor] = None,
gmem_tiled_copy_O: Optional[cute.TiledCopy] = None,
):
"""Apply final scaling and transformation to attention output before writing to global memory.
This correction_epilogue function handles the final processing step for attention output values.
It applies a scaling factor to the accumulated attention results and prepares the
data for efficient transfer back to global memory.
The method performs:
1. Loading of accumulated attention results from tensor memory
2. Application of the final output scaling factor
3. Type conversion if necessary (typically from higher precision accumulator to output precision)
4. Reorganization of data for optimal memory access patterns
5. Preparation for efficient TMA store operations
:param thr_mma: Thread MMA operation for the computation
:type thr_mma: cute.ThrMma
:param tOtO: Tensor containing accumulated attention output
:type tOtO: cute.Tensor
:param scale: Final scaling factor(softmax denominator) to apply to the output
:type scale: Float32
:param sO: Shared memory tensor for the final output
:type sO: cute.Tensor
"""
corr_tile_size = 32 * 8 // self.o_dtype.width
tOsO = thr_mma.partition_C(sO)
tOcO = thr_mma.partition_C(cute.make_identity_tensor(self.mma_tiler_pv[:2]))
tOtO_i = cute.logical_divide(tOtO, cute.make_layout((self.m_block_size, corr_tile_size)))
tOcO_i = cute.logical_divide(tOcO, cute.make_layout((self.m_block_size, corr_tile_size)))
tOsO_i = cute.logical_divide(tOsO, cute.make_layout((self.m_block_size, corr_tile_size)))
epi_subtile = (self.epi_tile[0], corr_tile_size)
tmem_copy_atom = sm100_utils_basic.get_tmem_load_op(
self.mma_tiler_pv,
self.o_layout,
self.o_dtype,
self.pv_acc_dtype,
epi_subtile,
use_2cta_instrs=False,
)
tiled_tmem_load = tcgen05.make_tmem_copy(tmem_copy_atom, tOtO_i[(None, None), 0]).get_slice(
tidx
)
thr_tmem_load = tiled_tmem_load.get_slice(tidx)
smem_copy_atom = sm100_utils_basic.get_smem_store_op(
self.o_layout, self.o_dtype, self.pv_acc_dtype, tiled_tmem_load
)
tiled_smem_store = cute.make_tiled_copy_D(smem_copy_atom, tiled_tmem_load)
tOtO_t2r = thr_tmem_load.partition_S(tOtO_i[(None, None), None])
tOsO_s2r = thr_tmem_load.partition_D(tOsO_i[(None, None), None])
tOcO_t2r = thr_tmem_load.partition_D(tOcO_i[(None, None), None])
for i in cutlass.range_constexpr(self.head_dim_v_padded // corr_tile_size):
tOtO_t2r_i = tOtO_t2r[None, 0, 0, i]
tOsO_r2s_i = tOsO_s2r[None, 0, 0, i]
tOrO_frg = cute.make_rmem_tensor(tOcO_t2r[None, 0, 0, i].shape, self.pv_acc_dtype)
cute.copy(tiled_tmem_load, tOtO_t2r_i, tOrO_frg)
for j in cutlass.range_constexpr(0, cute.size(tOrO_frg), 2):
tOrO_frg[j], tOrO_frg[j + 1] = utils.mul_packed_f32x2(
(tOrO_frg[j], tOrO_frg[j + 1]),
(scale, scale),
)
tOrO_frg_cvt = cute.make_rmem_tensor(tOrO_frg.shape, self.o_dtype)
tOrO_frg_cvt.store(tOrO_frg.load().to(self.o_dtype))
cute.copy(tiled_smem_store, tOrO_frg_cvt, tOsO_r2s_i)
# fence view async shared
cute.arch.fence_proxy("async.shared", space="cta")
if const_expr(self.use_correction_warps_for_epi):
assert(not self.use_tma_O)
assert(gmem_tiled_copy_O is not None)
cute.arch.barrier(barrier_id=int(NamedBarrierFwd.Epilogue),
number_of_threads=len(self.epilogue_warp_ids) * cute.arch.WARP_SIZE)
gmem_thr_copy_O = gmem_tiled_copy_O.get_slice(tidx)
tOsO = gmem_thr_copy_O.partition_S(sO)
cO = cute.make_identity_tensor((self.m_block_size, self.head_dim_v_padded))
tOgO = gmem_thr_copy_O.partition_D(gO)
tOcO = gmem_thr_copy_O.partition_S(cO)
t0OcO = gmem_tiled_copy_O.get_slice(0).partition_S(cO)
tOpO = utils.predicate_k(tOcO, limit=mO_cur.shape[1])
# This output path does not support packed GQA.
assert not self.pack_gqa
pack_gqa = PackGQA(
self.m_block_size,
self.head_dim_v_padded,
self.check_hdim_v_oob,
self.qhead_per_kvhead,
)
# load acc O from smem to rmem for wider vectorization
tOrO = cute.make_fragment_like(tOsO, self.o_dtype)
cute.autovec_copy(tOsO, tOrO)
effective_seqlen_q_corr = (
Int32(1)
if const_expr(self.seqlen_q_static_one)
else seqlen_q
)
if const_expr(not self.pack_gqa):
for rest_m in cutlass.range_constexpr(cute.size(tOrO.shape[1])):
if (
t0OcO[0, rest_m, 0][0]
< effective_seqlen_q_corr
- (self.q_stage * m_block + stage) * self.m_block_size
- tOcO[0][0]
):
cute.copy(
gmem_tiled_copy_O,
tOrO[None, rest_m, None],
tOgO[None, rest_m, None, self.q_stage * m_block + stage],
pred=tOpO[None, rest_m, None]
if const_expr(self.check_hdim_v_oob)
else None,
)
else:
pack_gqa.store_O(
mO_cur,
tOrO,
gmem_tiled_copy_O,
tidx,
self.q_stage * m_block + stage,
effective_seqlen_q_corr,
)
@cute.jit
def epilogue_s2g(
self,
mO: cute.Tensor,
sO: cute.Tensor,
gmem_tiled_copy_O: cute.TiledCopy,
tma_atom_O: Optional[cute.CopyAtom],
mbar_ptr: cute.Pointer,
block_info: BlockInfo,
num_splits: int,
SeqlenInfoCls: Callable,
TileSchedulerCls: Callable,
mOutIndices: Optional[cute.Tensor] = None,
):
epi_consumer_phase = Int32(0)
tile_scheduler = TileSchedulerCls()
work_tile = tile_scheduler.initial_work_tile_info()
while work_tile.is_valid_tile:
m_block, head_idx, batch_idx, split_idx = work_tile.tile_idx
seqlen = SeqlenInfoCls(batch_idx)
n_block_min, n_block_max = self.split_block_range(
block_info, seqlen, m_block, split_idx, num_splits
)
if self.tile_has_work(n_block_min, n_block_max, split_idx, num_splits):
if const_expr(self.use_out_indices):
mO_cur = seqlen.offset_batch_O_via_indices(
mO, batch_idx, mOutIndices, dim=3,
)[None, None, head_idx]
elif const_expr(self.is_split_kv):
mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[None, None, head_idx, split_idx]
else:
mO_cur = seqlen.offset_batch_Q(mO, batch_idx, dim=3)[None, None, head_idx]
gO = cute.local_tile(mO_cur, (self.m_block_size, self.head_dim_v_padded), (None, 0))
if const_expr(self.use_tma_O):
store_O, _, _ = copy_utils.tma_get_copy_fn(
tma_atom_O, 0, cute.make_layout(1), sO, gO
)
for stage in cutlass.range_constexpr(self.q_stage):
# wait from corr, issue tma store on smem
# 1. wait for O0 / O1 final
iket.range_push("epi_wait_corr")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_corr_epi_full_offset + stage, epi_consumer_phase
)
iket.range_pop()
# 2. copy O0 / O1 to gmem
store_O(src_idx=stage, dst_idx=self.q_stage * m_block + stage)
cute.arch.cp_async_bulk_commit_group()
for stage in cutlass.range_constexpr(self.q_stage):
# Ensure O0 / O1 buffer is ready to be released
cute.arch.cp_async_bulk_wait_group(1 - stage, read=True)
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_corr_epi_empty_offset + stage)
else:
tidx = cute.arch.thread_idx()[0] % (
cute.arch.WARP_SIZE * len(self.epilogue_warp_ids)
)
gmem_thr_copy_O = gmem_tiled_copy_O.get_slice(tidx)
tOsO = gmem_thr_copy_O.partition_S(sO)
cO = cute.make_identity_tensor((self.m_block_size, self.head_dim_v_padded))
tOgO = gmem_thr_copy_O.partition_D(gO)
tOcO = gmem_thr_copy_O.partition_S(cO)
t0OcO = gmem_tiled_copy_O.get_slice(0).partition_S(cO)
tOpO = utils.predicate_k(tOcO, limit=mO.shape[1])
if const_expr(not self.use_out_indices and not self.seqlen_q_static_one):
assert not self.pack_gqa
pack_gqa = PackGQA(
self.m_block_size,
self.head_dim_v_padded,
self.check_hdim_v_oob,
self.qhead_per_kvhead,
)
# With seqlen_q statically one, PackGQA folds exactly
# qhead_per_kvhead rows, and since that is at most one
# m_block_size there is a single m_block and every reachable
# row lives in Q stage 0. Both the row bound and the dead
# stage are therefore compile-time constants.
reachable_rows = None
if const_expr(self.pack_gqa and self.seqlen_q_static_one):
assert self.qhead_per_kvhead <= self.m_block_size
# Replication spreads the same heads over the whole tile,
# one every q_replicate rows, so the last reachable row
# moves to the end of the tile.
reachable_rows = self.qhead_per_kvhead * self.q_replicate
for stage in cutlass.range_constexpr(self.q_stage):
# wait from corr, issue tma store on smem
# 1. wait for O0 / O1 final
iket.range_push("epi_wait_corr")
cute.arch.mbarrier_wait(
mbar_ptr + self.mbar_corr_epi_full_offset + stage, epi_consumer_phase
)
iket.range_pop()
# 2. copy O0 / O1 to gmem
stage_is_live = (
reachable_rows is None
or stage * self.m_block_size < reachable_rows
)
if const_expr(stage_is_live):
if const_expr(not self.pack_gqa):
# load acc O from smem to rmem for wider vectorization
tOrO = cute.make_fragment_like(
tOsO[None, None, None, 0], self.o_dtype
)
cute.autovec_copy(tOsO[None, None, None, stage], tOrO)
effective_seqlen_q_nonpack = (
Int32(1)
if const_expr(self.seqlen_q_static_one)
else seqlen.seqlen_q
)
for rest_m in cutlass.range_constexpr(cute.size(tOrO.shape[1])):
if (
t0OcO[0, rest_m, 0][0]
< effective_seqlen_q_nonpack
- (self.q_stage * m_block + stage) * self.m_block_size
- tOcO[0][0]
):
cute.copy(
gmem_tiled_copy_O,
tOrO[None, rest_m, None],
tOgO[
None,
rest_m,
None,
self.q_stage * m_block + stage,
],
pred=tOpO[None, rest_m, None]
if const_expr(self.check_hdim_v_oob)
else None,
)
else:
effective_seqlen_q = (
Int32(1)
if const_expr(self.use_out_indices or self.seqlen_q_static_one)
else seqlen.seqlen_q
)
pack_gqa.store_O(
mO_cur,
tOsO[None, None, None, stage],
gmem_tiled_copy_O,
tidx,
self.q_stage * m_block + stage,
effective_seqlen_q,
max_rows=reachable_rows,
stage_from_smem=True,
row_stride=self.q_replicate,
)
cute.arch.mbarrier_arrive(mbar_ptr + self.mbar_corr_epi_empty_offset + stage)
epi_consumer_phase ^= 1
# Advance to next tile
tile_scheduler.advance_to_next_work()
work_tile = tile_scheduler.get_current_work()
def load_Q(
self,
load_Q_fn: Callable,
mbar_full_ptr: cute.Pointer,
mbar_empty_ptr: cute.Pointer,
block: Int32,
stage: int,
phase: Int32,
load_SFQ_fn: Optional[Callable] = None,
load_Q_second_fn: Optional[Callable] = None,
load_SFQ_second_fn: Optional[Callable] = None,
):
iket.range_push("load_wait_q")
cute.arch.mbarrier_wait(mbar_empty_ptr + stage, phase)
iket.range_pop()
with cute.arch.elect_one():
cute.arch.mbarrier_arrive_and_expect_tx(mbar_full_ptr + stage, self.tma_copy_bytes["Q"])
load_Q_fn(
src_idx=block,
dst_idx=stage,
tma_bar_ptr=mbar_full_ptr + stage,
)
if const_expr(load_SFQ_fn is not None):
load_SFQ_fn(
src_idx=block,
dst_idx=stage,
tma_bar_ptr=mbar_full_ptr + stage,
)
if const_expr(self.query_terms == 2):
load_Q_second_fn(
src_idx=block,
dst_idx=stage,
tma_bar_ptr=mbar_full_ptr + stage,
)
if const_expr(load_SFQ_second_fn is not None):
load_SFQ_second_fn(
src_idx=block,
dst_idx=stage,
tma_bar_ptr=mbar_full_ptr + stage,
)
@cute.jit
def stack_Q_fp4_terms(
self,
sQ_uint8: cute.Tensor,
sQ_second: cute.Tensor,
sSFQ: cute.Tensor,
sSFQ_second: cute.Tensor,
mbar_full_ptr: cute.Pointer,
mbar_ready_ptr: cute.Pointer,
stage: int,
phase: Int32,
tidx: Int32,
):
"""Put the second term after the first term's live rows in SMEM.
Both full query tiles and both scale tiles have already arrived by
the time the load barrier completes. Their sizes and TMA transaction
counts stay unchanged. Only padding in the first tile is replaced;
the caller-owned scratch and its zero padding are never modified.
One load warp copies complete bytes and scales, fences its writes for
the Tensor Core/S2T consumers, then publishes one ready arrival.
"""
assert self.q_stage == 1
assert self.head_dim_padded == 128
cute.arch.mbarrier_wait(mbar_full_ptr + stage, phase)
rows = const_expr(self.transposed_query_rows)
# Pointer swizzles act on byte addresses, so the word view retains
# the byte view's SW(2,4,3); only its layout strides change units.
# Each aligned word preserves eight packed nibbles. Two rows fill a
# warp, with sixteen lanes moving the sixteen words of each row.
q_word_layout = cute.make_layout(
(self.m_block_size, 16), stride=(16, 1)
)
q_word_swizzle = cute.make_swizzle(2, 4, 3)
src_q = cute.make_tensor(
cute.recast_ptr(sQ_second.iterator, q_word_swizzle, cutlass.Uint32),
q_word_layout,
)
dst_q = cute.make_tensor(
cute.recast_ptr(sQ_uint8.iterator, q_word_swizzle, cutlass.Uint32),
q_word_layout,
)
for row_pair in cutlass.range_constexpr(rows // 2):
row = 2 * row_pair + tidx // 16
word = tidx % 16
dst_q[rows + row, word] = src_q[row, word]
src_sf = cute.recast_tensor(
sSFQ_second[None, None, None, stage], cutlass.Uint8
)
dst_sf = cute.recast_tensor(
sSFQ[None, None, None, stage], cutlass.Uint8
)
# The scale layout is unchanged; only lane ownership changes, from
# one row's lanes to a warp covering several rows at once. A row
# holds head_dim / sf_vec_size scales, sf_groups_per_mma_k per MMA K
# step: eight and four for NVFP4, four and two for MXFP4.
scale_count = const_expr(
rows * self.head_dim_padded // self.sf_vec_size
)
for scale_group in cutlass.range_constexpr(
-(-scale_count // cute.arch.WARP_SIZE)
):
scale_index = scale_group * cute.arch.WARP_SIZE + tidx
if const_expr(scale_count % cute.arch.WARP_SIZE):
# Fewer scales than lanes (MXFP4 at four rows): a spare lane
# repeats another lane's copy, the same byte to the same
# address.
scale_index = scale_index % scale_count
row = scale_index % rows
scale = scale_index // rows
mma_k = scale // self.sf_groups_per_mma_k
k_inst = scale % self.sf_groups_per_mma_k
dst = rows + row
dst_sf[(((dst % 32, dst // 32), 0), (0, k_inst)), 0, mma_k] = (
src_sf[(((row % 32, row // 32), 0), (0, k_inst)), 0, mma_k]
)
cute.arch.fence_view_async_shared()
cute.arch.sync_warp()
with cute.arch.elect_one():
cute.arch.mbarrier_arrive(mbar_ready_ptr + stage)
@cute.jit
def replicate_Q_fp4_rows(
self,
sQ_uint8: cute.Tensor,
sSFQ: cute.Tensor,
mbar_full_ptr: cute.Pointer, # mbar_load_q_full (FP4 TMA arrival)
mbar_ready_ptr: cute.Pointer, # mbar_q_fp4_ready
stage: int,
phase: Int32,
tidx: Int32,
):
"""Repeat each query row across its block of the Q tile, in place.
The TMA lands ``qhead_per_kvhead`` real rows and zero-fills the rest.
This rewrites row ``m`` from row ``m // q_replicate`` so every row of the
tile carries a query, which is what lets a softmax thread own a short
column range instead of a whole row.
Source rows are also destinations — row 1 feeds the second block and is
itself overwritten from row 0 — so the order matters. Each thread owns a
fixed set of byte columns for the whole tile, which removes any
cross-thread hazard, and writes block 0 last, after every other block
has consumed the source rows that live inside it.
"""
cute.arch.mbarrier_wait(mbar_full_ptr + stage, phase)
replicate = const_expr(self.q_replicate)
real_rows = const_expr(self.m_block_size // replicate)
bytes_per_row = const_expr(self.head_dim_padded // 2) # two FP4 per byte
sf_groups_per_row = const_expr(self.head_dim_padded // self.sf_vec_size)
num_threads = const_expr(len(self.load_warp_ids) * cute.arch.WARP_SIZE)
assert bytes_per_row % num_threads == 0
sQ_u8_stg = sQ_uint8[None, None, None, stage]
sSFQ_u8_stg = cute.recast_tensor(sSFQ[None, None, None, stage], cutlass.Uint8)
cols_per_thread = const_expr(bytes_per_row // num_threads)
# Descending so that block 0, the one holding every source row, is
# written after all of them have been read.
for r_rev in cutlass.range_constexpr(real_rows):
r = const_expr(real_rows - 1 - r_rev)
for c in cutlass.range_constexpr(cols_per_thread):
k_byte = tidx * cols_per_thread + c
atom_k = k_byte % 32
kH = k_byte // 32
val = sQ_u8_stg[(r, atom_k), 0, kH]
for i in cutlass.range_constexpr(replicate):
sQ_u8_stg[(r * replicate + i, atom_k), 0, kH] = val
if tidx < sf_groups_per_row:
mma_k = tidx // 4
k_inst = tidx % 4
sf_val = sSFQ_u8_stg[(((r % 32, r // 32), 0), (0, k_inst)), 0, mma_k]
for i in cutlass.range_constexpr(replicate):
m = r * replicate + i
sSFQ_u8_stg[(((m % 32, m // 32), 0), (0, k_inst)), 0, mma_k] = sf_val
cute.arch.fence_view_async_shared()
cute.arch.sync_warp()
with cute.arch.elect_one():
cute.arch.mbarrier_arrive(mbar_ready_ptr + stage)
@cute.jit
def load_KV(
self,
tma_atom: Optional[cute.CopyAtom],
tXgX: Optional[cute.Tensor],
tXsX: Optional[cute.Tensor],
paged_kv_manager: Optional[PagedKVManager],
sX: cute.Tensor,
mbar_full_ptr: cute.Pointer,
mbar_empty_ptr: cute.Pointer,
block: Int32,
producer_state: cutlass.pipeline.PipelineState,
K_or_V: Literal["K", "V"],
page_idx: Optional[Int32] = None,
tma_atom_sf: Optional[cute.CopyAtom] = None,
tXgSF: Optional[cute.Tensor] = None,
tXsSF: Optional[cute.Tensor] = None,
):
assert K_or_V in ("K", "V")
stage, phase = producer_state.index, producer_state.phase
iket.range_push("load_wait_kv")
cute.arch.mbarrier_wait(mbar_empty_ptr + stage, phase)
if const_expr(K_or_V == "K" and self.uneven_kv_smem):
# Before this round, the smem location was occupied by V, which is smaller than
# K. So we need to wait for the stage after that (stage 1) to be empty as well.
if stage == 0:
cute.arch.mbarrier_wait(mbar_empty_ptr + 1, phase)
iket.range_pop()
if const_expr(self.use_tma_KV):
assert (
tXgX is not None and
tXsX is not None and
tma_atom is not None
)
with cute.arch.elect_one():
cute.arch.mbarrier_arrive_and_expect_tx(
mbar_full_ptr + stage, self.tma_copy_bytes[K_or_V],
)
tXsX_cur = tXsX[None, stage]
if const_expr(self.uneven_kv_smem):
# Since this is the producer_state, the phase starts at 1, so we have to invert it
tXsX_cur = self.offset_kv_smem(tXsX_cur, stage, phase ^ 1)
# Currently we assume that page_size == n_block_size so we index into tXgX with block = 0
tXgX_cur = tXgX[None, block] if const_expr(page_idx is None) else tXgX[None, 0, page_idx]
cute.copy(tma_atom, tXgX_cur, tXsX_cur, tma_bar_ptr=mbar_full_ptr + stage)
else:
assert paged_kv_manager is not None
paged_kv_manager.load_KV(block, sX[None, None, None, stage], K_or_V)
cute.arch.cp_async_commit_group()
cute.arch.cp_async_mbarrier_arrive_noinc(mbar_full_ptr + stage)
# Load scale factor for K or V if provided (uses same barrier as K/V)
if const_expr(tma_atom_sf is not None and tXgSF is not None and tXsSF is not None):
tXsSF_cur = tXsSF[None, stage]
# After tma_partition with rank-1 grouping, tXgSF has structure: ((atom_v, rest_v), RestL)
if const_expr(page_idx is None):
tXgSF_cur = tXgSF[None, block]
else:
tXgSF_cur = tXgSF[None, 0, page_idx]
cute.copy(tma_atom_sf, tXgSF_cur, tXsSF_cur, tma_bar_ptr=mbar_full_ptr + stage)
def mainloop_s2t_copy_and_partition(
self,
sSF: cute.Tensor,
tSF: cute.Tensor,
) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor]:
"""
Make tiledCopy for smem to tmem load for scale factor tensor, then use it to partition smem memory (source) and tensor memory (destination).
:param sSF: The scale factor tensor in smem
:type sSF: cute.Tensor
:param tSF: The scale factor tensor in tmem
:type tSF: cute.Tensor
:return: A tuple containing (tiled_copy_s2t, tCsSF_compact_s2t, tCtSF_compact_s2t) where:
- tiled_copy_s2t: The tiled copy operation for smem to tmem load for scale factor tensor(s2t)
- tCsSF_compact_s2t: The partitioned scale factor tensor in smem
- tCtSF_compact_s2t: The partitioned scale factor tensor in tmem
:rtype: Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor]
"""
# (MMA, MMA_MN, MMA_K, STAGE)
tCsSF_compact = cute.filter_zeros(sSF)
# (MMA, MMA_MN, MMA_K)
tCtSF_compact = cute.filter_zeros(tSF)
# Make S2T CopyAtom and tiledCopy
copy_atom_s2t = cute.make_copy_atom(
tcgen05.Cp4x32x128bOp(self.cta_group),
self.sf_dtype,
)
tiled_copy_s2t = tcgen05.make_s2t_copy(copy_atom_s2t, tCtSF_compact)
thr_copy_s2t = tiled_copy_s2t.get_slice(0)
# ((ATOM_V, REST_V), Rest_Tiler, MMA_MN, MMA_K, STAGE)
tCsSF_compact_s2t_ = thr_copy_s2t.partition_S(tCsSF_compact)
# ((ATOM_V, REST_V), Rest_Tiler, MMA_MN, MMA_K, STAGE)
tCsSF_compact_s2t = tcgen05.get_s2t_smem_desc_tensor(
tiled_copy_s2t, tCsSF_compact_s2t_
)
# ((ATOM_V, REST_V), Rest_Tiler, MMA_MN, MMA_K)
tCtSF_compact_s2t = thr_copy_s2t.partition_D(tCtSF_compact)
return tiled_copy_s2t, tCsSF_compact_s2t, tCtSF_compact_s2t
@cute.jit
def offset_kv_smem(self, sX: cute.Tensor, stage: Int32, phase: Int32):
if const_expr(self.uneven_kv_smem):
# smem layout is [smem_large, smem_small, smem_large], and the current stride is
# (smem_large + smem_small) // 2. So for stage == 1, move right by offset if
# phase == 0, or left by offset if phase == 1.
offset = 0 if stage != 1 else self.uneven_kv_smem_offset * (1 - 2 * phase)
return cute.make_tensor(sX.iterator + offset, sX.layout)
else:
return sX
def make_and_init_load_kv_pipeline(self, load_kv_mbar_ptr, use_k_bytes: bool = True):
load_kv_consumer_group = cutlass.pipeline.CooperativeGroup(
cutlass.pipeline.Agent.Thread, len([self.mma_warp_id])
)
if self.use_tma_KV:
load_kv_producer_group = cutlass.pipeline.CooperativeGroup(
cutlass.pipeline.Agent.Thread, len(self.load_warp_ids)
)
return cutlass.pipeline.PipelineTmaUmma.create(
barrier_storage=load_kv_mbar_ptr,
num_stages=self.kv_stage,
producer_group=load_kv_producer_group,
consumer_group=load_kv_consumer_group,
tx_count=self.tma_copy_bytes["K"] if use_k_bytes else self.tma_copy_bytes["V"],
)
else:
load_kv_producer_group = cutlass.pipeline.CooperativeGroup(
cutlass.pipeline.Agent.Thread, len(self.load_warp_ids) * cute.arch.WARP_SIZE
)
return cutlass.pipeline.PipelineAsyncUmma.create(
num_stages=self.kv_stage,
producer_group=load_kv_producer_group,
consumer_group=load_kv_consumer_group,
barrier_storage=load_kv_mbar_ptr,
)
# @cute.jit
# def warp_scheduler_barrier_init(self):
# warp_group_idx = utils.canonical_warp_group_idx(sync=False)
# if warp_group_idx == 0:
# cute.arch.barrier_arrive(
# barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1), number_of_threads=2 * 128,
# )
# def warp_scheduler_barrier_sync(self):
# cute.arch.barrier(
# barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1) + utils.canonical_warp_group_idx(sync=False),
# number_of_threads=2 * 128
# )
# def warp_scheduler_barrier_arrive(self):
# cur_wg = utils.canonical_warp_group_idx(sync=False)
# next_wg = 1 - cur_wg
# cute.arch.barrier_arrive(
# barrier_id=int(NamedBarrierFwd.WarpSchedulerWG1) + next_wg, number_of_threads=2 * 128,
# )
@cute.jit
def apply_score_mod(
self,
tSrS_t2r,
thr_tmem_load,
thr_mma_qk,
batch_idx,
head_idx,
m_block,
n_block,
softmax,
aux_tensors=None,
fastdiv_mods=(None, None),
):
"""Apply score modification for SM100 (constant q_idx)."""
# Prepare index tensor with extra partition
cS = cute.make_identity_tensor((self.m_block_size, self.n_block_size))
cS = cute.domain_offset((m_block * self.m_block_size, n_block * self.n_block_size), cS)
tScS = thr_mma_qk.partition_C(cS)
tScS_t2r = thr_tmem_load.partition_D(tScS)
# Shared q_idx for all scores
q_idx_logical = tScS_t2r[0][0]
# For Pack-GQA, compute the logical head index for this tile
if cutlass.const_expr(self.pack_gqa):
# Building up the logical q_head idx: final_q_head = kv_head * qhead_per_kvhead + (q_physical % qhead_per_kvhead)
q_physical = q_idx_logical
q_idx_logical = q_physical // self.qhead_per_kvhead
head_offset = q_physical - q_idx_logical * self.qhead_per_kvhead
head_idx = head_idx * self.qhead_per_kvhead + head_offset
if cutlass.const_expr(aux_tensors is not None):
seqlen_q_divmod, _ = fastdiv_mods
_, q_idx_logical = seqlen_q_divmod.divmod(q_idx_logical)
apply_score_mod_inner(
tSrS_t2r,
tScS_t2r,
self.score_mod,
batch_idx,
head_idx,
softmax.softmax_scale,
self.vec_size,
self.qk_acc_dtype,
aux_tensors,
fastdiv_mods,
constant_q_idx=q_idx_logical,
qhead_per_kvhead=self.qhead_per_kvhead if cutlass.const_expr(self.pack_gqa) else 1,
)