Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python

h3: optional FlashAttention Hopper kernels for dense attention

henry-y5a4f50857f98
diff --git a/src/solarwm/backends/minimax_h3/fsdp.py b/src/solarwm/backends/minimax_h3/fsdp.py
index 48eb93f..7b421c4 100644
--- a/src/solarwm/backends/minimax_h3/fsdp.py
+++ b/src/solarwm/backends/minimax_h3/fsdp.py
@@ -129,6 +129,8 @@ def _attention_ops() -> frozenset:
for name in ("_flash_attn_forward", "_flash_attn_varlen_forward"):
if hasattr(flash, name):
ops.add(getattr(flash, name).default)
+ if hasattr(torch.ops.solar_hopper, "forward"):
+ ops.add(torch.ops.solar_hopper.forward.default)
return frozenset(ops)
diff --git a/src/solarwm/backends/minimax_h3/hopper_attention.py b/src/solarwm/backends/minimax_h3/hopper_attention.py
new file mode 100644
index 0000000..71e93f5
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_attention.py
@@ -0,0 +1,109 @@
+"""Dense bf16 H3 attention using the locally shipped FlashAttention Hopper kernels."""
+
+from __future__ import annotations
+
+import functools
+import os
+import tempfile
+from pathlib import Path
+
+import torch
+
+
+def initialize():
+ """Compile only the supported bf16/128 specialization, once per process group."""
+ from torch.utils.cpp_extension import load
+
+ source = Path(__file__).with_name("hopper_src")
+ # CUTLASS headers: the NGC PyTorch image ships them with its FlashAttention sources.
+ cutlass = Path(os.environ.get("SOLAR_CUTLASS_INCLUDE")
+ or "/opt/pytorch/pytorch/third_party/flash-attention/csrc/cutlass/include")
+ if not cutlass.is_dir():
+ raise RuntimeError(f"Hopper attention needs the CUTLASS headers (not found at {cutlass}); "
+ "point SOLAR_CUTLASS_INCLUDE at a CUTLASS include directory")
+ root = Path(os.environ.get("TORCH_EXTENSIONS_DIR")
+ or os.environ.get("TORCHINDUCTOR_CACHE_DIR") or tempfile.gettempdir())
+ build = root / "solar_hopper128"
+ build.mkdir(parents=True, exist_ok=True)
+ disabled = ("SPLIT", "PAGEDKV", "APPENDKV", "LOCAL", "SOFTCAP", "PACKGQA",
+ "FP16", "FP8", "VARLEN", "HDIM64", "HDIM96", "HDIM192", "HDIM256", "SM8x")
+ defines = [f"-DFLASHATTENTION_DISABLE_{name}" for name in disabled]
+ if os.environ.get("SOLAR_HOPPER_FWD192") == "1":
+ defines.append("-DSOLAR_HOPPER_FWD192")
+ os.environ.setdefault("MAX_JOBS", "4")
+ return load(
+ name="solar_hopper128_cuda",
+ sources=[str(source / name) for name in (
+ "flash_api.cpp", "flash_fwd_hdim128_bf16_sm90.cu", "flash_bwd_hdim128_bf16_sm90.cu")],
+ extra_include_paths=[str(source), str(cutlass)],
+ extra_cflags=["-O3", "-std=c++17", *defines],
+ extra_cuda_cflags=["-O3", "-std=c++17", "--use_fast_math", "--threads=2",
+ "-gencode=arch=compute_90a,code=sm_90a",
+ "-DCUTE_SM90_EXTENDED_MMA_SHAPES_ENABLED", "-DNDEBUG", *defines],
+ build_directory=str(build),
+ )
+
+
[email protected]_op("solar_hopper::forward", mutates_args=(), device_types="cuda")
+def _forward(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ q, k, v = (x.contiguous() if x.stride(-1) != 1 else x for x in (q, k, v))
+ out, lse, *_ = initialize().fwd(
+ q, k, v, None, None, None, None, None, None, None, None, None, None,
+ None, None, None, None, None, None, None, None,
+ q.shape[-1] ** -0.5, False, -1, -1, 0, 0.0, True, 1, False, 0,
+ )
+ return out, lse
+
+
+@_forward.register_fake
+def _forward_fake(q, k, v):
+ return (torch.empty(q.shape, device=q.device, dtype=q.dtype),
+ torch.empty((q.shape[0], q.shape[2], q.shape[1]), device=q.device, dtype=torch.float32))
+
+
[email protected]_op("solar_hopper::backward", mutates_args=(), device_types="cuda")
+def _backward(dout: torch.Tensor, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
+ out: torch.Tensor, lse: torch.Tensor, deterministic: bool
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ dout, q, k, v, out = (x.contiguous() if x.stride(-1) != 1 else x for x in (dout, q, k, v, out))
+ dq, dk, dv = (torch.empty(x.shape, dtype=x.dtype, device=x.device) for x in (q, k, v))
+ initialize().bwd(dout, q, k, v, out, lse, dq, dk, dv,
+ None, None, None, None, None, None,
+ q.shape[-1] ** -0.5, False, -1, -1, 0, 0.0, deterministic, 0)
+ return dq, dk, dv
+
+
+@_backward.register_fake
+def _backward_fake(dout, q, k, v, out, lse, deterministic):
+ return tuple(torch.empty(x.shape, device=x.device, dtype=x.dtype) for x in (q, k, v))
+
+
+def _setup_context(ctx, inputs, output):
+ q, k, v = inputs
+ out, lse = output
+ ctx.save_for_backward(q, k, v, out, lse)
+ ctx.mark_non_differentiable(lse)
+ ctx.deterministic = torch.are_deterministic_algorithms_enabled()
+
+
+def _autograd_backward(ctx, dout, dlse):
+ return _backward(dout, *ctx.saved_tensors, ctx.deterministic)
+
+
+torch.library.register_autograd("solar_hopper::forward", _autograd_backward, setup_context=_setup_context)
+
+
+def attention(q, k, v, *, attn_mask=None):
+ """Full attention with no dropout; bf16 I/O, fp32 softmax and accumulation."""
+ if attn_mask is not None:
+ raise ValueError("Hopper H3 backend supports dense unmasked attention only")
+ if any(x.dtype != torch.bfloat16 or x.ndim != 4 or x.shape[-1] != 128 for x in (q, k, v)):
+ raise ValueError("Hopper H3 backend requires bf16 BSHD inputs with head dimension 128")
+ if torch.are_deterministic_algorithms_enabled():
+ # The image's Hopper revision has a non-repeatable dQ even with its
+ # deterministic flag. FA2 keeps the same bf16 I/O and fp32 math.
+ from flash_attn import flash_attn_func
+
+ return flash_attn_func(q, k, v, dropout_p=0.0, deterministic=True)
+ return _forward(q, k, v)[0]
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/AUTHORS b/src/solarwm/backends/minimax_h3/hopper_src/AUTHORS
new file mode 100644
index 0000000..e35a781
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/AUTHORS
@@ -0,0 +1 @@
\ No newline at end of file
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/LICENSE b/src/solarwm/backends/minimax_h3/hopper_src/LICENSE
new file mode 100644
index 0000000..5860e4b
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/LICENSE
@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2022, the respective contributors, as shown by the AUTHORS file.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/README.md b/src/solarwm/backends/minimax_h3/hopper_src/README.md
new file mode 100644
index 0000000..82479d2
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/README.md
@@ -0,0 +1,6 @@
+FlashAttention Hopper (FlashAttention 3.0.0.b1, `hopper/`) sources as shipped in the NGC PyTorch 25.06 container
+(`third_party/flash-attention`), BSD-3-Clause; see LICENSE and AUTHORS. The two `*_hdim128_bf16_sm90.cu` files are
+the upstream `hopper/instantiations` files and `kernel_traits.h` comes from `csrc/flash_attn/src`. The one change is
+in `tile_size.h`: an opt-in 192x128 forward tile for dense non-causal attention (`SOLAR_HOPPER_FWD192=1`).
+Only the bf16, head dimension 128, dense forward/backward kernels are compiled. CUTLASS headers are not included here;
+see `hopper_attention.py`.
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/combine.h b/src/solarwm/backends/minimax_h3/hopper_src/combine.h
new file mode 100644
index 0000000..c26f7ea
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/combine.h
@@ -0,0 +1,248 @@
+
+#pragma once
+
+#include <cute/tensor.hpp>
+
+#include <cutlass/cutlass.h>
+#include "cutlass/layout/layout.h"
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+
+#include "kernel_traits.h"
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <class Element, class SmemShape, class SmemShapeMaxSplits>
+struct SharedStorageLSE {
+ cute::array_aligned<Element, cute::size_v<SmemShape>> smem_lse;
+ cute::array_aligned<bool, cute::size_v<SmemShapeMaxSplits>> smem_valid_splits;
+};
+
+// DONT use Kernel_traits here to avoid redundant compilation.
+// template<typename Kernel_traits, int kBlockM, int Log_max_splits, bool Is_even_K, typename Params>
+template<typename Element, typename ElementAccum, int kHeadDim, int kBlockM, int Log_max_splits, bool Is_even_K, typename Params>
+__global__ void combine_attn_seqk_parallel(Params const params) {
+ // using Element = typename Kernel_traits::OutputType;
+ // using ElementAccum = typename Kernel_traits::ElementAccum;
+ using index_t = int64_t; // Kernel_traits::index_t
+ constexpr int kMaxSplits = 1 << Log_max_splits;
+ // constexpr int kHeadDim = Kernel_traits::kHeadDim;
+ constexpr int kNThreads = 128; //Kernel_traits::kNThreads;
+
+ static_assert(kMaxSplits <= 128, "kMaxSplits must be <= 128");
+ static_assert(kBlockM == 4 || kBlockM == 8 || kBlockM == 16 || kBlockM == 32, "kBlockM must be 4, 8, 16 or 32");
+ static_assert(kNThreads == 128, "We assume that each block has 128 threads");
+
+ // Shared memory.
+ // kBlockM + 1 instead of kBlockM to reduce bank conflicts.
+ //__shared__ __align__(16) ElementAccum sLSE[kMaxSplits][kBlockM+1];
+ extern __shared__ char smem_[];
+ using SharedStorage = SharedStorageLSE<ElementAccum, Shape<Int<kMaxSplits>, Int<kBlockM+1>>, Shape<Int<kMaxSplits>>>;
+ SharedStorage &shared_storage =
+ *reinterpret_cast<SharedStorage *>(smem_);
+ Tensor sLSE = make_tensor(make_smem_ptr(shared_storage.smem_lse.data()), Shape<Int<kMaxSplits>, Int<kBlockM+1>>{});
+ Tensor sValidSplits = make_tensor(make_smem_ptr(shared_storage.smem_valid_splits.data()), Shape<Int<kMaxSplits>>{});
+
+ // The thread and block index.
+ const int tidx = threadIdx.x;
+ const int bidx = blockIdx.x;
+
+ const index_t lse_size = params.b * params.h * params.seqlen_q;
+ //if (cute::thread0()) print ("final %d %d %d %d\n", params.b, params.h, params.seqlen_q, params.b * params.h * params.seqlen_q);
+
+ const index_t row_offset_lse = bidx * kBlockM;
+ Tensor gLSEaccum = make_tensor(make_gmem_ptr(reinterpret_cast<ElementAccum *>(params.softmax_lseaccum_ptr) + row_offset_lse),
+ Shape<Int<kMaxSplits>, Int<kBlockM>>{},
+ make_stride(lse_size, _1{}));
+
+ // LSE format is different depending on params.unpadded_lse and params.seqlenq_ngroups_swapped, see comment in get_lse_tile.
+ // This tensor's layout maps row_offset_lse to {bidb, bidh, q_offset}.
+ Tensor gLSE = make_tensor(make_gmem_ptr(reinterpret_cast<ElementAccum *>(params.softmax_lse_ptr) + row_offset_lse),
+ Shape<Int<kBlockM>>{}, Stride<_1>{});
+
+ // This layout maps row_offset_lse to {bidh, q_offset, bidb} or {bidh, bidb, q_offset}.
+ Layout flat_layout = make_layout(lse_size);
+ Layout orig_layout = make_layout(make_shape(params.seqlen_q, params.h, params.b));
+ auto transposed_stride = params.seqlenq_ngroups_swapped ? make_stride(params.b, params.seqlen_q * params.b, 1) : make_stride(1, params.seqlen_q * params.b, params.seqlen_q);
+ Layout remapped_layout = make_layout(make_shape(params.seqlen_q, params.h, params.b), transposed_stride);
+ Layout final_layout = cute::composition(remapped_layout, cute::composition(orig_layout, flat_layout));
+
+ Tensor gLSE_unpadded = make_tensor(make_gmem_ptr(reinterpret_cast<ElementAccum *>(params.softmax_lse_ptr)), final_layout);
+
+ constexpr int kNLsePerThread = (kMaxSplits * kBlockM + kNThreads - 1) / kNThreads;
+
+ // Read the LSE values from gmem and store them in shared memory, then transpose them.
+ constexpr int kRowsPerLoadLSE = kNThreads / kBlockM;
+ #pragma unroll
+ for (int l = 0; l < kNLsePerThread; ++l) {
+ const int row = l * kRowsPerLoadLSE + tidx / kBlockM;
+ const int col = tidx % kBlockM;
+ ElementAccum lse = (row < params.num_splits && col < lse_size - bidx * kBlockM) ? gLSEaccum(row, col) : -INFINITY;
+ if (row < kMaxSplits) { sLSE(row,col) = lse; }
+ // if (bidx == 0 && tidx < 32) { printf("tidx = %d, row = %d, col = %d, lse = %f\n", tidx, row, col, lse); }
+ }
+ __syncthreads();
+
+ // Reduce along the kBlockM dimension to determine valid splits (store in SMEM)
+ // One thread per split. Know NumThreads = 128 >= NumMaxSplits
+ if (tidx < kMaxSplits) {
+ bool is_valid_split = false;
+ #pragma unroll
+ for (int col = 0; col < kBlockM; ++col) {
+ if(sLSE(tidx,col) != -INFINITY) {
+ is_valid_split = true;
+ }
+ }
+ sValidSplits(tidx) = is_valid_split;
+ }
+ __syncthreads();
+ // if (bidx == 1 && tidx < 32) { printf("tidx = %d, row_offset_lse = %d, lse = %f\n", tidx, row_offset_lse, lse_accum(0)); }
+
+ Tensor lse_accum = make_tensor<ElementAccum>(Shape<Int<kNLsePerThread>>{});
+ constexpr int kRowsPerLoadTranspose = std::min(kRowsPerLoadLSE, kMaxSplits);
+ // To make sure that kMaxSplits is within 1 warp: we decide how many elements within kMaxSplits
+ // each thread should hold. If kMaxSplits = 16, then each thread holds 2 elements (128 threads,
+ // kBlockM rows, so each time we load we can load 128 / kBlockM rows).
+ // constexpr int kThreadsPerSplit = kMaxSplits / kRowsPerLoadTranspose;
+ // static_assert(kThreadsPerSplit <= 32);
+ static_assert(kRowsPerLoadTranspose <= 32);
+ static_assert(kNLsePerThread * kRowsPerLoadTranspose <= kMaxSplits);
+ #pragma unroll
+ for (int l = 0; l < kNLsePerThread; ++l) {
+ const int row = l * kRowsPerLoadTranspose + tidx % kRowsPerLoadTranspose;
+ const int col = tidx / kRowsPerLoadTranspose;
+ //if (bidx == 0 && tidx < 128) { printf("tidx = %d, row = %d, col = %d, lse = %f\n", tidx, row, col, lse_accum(l)); }
+ lse_accum(l) = (row < kMaxSplits && col < kBlockM) ? sLSE(row,col) : -INFINITY;
+
+ }
+ //return;
+
+ // Compute the logsumexp of the LSE along the split dimension.
+ ElementAccum lse_max = lse_accum(0);
+ #pragma unroll
+ for (int l = 1; l < kNLsePerThread; ++l) { lse_max = max(lse_max, lse_accum(l)); }
+ MaxOp<float> max_op;
+ lse_max = Allreduce<kRowsPerLoadTranspose>::run(lse_max, max_op);
+ lse_max = lse_max == -INFINITY ? 0.0f : lse_max; // In case all local LSEs are -inf
+ float lse_sum = expf(lse_accum(0) - lse_max);
+ #pragma unroll
+ for (int l = 1; l < kNLsePerThread; ++l) { lse_sum += expf(lse_accum(l) - lse_max); }
+ SumOp<float> sum_op;
+ lse_sum = Allreduce<kRowsPerLoadTranspose>::run(lse_sum, sum_op);
+ // For the case where all local lse == -INFINITY, we want to set lse_logsum to INFINITY. Otherwise
+ // lse_logsum is log(0.0) = -INFINITY and we get NaN when we do lse_accum(l) - lse_logsum.
+ ElementAccum lse_logsum = (lse_sum == 0.f || lse_sum != lse_sum) ? INFINITY : logf(lse_sum) + lse_max;
+ // if (bidx == 0 && tidx < 32) { printf("tidx = %d, lse = %f, lse_max = %f, lse_logsum = %f\n", tidx, lse_accum(0), lse_max, lse_logsum); }
+ if (tidx % kRowsPerLoadTranspose == 0 && tidx / kRowsPerLoadTranspose < kBlockM) {
+ if (params.unpadded_lse) {
+ const index_t lse_offset = row_offset_lse + tidx / kRowsPerLoadTranspose;
+ if (lse_offset < lse_size) {
+ gLSE_unpadded(lse_offset) = lse_logsum;
+ }
+ } else {
+ gLSE(tidx / kRowsPerLoadTranspose) = lse_logsum;
+ }
+ }
+ //if (cute::thread0()) printf ("lse_logsum = %f\n", lse_logsum);
+
+ // Store the scales exp(lse - lse_logsum) in shared memory.
+ #pragma unroll
+ for (int l = 0; l < kNLsePerThread; ++l) {
+ const int row = l * kRowsPerLoadTranspose + tidx % kRowsPerLoadTranspose;
+ const int col = tidx / kRowsPerLoadTranspose;
+ if (row < params.num_splits && col < kBlockM) { sLSE(row,col) = expf(lse_accum(l) - lse_logsum); }
+ }
+ __syncthreads();
+
+ const index_t row_offset_oaccum = bidx * kBlockM * params.d_rounded;
+ Tensor gOaccum = make_tensor(make_gmem_ptr(reinterpret_cast<ElementAccum *>(params.oaccum_ptr) + row_offset_oaccum),
+ Shape<Int<kBlockM>, Int<kHeadDim>>{},
+ Stride<Int<kHeadDim>, _1>{});
+ constexpr int kBlockN = kNThreads / kBlockM;
+ using GmemLayoutAtomOaccum = Layout<Shape<Int<kBlockM>, Int<kBlockN>>, Stride<Int<kBlockN>, _1>>;
+ using GmemTiledCopyOaccum = decltype(
+ make_tiled_copy(Copy_Atom<DefaultCopy, ElementAccum>{},
+ GmemLayoutAtomOaccum{},
+ Layout<Shape < _1, _4>>{})); // Val layout, 4 vals per store
+ GmemTiledCopyOaccum gmem_tiled_copy_Oaccum;
+ auto gmem_thr_copy_Oaccum = gmem_tiled_copy_Oaccum.get_thread_slice(tidx);
+ Tensor tOgOaccum = gmem_thr_copy_Oaccum.partition_S(gOaccum);
+ Tensor tOrO = make_tensor<ElementAccum>(shape(tOgOaccum));
+ Tensor tOrOaccum = make_tensor<ElementAccum>(shape(tOgOaccum));
+ clear(tOrO);
+
+ // Predicates
+ Tensor cOaccum = make_identity_tensor(Shape<Int<kBlockM>, Int<kHeadDim>>{});
+ //if (cute::thread0()) print_tensor (cOaccum);
+ // Repeat the partitioning with identity layouts
+ Tensor tOcOaccum = gmem_thr_copy_Oaccum.partition_S(cOaccum);
+ Tensor tOpOaccum = make_tensor<bool>(make_shape(size<2>(tOgOaccum)));
+ if (!Is_even_K) {
+ #pragma unroll
+ for (int k = 0; k < size(tOpOaccum); ++k) { tOpOaccum(k) = get<1>(tOcOaccum(0, 0, k)) < params.d; }
+ }
+ // Load Oaccum in then scale and accumulate to O
+ for (int split = 0; split < params.num_splits; ++split) {
+ // DONT copy in Oaccum if lse(split) = -inf for all kBlockM.
+ if(sValidSplits(split)) {
+ flash::copy</*Is_even_MN=*/false, Is_even_K>(
+ gmem_tiled_copy_Oaccum, tOgOaccum, tOrOaccum, tOcOaccum, tOpOaccum, params.b * params.h * params.seqlen_q - bidx * kBlockM
+ );
+ #pragma unroll
+ for (int m = 0; m < size<1>(tOrOaccum); ++m) {
+ int row = get<0>(tOcOaccum(0, m, 0));
+ ElementAccum lse_scale = sLSE(split,row);
+ if (lse_scale != 0.f) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(tOrOaccum); ++k) {
+ #pragma unroll
+ for (int i = 0; i < size<0>(tOrOaccum); ++i) {
+ tOrO(i, m, k) += lse_scale * tOrOaccum(i, m, k);
+ //tOrO(i, m, k) += tOrOaccum(i, m, k);
+ }
+ }
+ }
+ //if (cute::thread0()) { printf("lse_scale = %f, %f\n", sLSE(split, 0), sLSE(split, 1)); print_tensor(tOrOaccum); }
+ }
+ }
+ tOgOaccum.data() = tOgOaccum.data() + params.b * params.h * params.seqlen_q * params.d_rounded;
+ }
+ //if (cute::thread0()) { print_tensor(tOrO); }
+
+ Tensor rO = flash::convert_type<Element>(tOrO);
+ // Write to gO
+ #pragma unroll
+ for (int m = 0; m < size<1>(rO); ++m) {
+ const int idx = bidx * kBlockM + get<0>(tOcOaccum(0, m, 0));
+ //if (cute::thread0()) print ("final %d %d %d %d %d\n", idx, params.b, params.h, params.seqlen_q, params.b * params.h * params.seqlen_q);
+ if (idx < params.b * params.h * params.seqlen_q) {
+ //print ("final2\n");
+ const int batch_idx = idx / (params.h * params.seqlen_q);
+ const int head_idx = (idx - batch_idx * (params.h * params.seqlen_q)) / params.seqlen_q;
+ // The index to the rows of Q
+ const int row = idx - batch_idx * (params.h * params.seqlen_q) - head_idx * params.seqlen_q;
+ auto o_ptr = reinterpret_cast<Element *>(params.o_ptr) + batch_idx * params.o_batch_stride
+ + head_idx * params.o_head_stride + row * params.o_row_stride;
+ #pragma unroll
+ for (int k = 0; k < size<2>(rO); ++k) {
+ if (Is_even_K || tOpOaccum(k)) {
+ const int col = get<1>(tOcOaccum(0, m, k));
+ Tensor gO = make_tensor(make_gmem_ptr(o_ptr + col),
+ Shape<Int<decltype(size<0>(rO))::value>>{}, Stride<_1>{});
+ // TODO: Should check if this is using vectorized store, but it seems pretty fast
+ copy(rO(_, m, k), gO);
+ //if (cute::thread0()) { print ("final\n"); print_tensor(gO); }
+ // if (bidx == 0 && tidx == 0) { printf("tidx = %d, idx = %d, batch_idx = %d, head_idx = %d, row = %d, col = %d\n", tidx, idx, batch_idx, head_idx, row, col); print(rO(_, m, k)); print(gO); }
+ // reinterpret_cast<uint64_t *>(o_ptr)[col / 4] = recast<uint64_t>(rO)(0, m, k);
+ }
+ }
+ }
+ }
+}
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/copy_sm90_bulk_reduce.hpp b/src/solarwm/backends/minimax_h3/hopper_src/copy_sm90_bulk_reduce.hpp
new file mode 100644
index 0000000..8556fae
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/copy_sm90_bulk_reduce.hpp
@@ -0,0 +1,49 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include<cute/arch/copy_sm90_tma.hpp>
+
+namespace cute
+{
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+struct SM90_BULK_REDUCE_ADD
+{
+ CUTE_HOST_DEVICE static void
+ copy(float const* smem_ptr,
+ float * gmem_ptr, int32_t store_bytes)
+ {
+#if defined(CUTE_ARCH_TMA_SM90_ENABLED)
+ uint32_t smem_int_ptr = cast_smem_ptr_to_uint(smem_ptr);
+ asm volatile("cp.reduce.async.bulk.global.shared::cta.bulk_group.add.f32 [%0], [%1], %2;\n"
+ :
+ : "l"(gmem_ptr), "r"(smem_int_ptr), "r"(store_bytes)
+ : "memory");
+#else
+ CUTE_INVALID_CONTROL_PATH("Trying to use BULK_REDUCE_ADD without CUTE_ARCH_TMA_SM90_ENABLED.");
+#endif
+ }
+
+ CUTE_HOST_DEVICE static void
+ copy(float const* smem_ptr,
+ float * gmem_ptr, int32_t store_bytes, uint64_t cache_hint)
+ {
+#if defined(CUTE_ARCH_TMA_SM90_ENABLED)
+ uint32_t smem_int_ptr = cast_smem_ptr_to_uint(smem_ptr);
+ asm volatile("cp.reduce.async.bulk.global.shared::cta.bulk_group.L2::cache_hint.add.f32 [%0], [%1], %2, %3;\n"
+ :
+ : "l"(gmem_ptr), "r"(smem_int_ptr), "r"(store_bytes), "l"(cache_hint)
+ : "memory");
+#else
+ CUTE_INVALID_CONTROL_PATH("Trying to use BULK_REDUCE_ADD without CUTE_ARCH_TMA_SM90_ENABLED.");
+#endif
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // end namespace cute
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/epilogue_bwd.hpp b/src/solarwm/backends/minimax_h3/hopper_src/epilogue_bwd.hpp
new file mode 100644
index 0000000..f99dfe9
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/epilogue_bwd.hpp
@@ -0,0 +1,523 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cutlass/cutlass.h>
+#include <cutlass/barrier.h>
+#include "cute/tensor.hpp"
+
+#include "cutlass/gemm/collective/builders/sm90_common.inl"
+
+#include "seqlen.h"
+#include "named_barrier.hpp"
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <class TileShape_MNK_, class Element_, class ArchTag_,
+ int NumEpilogueThreads_, bool Varlen_, bool dKV_swapAB_, int AtomLayoutKdKV=1>
+struct CollectiveEpilogueBwd {
+
+ using TileShape_MNK = TileShape_MNK_;
+ using Element = Element_;
+ using ArchTag = ArchTag_;
+ static constexpr int NumEpilogueThreads = NumEpilogueThreads_;
+ static constexpr bool Varlen = Varlen_;
+ static constexpr bool dKV_swapAB = dKV_swapAB_;
+ static constexpr bool Use_TMA = !Varlen && ArchTag::kMinComputeCapability >= 90;
+
+ static_assert(ArchTag::kMinComputeCapability >= 80);
+
+ using GmemTiledCopydKVTMA = cute::SM90_TMA_STORE;
+
+ // These are for storing the output tensor without TMA (e.g., for setting output to zero)
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(Element);
+ static_assert(get<2>(TileShape_MNK{}) % kGmemElemsPerLoad == 0, "Headdim must be a multiple of kGmemElemsPerLoad");
+ static constexpr int kHeadDim = get<2>(TileShape_MNK{});
+ static constexpr int kGmemThreadsPerRow = cutlass::gcd(kHeadDim / kGmemElemsPerLoad, NumEpilogueThreads);
+ static_assert(NumEpilogueThreads % kGmemThreadsPerRow == 0, "NumEpilogueThreads must be a multiple of kGmemThreadsPerRow");
+ using GmemLayoutAtom = Layout<Shape <Int<NumEpilogueThreads / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+ using GmemTiledCopydKV = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per store
+
+ using SmemLayoutAtomdKVTMA = decltype(cutlass::gemm::collective::detail::ss_smem_selector<GMMA::Major::K, Element,
+ // TODO: do we have to change this if dKV_swapAB is true?
+ decltype(cute::get<1>(TileShape_MNK{})), Int<CUTE_STATIC_V(cute::get<2>(TileShape_MNK{})) / AtomLayoutKdKV>>());
+ using SmemLayoutdKVTMA = decltype(tile_to_shape(SmemLayoutAtomdKVTMA{}, select<1, 2>(TileShape_MNK{})));
+ using SmemLayoutdKVtTMA =
+ decltype(cute::composition(SmemLayoutdKVTMA{},
+ make_layout(make_shape(get<2>(TileShape_MNK{}), get<1>(TileShape_MNK{})),
+ make_stride(decltype(get<1>(TileShape_MNK{})){}, _1{}))));
+
+ // If we don't use TMA
+ static constexpr int kBlockKSmem = kHeadDim % 64 == 0 ? 64 : (kHeadDim % 32 == 0 ? 32 : 16);
+ static constexpr int kSwizzle = kBlockKSmem == 64 ? 3 : (kBlockKSmem == 32 ? 2 : 1);
+ using SmemLayoutAtomdKVSTG =
+ decltype(composition(Swizzle<kSwizzle, 3, 3>{},
+ Layout<Shape<Int<8>, Int<kBlockKSmem>>,
+ Stride<Int<kBlockKSmem>, _1>>{}));
+
+ using SmemLayoutAtomdKV = std::conditional_t<Use_TMA, SmemLayoutAtomdKVTMA, SmemLayoutAtomdKVSTG>;
+ using SmemLayoutdKV = decltype(tile_to_shape(SmemLayoutAtomdKV{}, select<1, 2>(TileShape_MNK{})));
+ using SmemLayoutdKVt =
+ decltype(cute::composition(SmemLayoutdKV{},
+ make_layout(make_shape(get<2>(TileShape_MNK{}), get<1>(TileShape_MNK{})),
+ make_stride(decltype(get<1>(TileShape_MNK{})){}, _1{}))));
+
+ using SmemCopyAtomdKV = Copy_Atom<
+ std::conditional_t<
+ ArchTag::kMinComputeCapability >= 90,
+ std::conditional_t<!dKV_swapAB, cute::SM90_U32x4_STSM_N, cute::SM90_U16x8_STSM_T>,
+ AutoVectorizingCopyWithAssumedAlignment<128>
+ >,
+ Element>;
+
+ static constexpr size_t SmemAlignmentdKV = ArchTag::kMinComputeCapability >= 90 ? cutlass::detail::alignment_for_swizzle(SmemLayoutdKV{}) : 128;
+ static_assert(SmemAlignmentdKV >= 128, "Require at least 128B alignment");
+
+ struct TensorStorage : cute::aligned_struct<SmemAlignmentdKV> {
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutdKV>, SmemAlignmentdKV> smem_dk;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutdKV>, SmemAlignmentdKV> smem_dv;
+ };
+
+ using ShapedKV = cute::Shape<int32_t, int32_t, int32_t, int32_t>; // (seqlen_k, d, head, batch)
+ using StridedKV = cute::Stride<int64_t, _1, int64_t, int64_t>;
+
+ using TMA_dKV = std::conditional_t<
+ Use_TMA,
+ decltype(make_tma_copy(
+ GmemTiledCopydKVTMA{},
+ make_tensor(make_gmem_ptr(static_cast<Element*>(nullptr)), ShapedKV{}, StridedKV{}),
+ SmemLayoutdKVTMA{},
+ select<1, 2>(TileShape_MNK{}),
+ _1{})), // no mcast for dKV
+ std::nullptr_t
+ >;
+
+ // Host side kernel arguments
+ struct Arguments {
+ Element* ptr_dK;
+ ShapedKV const shape_dK;
+ StridedKV const stride_dK;
+ Element* ptr_dV;
+ StridedKV const stride_dV;
+ int const num_heads_q;
+ int* dk_semaphore;
+ int* dv_semaphore;
+ int const* cu_seqlens;
+ int const* seqused;
+ };
+
+ // Device side kernel params
+ struct Params {
+ Element* ptr_dK;
+ ShapedKV const shape_dK;
+ StridedKV const stride_dK;
+ Element* ptr_dV;
+ StridedKV const stride_dV;
+ TMA_dKV tma_store_dK, tma_store_dV;
+ int const* cu_seqlens = nullptr;
+ int const* seqused = nullptr;
+ };
+
+ static Params
+ to_underlying_arguments(Arguments const& args) {
+ Tensor mdK = make_tensor(make_gmem_ptr(args.ptr_dK), args.shape_dK, args.stride_dK);
+ Tensor mdV = make_tensor(make_gmem_ptr(args.ptr_dV), args.shape_dK, args.stride_dV);
+ TMA_dKV tma_store_dK = [&] {
+ if constexpr (Use_TMA) {
+ return make_tma_copy(GmemTiledCopydKVTMA{}, mdK, SmemLayoutdKVTMA{}, select<1, 2>(TileShape_MNK{}), _1{}); // no mcast for dKV
+ } else {
+ return nullptr;
+ }
+ }();
+ TMA_dKV tma_store_dV = [&] {
+ if constexpr (Use_TMA) {
+ return make_tma_copy(GmemTiledCopydKVTMA{}, mdV, SmemLayoutdKVTMA{}, select<1, 2>(TileShape_MNK{}), _1{}); // no mcast for dKV
+ } else {
+ return nullptr;
+ }
+ }();
+ return {args.ptr_dK, args.shape_dK, args.stride_dK, args.ptr_dV, args.stride_dV,
+ tma_store_dK, tma_store_dV, args.cu_seqlens, args.seqused};
+ }
+
+ /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best performance
+ CUTLASS_DEVICE
+ static void prefetch_tma_descriptors(Params const& params) {
+ if constexpr (Use_TMA) {
+ cute::prefetch_tma_descriptor(params.tma_store_dK.get_tma_descriptor());
+ cute::prefetch_tma_descriptor(params.tma_store_dV.get_tma_descriptor());
+ }
+ }
+
+ template <typename SharedStorage, typename FrgTensorO, typename TiledMma>
+ CUTLASS_DEVICE void
+ store(Params const& params,
+ FrgTensorO const& tdKrdK,
+ FrgTensorO const& tdVrdV,
+ SharedStorage& shared_storage,
+ TiledMma tiled_mma,
+ int thread_idx,
+ cute::tuple<int32_t, int32_t, int32_t> const& block_coord
+ ) {
+
+ auto [n_block, bidh, bidb] = block_coord;
+ Tensor sdK = cute::as_position_independent_swizzle_tensor(make_tensor(make_smem_ptr(shared_storage.tensors.epilogue.smem_dk.data()), SmemLayoutdKV{}));
+ Tensor sdV = cute::as_position_independent_swizzle_tensor(make_tensor(make_smem_ptr(shared_storage.tensors.epilogue.smem_dv.data()), SmemLayoutdKV{}));
+ Tensor sdKt = cute::as_position_independent_swizzle_tensor(make_tensor(make_smem_ptr(shared_storage.tensors.epilogue.smem_dk.data()), SmemLayoutdKVt{}));
+ Tensor sdVt = cute::as_position_independent_swizzle_tensor(make_tensor(make_smem_ptr(shared_storage.tensors.epilogue.smem_dv.data()), SmemLayoutdKVt{}));
+ auto smem_tiled_copy_dKV = make_tiled_copy_C(SmemCopyAtomdKV{}, tiled_mma);
+ auto smem_thr_copy_dKV = smem_tiled_copy_dKV.get_thread_slice(thread_idx);
+
+ Tensor tdVrdV_out = make_tensor_like<Element>(tdVrdV);
+ flash::convert_type_out(tdVrdV, tdVrdV_out);
+ Tensor tdKrdK_out = make_tensor_like<Element>(tdKrdK);
+ flash::convert_type_out(tdKrdK, tdKrdK_out);
+ Tensor taccdKrdK = smem_thr_copy_dKV.retile_S(tdKrdK_out); // ((Atom,AtomNum), MMA_M, MMA_N)
+ Tensor taccdVrdV = smem_thr_copy_dKV.retile_S(tdVrdV_out); // ((Atom,AtomNum), MMA_M, MMA_N)
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print(smem_thr_copy_dKV); print(sdK); printf("\n"); print(sdKt); printf("\n"); }
+ Tensor taccdKsdK = smem_thr_copy_dKV.partition_D(cute::conditional_return<!dKV_swapAB>(sdK, sdKt)); // ((Atom,AtomNum),PIPE_M,PIPE_N)
+ Tensor taccdVsdV = smem_thr_copy_dKV.partition_D(cute::conditional_return<!dKV_swapAB>(sdV, sdVt)); // ((Atom,AtomNum),PIPE_M,PIPE_N)
+
+ // Make sure all WGs have finished reading K and V
+ flash::named_barrier_sync(NumEpilogueThreads, cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+ cute::copy(smem_tiled_copy_dKV, taccdVrdV, taccdVsdV);
+ cute::copy(smem_tiled_copy_dKV, taccdKrdK, taccdKsdK);
+ if constexpr (Use_TMA) {
+ cutlass::arch::fence_view_async_shared(); // ensure smem writes are visible to TMA
+ cutlass::arch::NamedBarrier::arrive(NumEpilogueThreads + cutlass::NumThreadsPerWarp,
+ cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+
+ Tensor mdK = params.tma_store_dK.get_tma_tensor(params.shape_dK);
+ Tensor mdV = params.tma_store_dV.get_tma_tensor(params.shape_dK);
+ Tensor gdK = local_tile(mdK(_, _, bidh, bidb), select<1, 2>(TileShape_MNK{}), make_coord(n_block, _0{})); // (M, K)
+ Tensor gdV = local_tile(mdV(_, _, bidh, bidb), select<1, 2>(TileShape_MNK{}), make_coord(n_block, _0{})); // (M, K)
+ auto block_tma_dK = params.tma_store_dK.get_slice(_0{});
+ auto block_tma_dV = params.tma_store_dV.get_slice(_0{});
+ Tensor tdKgdK = block_tma_dK.partition_D(gdK); // (TMA, TMA_M, TMA_K)
+ Tensor tdKsdK = block_tma_dK.partition_S(sdK); // (TMA, TMA_M, TMA_K)
+ Tensor tdVgdV = block_tma_dV.partition_D(gdV); // (TMA, TMA_M, TMA_K)
+ Tensor tdVsdV = block_tma_dV.partition_S(sdV); // (TMA, TMA_M, TMA_K)
+ int warp_idx_sync = __shfl_sync(0xffffffff, thread_idx / cutlass::NumThreadsPerWarp, 0);
+ if (warp_idx_sync == NumEpilogueThreads / cutlass::NumThreadsPerWarp - 1) {
+ cutlass::arch::NamedBarrier::sync(NumEpilogueThreads + cutlass::NumThreadsPerWarp,
+ cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+ if (cute::elect_one_sync()) {
+ cute::copy(params.tma_store_dV, tdVsdV, tdVgdV);
+ cute::copy(params.tma_store_dK, tdKsdK, tdKgdK);
+ tma_store_arrive();
+ }
+ }
+ tma_store_wait<0>();
+ // // Tell warp 0 that smem_k and smem_v are ready
+ // cutlass::arch::NamedBarrier::arrive(NumEpilogueThreads + cutlass::NumThreadsPerWarp, static_cast<uint32_t>(BwdNamedBarriers::KVEmpty) /*id*/);
+
+ } else {
+ flash::named_barrier_sync(NumEpilogueThreads, cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ flash::SeqlenInfo<Varlen, kBlockN> seqlen_info{bidb, size<0>(params.shape_dK), params.cu_seqlens, params.seqused};
+ bool const is_varlen = Varlen && params.cu_seqlens;
+ Tensor mdK = make_tensor(make_gmem_ptr(params.ptr_dK), params.shape_dK, params.stride_dK)(_, _, bidh, !is_varlen ? bidb : 0);
+ Tensor gdK = local_tile(cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdK), select<1, 2>(TileShape_MNK{}), make_coord(n_block, _0{})); // (M, K)
+ Tensor mdV = make_tensor(make_gmem_ptr(params.ptr_dV), params.shape_dK, params.stride_dV)(_, _, bidh, !is_varlen ? bidb : 0);
+ Tensor gdV = local_tile(cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdV), select<1, 2>(TileShape_MNK{}), make_coord(n_block, _0{})); // (M, K)
+
+ GmemTiledCopydKV gmem_tiled_copy_dKV;
+ auto gmem_thr_copy_dKV = gmem_tiled_copy_dKV.get_thread_slice(thread_idx);
+ Tensor tdKVgdV = gmem_thr_copy_dKV.partition_D(gdV);
+ Tensor tdKVsdV = gmem_thr_copy_dKV.partition_S(sdV); // (TMA, TMA_M, TMA_K)
+ Tensor tdKVgdK = gmem_thr_copy_dKV.partition_D(gdK);
+ Tensor tdKVsdK = gmem_thr_copy_dKV.partition_S(sdK); // (TMA, TMA_M, TMA_K)
+ Tensor tdKVrdV = make_fragment_like(tdKVgdV);
+ Tensor tdKVrdK = make_fragment_like(tdKVgdK);
+ Tensor cdKV = cute::make_identity_tensor(select<1, 2>(TileShape_MNK{})); // (BLK_M,BLK_K) -> (blk_m,blk_k)
+ // Repeat the partitioning with identity layouts
+ Tensor tdKVcdKV = gmem_thr_copy_dKV.partition_D(cdKV);
+ Tensor tdKVpdKV = make_tensor<bool>(make_shape(size<2>(tdKVgdV)));
+ #pragma unroll
+ for (int k = 0; k < size(tdKVpdKV); ++k) { tdKVpdKV(k) = get<1>(tdKVcdKV(_0{}, _0{}, k)) < get<1>(params.shape_dK); }
+ // Need to check OOB when reading from smem if kBlockN isn't evenly tiled
+ static constexpr bool EvenN = kBlockN % CUTE_STATIC_V(size<0>(GmemLayoutAtom{})) == 0;
+ flash::copy</*Is_even_MN=*/EvenN, /*Is_even_K=*/true, /*Clear_OOB_MN=*/false>(
+ gmem_tiled_copy_dKV, tdKVsdV, tdKVrdV, tdKVcdKV, tdKVpdKV, kBlockN);
+ flash::copy</*Is_even_MN=*/EvenN, /*Is_even_K=*/true, /*Clear_OOB_MN=*/false>(
+ gmem_tiled_copy_dKV, tdKVsdK, tdKVrdK, tdKVcdKV, tdKVpdKV, kBlockN);
+ // // Tell warp 0 that smem_k and smem_v are ready
+ // cutlass::arch::fence_view_async_shared(); // ensure smem reads are done before next TMA to smem_k/v
+ // flash::named_barrier_arrive(NumEpilogueThreads + cutlass::NumThreadsPerWarp, static_cast<uint32_t>(BwdNamedBarriers::KVEmpty) /*id*/);
+ // Construct identity layout for gdKV
+ // Clear_OOB_K must be false since we don't want to write zeros to gmem
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/false>(
+ gmem_tiled_copy_dKV, tdKVrdV, tdKVgdV, tdKVcdKV, tdKVpdKV, std::min(seqlen_info.seqlen - n_block * kBlockN, kBlockN)
+ );
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/false>(
+ gmem_tiled_copy_dKV, tdKVrdK, tdKVgdK, tdKVcdKV, tdKVpdKV, std::min(seqlen_info.seqlen - n_block * kBlockN, kBlockN)
+ );
+ }
+ }
+
+ CUTLASS_DEVICE void
+ store_tail() {
+ // if constexpr (Use_TMA) { tma_store_wait<0>(); }
+ }
+
+ // Write 0 to dK and dV
+ CUTLASS_DEVICE void
+ store_zero(
+ Params const& params,
+ int thread_idx,
+ cute::tuple<int32_t, int32_t, int32_t> const& block_coord
+ ) {
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ auto [n_block, bidh, bidb] = block_coord;
+ flash::SeqlenInfo<Varlen, kBlockN> seqlen_info{bidb, size<0>(params.shape_dK), params.cu_seqlens, params.seqused};
+ bool const is_varlen = Varlen && params.cu_seqlens;
+ Tensor mdK = make_tensor(make_gmem_ptr(params.ptr_dK), params.shape_dK, params.stride_dK)(_, _, bidh, !is_varlen ? bidb : 0);
+ Tensor gdK = local_tile(cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdK), select<1, 2>(TileShape_MNK{}), make_coord(n_block, _0{})); // (M, K)
+ Tensor mdV = make_tensor(make_gmem_ptr(params.ptr_dV), params.shape_dK, params.stride_dV)(_, _, bidh, !is_varlen ? bidb : 0);
+ Tensor gdV = local_tile(cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdV), select<1, 2>(TileShape_MNK{}), make_coord(n_block, _0{})); // (M, K)
+
+ GmemTiledCopydKV gmem_tiled_copy_dKV;
+ auto gmem_thr_copy_dKV = gmem_tiled_copy_dKV.get_thread_slice(thread_idx);
+ Tensor tdKVgdK = gmem_thr_copy_dKV.partition_D(gdK);
+ Tensor tdKVgdV = gmem_thr_copy_dKV.partition_D(gdV);
+ Tensor tdKVrdKV = make_fragment_like(tdKVgdK);
+ clear(tdKVrdKV);
+ // Construct identity layout for gdKV
+ Tensor cdKV = cute::make_identity_tensor(select<1, 2>(TileShape_MNK{})); // (BLK_M,BLK_K) -> (blk_m,blk_k)
+ // Repeat the partitioning with identity layouts
+ Tensor tdKVcdKV = gmem_thr_copy_dKV.partition_D(cdKV);
+ Tensor tdKVpdKV = make_tensor<bool>(make_shape(size<2>(tdKVgdK)));
+ #pragma unroll
+ for (int k = 0; k < size(tdKVpdKV); ++k) { tdKVpdKV(k) = get<1>(tdKVcdKV(_0{}, _0{}, k)) < get<1>(params.shape_dK); }
+ // Clear_OOB_K must be false since we don't want to write zeros to gmem
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/false>(
+ gmem_tiled_copy_dKV, tdKVrdKV, tdKVgdK, tdKVcdKV, tdKVpdKV, seqlen_info.seqlen - n_block * kBlockN
+ );
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/false>(
+ gmem_tiled_copy_dKV, tdKVrdKV, tdKVgdV, tdKVcdKV, tdKVpdKV, seqlen_info.seqlen - n_block * kBlockN
+ );
+ }
+
+};
+
+template <class TileShape_MNK_, class ElementAccum, class ArchTag_,
+ int NumEpilogueThreads_, bool Varlen_, bool Deterministic>
+struct CollectiveEpilogueBwdGQA {
+
+ using TileShape_MNK = TileShape_MNK_;
+ using Element = ElementAccum;
+ using ArchTag = ArchTag_;
+ static constexpr int NumEpilogueThreads = NumEpilogueThreads_;
+ static constexpr bool Varlen = Varlen_;
+ static constexpr bool Use_TMA = ArchTag::kMinComputeCapability >= 90;
+
+ static_assert(ArchTag::kMinComputeCapability >= 80);
+
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ static constexpr int kHeadDim = get<2>(TileShape_MNK{});
+ static_assert(NumEpilogueThreads % cutlass::NumThreadsPerWarp == 0, "NumEpilogueThreads must be a multiple of NumThreadsPerWarp");
+ static constexpr int NumWarpGroups = NumEpilogueThreads / cutlass::NumThreadsPerWarpGroup;
+ // Thread layout, 256 or 384 threads per row
+ // We split into NumWarpGroups so that we can use the same postprocessing kernel as dQ
+ using R2SLayoutAtomdKVaccum = Layout<Shape<Int<cutlass::NumThreadsPerWarpGroup>, Int<NumWarpGroups>>>;
+ using R2STiledCopydKVaccum = decltype(make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementAccum>{}, R2SLayoutAtomdKVaccum{},
+ Layout<Shape < _4>>{})); // Val layout, 4 vals per store
+ // For Sm80
+ using R2GLayoutAtomdKVaccum = Layout<Shape<Int<NumEpilogueThreads>>>;
+ using R2GTiledCopydKVaccum = decltype(make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementAccum>{}, R2GLayoutAtomdKVaccum{},
+ Layout<Shape < _1>>{})); // Val layout, 1 vals per store
+
+ using SmemLayoutdKVaccum = Layout<Shape<Int<kBlockN * kHeadDim / NumWarpGroups>, Int<NumWarpGroups>>>;
+ using SmemLayoutdKVaccumFlat = Layout<Shape<Int<kBlockN * kHeadDim>>>;
+
+ // Strangely without this SmemAlignment, the total smem for hdim 128 (80 x 128) is 228KB even though we
+ // only need 227KB. We use the same alignment as the non-GQA epilogue to avoid this issue.
+ static constexpr int SmemAlignment = kHeadDim % 64 == 0 ? 1024 : (kHeadDim % 32 == 0 ? 512 : 256);
+ struct TensorStorageTMA : cute::aligned_struct<SmemAlignment> {
+ cute::array_aligned<ElementAccum, cute::cosize_v<SmemLayoutdKVaccum>, SmemAlignment> smem_dkv;
+ };
+ struct TensorStorageSTG {
+ cute::array<ElementAccum, 0> smem_dkv;
+ };
+ using TensorStorage = std::conditional_t<Use_TMA, TensorStorageTMA, TensorStorageSTG>;
+
+ using ShapedKV = cute::Shape<int32_t, int32_t, int32_t>; // (seqlen_k_rounded * d, head, batch)
+ using StridedKV = cute::Stride<_1, int64_t, int64_t>;
+
+ // Host side kernel arguments
+ struct Arguments {
+ ElementAccum* ptr_dKaccum;
+ ShapedKV const shape_dKaccum;
+ StridedKV const stride_dKaccum;
+ ElementAccum* ptr_dVaccum;
+ StridedKV const stride_dVaccum;
+ int num_heads_q;
+ int* dk_semaphore;
+ int* dv_semaphore;
+ int const* cu_seqlens;
+ int const* seqused;
+ };
+
+ // Device side kernel params
+ struct Params {
+ ElementAccum* ptr_dKaccum;
+ ShapedKV const shape_dKaccum;
+ StridedKV const stride_dKaccum;
+ ElementAccum* ptr_dVaccum;
+ StridedKV const stride_dVaccum;
+ cutlass::FastDivmod qhead_per_khead_divmod;
+ int* dk_semaphore;
+ int* dv_semaphore;
+ int const* cu_seqlens = nullptr;
+ int const* seqused = nullptr;
+ };
+
+ static Params
+ to_underlying_arguments(Arguments const& args) {
+ if constexpr (Deterministic) {
+ assert(args.dk_semaphore != nullptr);
+ assert(args.dv_semaphore != nullptr);
+ }
+ return {args.ptr_dKaccum, args.shape_dKaccum, args.stride_dKaccum, args.ptr_dVaccum, args.stride_dVaccum,
+ cutlass::FastDivmod(cute::ceil_div(args.num_heads_q, get<1>(args.shape_dKaccum))),
+ args.dk_semaphore, args.dv_semaphore,
+ args.cu_seqlens, args.seqused};
+ }
+
+ /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best performance
+ CUTLASS_DEVICE
+ static void prefetch_tma_descriptors(Params const& params) {
+ }
+
+ template <typename SharedStorage, typename FrgTensorO, typename TiledMma>
+ CUTLASS_DEVICE void
+ store(Params const& params,
+ FrgTensorO const& tdKrdK,
+ FrgTensorO const& tdVrdV,
+ SharedStorage& shared_storage,
+ TiledMma tiled_mma,
+ int thread_idx,
+ cute::tuple<int32_t, int32_t, int32_t> const& block_coord
+ ) {
+
+ auto [n_block, bidh, bidb] = block_coord;
+ int bidh_idx_in_group;
+ int bidh_kv = params.qhead_per_khead_divmod.divmod(bidh_idx_in_group, bidh);
+ Tensor sdKV = make_tensor(make_smem_ptr(shared_storage.tensors.epilogue.smem_dkv.data()), SmemLayoutdKVaccum{});
+ Tensor sdKV_flat = make_tensor(make_smem_ptr(shared_storage.tensors.epilogue.smem_dkv.data()), SmemLayoutdKVaccumFlat{});
+ static constexpr int dKV_TMA_num_bytes = CUTE_STATIC_V(size(sdKV_flat)) * sizeof(ElementAccum);
+
+ flash::SeqlenInfo<Varlen, kBlockN> seqlen_info{bidb, size<0>(params.shape_dKaccum), params.cu_seqlens, params.seqused};
+ bool const is_varlen = Varlen && params.cu_seqlens;
+ Tensor mdKaccum = make_tensor(make_gmem_ptr(params.ptr_dKaccum), params.shape_dKaccum, params.stride_dKaccum)(_, bidh_kv, !is_varlen ? bidb : 0);
+ Tensor mdVaccum = make_tensor(make_gmem_ptr(params.ptr_dVaccum), params.shape_dKaccum, params.stride_dVaccum)(_, bidh_kv, !is_varlen ? bidb : 0);
+ Tensor gdKaccum = local_tile(domain_offset(make_coord(seqlen_info.offset_padded * kHeadDim), mdKaccum), Shape<Int<kBlockN * kHeadDim>>{}, make_coord(n_block)); // (M * K)
+ Tensor gdVaccum = local_tile(domain_offset(make_coord(seqlen_info.offset_padded * kHeadDim), mdVaccum), Shape<Int<kBlockN * kHeadDim>>{}, make_coord(n_block)); // (M * K)
+
+ R2STiledCopydKVaccum r2s_tiled_copy_dKVaccum;
+ auto r2s_thr_copy_dKVaccum = r2s_tiled_copy_dKVaccum.get_thread_slice(thread_idx);
+ Tensor tdKVsdKVaccum = r2s_thr_copy_dKVaccum.partition_D(sdKV);
+
+ // Only used if !Use_TMA
+ R2GTiledCopydKVaccum r2g_tiled_copy_dKVaccum;
+ auto r2g_thr_copy_dKVaccum = r2g_tiled_copy_dKVaccum.get_thread_slice(thread_idx);
+
+ // Make sure all WGs have finished reading K and V, otherwise we get racy dQ
+ // because smem_q could be changed.
+ flash::named_barrier_sync(NumEpilogueThreads, cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+ if constexpr (Use_TMA) {
+ Tensor taccdKVrdV = r2s_thr_copy_dKVaccum.retile_S(tdVrdV); // ((Atom,AtomNum), MMA_M, MMA_N)
+ cute::copy(r2s_tiled_copy_dKVaccum, taccdKVrdV, tdKVsdKVaccum);
+ }
+
+ // int const num_batch = params.num_batch;
+ int const num_batch = get<2>(params.shape_dKaccum);
+ int const num_head_kv = get<1>(params.shape_dKaccum);
+ int *lock_ptr = !Deterministic ? nullptr : params.dv_semaphore + bidb * num_head_kv + bidh_kv;
+ using Barrier = cutlass::GenericBarrier<cutlass::detail::SyncwarpSync>;
+
+ // if (thread_idx == 0) { printf("blockIdx.x = %d, blockIdx.y = %d, blockIdx.z = %d, bidb = %d, bidh_kv = %d, lock_ptr = %p, dv_semaphore = %p, num_batch = %d, num_head_kv = %d, n_block = %d, bihd_idx_in_group = %d\n", blockIdx.x, blockIdx.y, blockIdx.z, bidb, bidh_kv, lock_ptr, params.dv_semaphore, num_batch, num_head_kv, n_block, bidh_idx_in_group);}
+
+ if constexpr (Deterministic) {
+ Barrier::wait_eq(lock_ptr, thread_idx, n_block * num_batch * num_head_kv, bidh_idx_in_group);
+ }
+ // if (thread_idx == 0) { printf("After barrier blockIdx.x = %d, blockIdx.y = %d, blockIdx.z = %d, bidb = %d, bidh_kv = %d, lock_ptr = %p, dv_semaphore = %p\n", blockIdx.x, blockIdx.y, blockIdx.z, bidb, bidh_kv, lock_ptr, params.dv_semaphore);}
+ if constexpr (Use_TMA) {
+ cutlass::arch::fence_view_async_shared();
+ cutlass::arch::NamedBarrier::sync(NumEpilogueThreads, cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+ if (thread_idx == 0) {
+ SM90_BULK_REDUCE_ADD::copy(raw_pointer_cast(sdKV_flat.data()), raw_pointer_cast(gdVaccum.data()), dKV_TMA_num_bytes, static_cast<uint64_t>(TMA::CacheHintSm90::EVICT_LAST));
+ tma_store_arrive();
+ tma_store_wait<0>();
+ }
+ } else {
+ Tensor tdVrdV_atomic = r2g_thr_copy_dKVaccum.retile_S(tdVrdV);
+ Tensor tdVgdV_atomic = r2g_thr_copy_dKVaccum.partition_D(gdVaccum);
+ static_assert(CUTE_STATIC_V(size(tdVrdV_atomic)) == CUTE_STATIC_V(size(tdVgdV_atomic)));
+ #pragma unroll
+ for (int i = 0; i < size(tdVrdV_atomic); ++i) { atomicAdd(&tdVgdV_atomic(i), tdVrdV_atomic(i)); }
+ }
+ if constexpr (Deterministic) {
+ Barrier::arrive_inc(lock_ptr, thread_idx, n_block * num_batch * num_head_kv);
+ }
+
+ if constexpr (Use_TMA) {
+ cutlass::arch::NamedBarrier::sync(NumEpilogueThreads, cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+ Tensor taccdKVrdK = r2s_thr_copy_dKVaccum.retile_S(tdKrdK); // ((Atom,AtomNum), MMA_M, MMA_N)
+ cute::copy(r2s_tiled_copy_dKVaccum, taccdKVrdK, tdKVsdKVaccum);
+ }
+ lock_ptr = !Deterministic ? nullptr : params.dk_semaphore + bidb * num_head_kv + bidh_kv;
+ // if (thread_idx == 0) { printf("blockIdx.x = %d, blockIdx.y = %d, blockIdx.z = %d, bidb = %d, bidh_kv = %d, lock_ptr = %p, dk_semaphore = %p, num_batch = %d, num_head_kv = %d, n_block = %d, bihd_idx_in_group = %d\n", blockIdx.x, blockIdx.y, blockIdx.z, bidb, bidh_kv, lock_ptr, params.dk_semaphore, num_batch, num_head_kv, n_block, bidh_idx_in_group);}
+
+ if constexpr (Deterministic) {
+ Barrier::wait_eq(lock_ptr, thread_idx, n_block * num_batch * num_head_kv, bidh_idx_in_group);
+ }
+ // if (thread_idx == 0) { printf("After barrier blockIdx.x = %d, blockIdx.y = %d, blockIdx.z = %d, bidb = %d, bidh_kv = %d, lock_ptr = %p, dk_semaphore = %p\n", blockIdx.x, blockIdx.y, blockIdx.z, bidb, bidh_kv, lock_ptr, params.dk_semaphore);}
+ if constexpr (Use_TMA) {
+ cutlass::arch::fence_view_async_shared();
+ cutlass::arch::NamedBarrier::sync(NumEpilogueThreads, cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+ if (thread_idx == 0) {
+ SM90_BULK_REDUCE_ADD::copy(raw_pointer_cast(sdKV_flat.data()), raw_pointer_cast(gdKaccum.data()), dKV_TMA_num_bytes, static_cast<uint64_t>(TMA::CacheHintSm90::EVICT_LAST));
+ tma_store_arrive();
+ tma_store_wait<0>();
+ }
+ } else {
+ Tensor tdKrdK_atomic = r2g_thr_copy_dKVaccum.retile_S(tdKrdK);
+ Tensor tdKgdK_atomic = r2g_thr_copy_dKVaccum.partition_D(gdKaccum);
+ static_assert(CUTE_STATIC_V(size(tdKrdK_atomic)) == CUTE_STATIC_V(size(tdKgdK_atomic)));
+ #pragma unroll
+ for (int i = 0; i < size(tdKrdK_atomic); ++i) { atomicAdd(&tdKgdK_atomic(i), tdKrdK_atomic(i)); }
+ }
+ if constexpr (Deterministic) {
+ Barrier::arrive_inc(lock_ptr, thread_idx, n_block * num_batch * num_head_kv);
+ }
+ // // Tell warp 0 that smem_k and smem_v are ready
+ // flash::named_barrier_arrive(NumEpilogueThreads + cutlass::NumThreadsPerWarp, static_cast<uint32_t>(BwdNamedBarriers::KVEmpty) /*id*/);
+ }
+
+ CUTLASS_DEVICE void
+ store_tail() {
+ }
+
+ // Write 0 to dK and dV
+ CUTLASS_DEVICE void
+ store_zero(
+ Params const& params,
+ int thread_idx,
+ cute::tuple<int32_t, int32_t, int32_t> const& block_coord
+ ) {
+ // Don't need to do anything since dKaccum and dVaccum are already zero-initialized
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/epilogue_fwd.hpp b/src/solarwm/backends/minimax_h3/hopper_src/epilogue_fwd.hpp
new file mode 100644
index 0000000..0f91606
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/epilogue_fwd.hpp
@@ -0,0 +1,419 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cutlass/cutlass.h>
+#include <cutlass/fast_math.h> // For FastDivMod
+#include "cute/tensor.hpp"
+
+#include "cutlass/gemm/collective/builders/sm90_common.inl"
+#include "cutlass/epilogue/collective/builders/sm90_common.inl"
+
+#include "seqlen.h"
+#include "named_barrier.hpp"
+#include "pack_gqa.h"
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <class TileShape_MNK_, class ClusterShape_, class Element_, class ArchTag_,
+ int NumEpilogueThreads_, bool Varlen_, bool PackGQA_, bool FP8PermuteCol=false>
+struct CollectiveEpilogueFwd {
+
+ using TileShape_MNK = TileShape_MNK_;
+ using ClusterShape = ClusterShape_;
+ using Element = Element_;
+ using ArchTag = ArchTag_;
+ static constexpr int NumEpilogueThreads = NumEpilogueThreads_;
+ static constexpr bool Varlen = Varlen_;
+ static constexpr bool PackGQA = PackGQA_;
+ static constexpr bool Use_smem = sizeof(Element) <= 2;
+ static constexpr bool Use_TMA_O = ArchTag::kMinComputeCapability >= 90 && !Varlen && Use_smem && !PackGQA;
+
+ static_assert(ArchTag::kMinComputeCapability >= 80);
+ static_assert(ArchTag::kMinComputeCapability >= 90 || CUTE_STATIC_V(size(ClusterShape{})) == 1);
+
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kHeadDim = get<2>(TileShape_MNK{});
+
+ using GmemTiledCopyOTMA = cute::SM90_TMA_STORE;
+
+ // These are for storing the output tensor without TMA (e.g., for setting output to zero)
+ static constexpr int kGmemElemsPerStore = sizeof(cute::uint128_t) / sizeof(Element);
+ static_assert(kHeadDim % kGmemElemsPerStore == 0, "Headdim must be a multiple of kGmemElemsPerStore");
+ // We want each "row" to have 64 elements (128 bytes, i.e. 1 cache line). We want each thread to have 4 elements
+ // in the M direction and 2 elements in the K direction. In the case of PackGQA, this reduces the number of times
+ // we need to call divmod.
+ static constexpr int kBytePerRow = kHeadDim * sizeof(Element);
+ static constexpr int kBlockKGmem = (kBytePerRow % 128 == 0 ? 128 : (kBytePerRow % 64 == 0 ? 64 : 32)) / sizeof(Element);
+ // static constexpr int kBlockKGmem = kHeadDim % 128 == 0 ? 128 : (kHeadDim % 64 == 0 ? 64 : 32);
+ // static constexpr int kGmemThreadsPerRow = cutlass::gcd(kHeadDim / kGmemElemsPerStore, NumEpilogueThreads);
+ static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerStore;
+ // If PackGQA, we split the work of compute O_ptr among threads in the same row, so we need this to within a warp
+ static_assert(cutlass::NumThreadsPerWarp % kGmemThreadsPerRow == 0);
+ static_assert(NumEpilogueThreads % kGmemThreadsPerRow == 0, "NumEpilogueThreads must be a multiple of kGmemThreadsPerRow");
+ using GmemLayoutAtom = Layout<Shape <Int<NumEpilogueThreads / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+ static_assert(kBlockM % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0, "kBlockM must be a multiple of NumEpilogueThreads / kGmemThreadsPerRow");
+ using GmemTiledCopyO = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerStore>>>{})); // Val layout, 8 or 16 vals per store
+
+ using SmemLayoutAtomOTMA = decltype(cutlass::gemm::collective::detail::ss_smem_selector<GMMA::Major::K, Element,
+ decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{}))>());
+ using SmemLayoutOTMA = decltype(tile_to_shape(SmemLayoutAtomOTMA{}, select<0, 2>(TileShape_MNK{})));
+ static constexpr int kSwizzle = kBlockKGmem == 128 ? 4 : (kBlockKGmem == 64 ? 3 : (kBlockKGmem == 32 ? 2 : 1));
+ static constexpr int kSwizzleBase = sizeof(Element) == 4 ? 2 : (sizeof(Element) == 2 ? 3 : 4);
+ using SmemLayoutAtomO = decltype(
+ composition(Swizzle<kSwizzle, kSwizzleBase, kSwizzleBase>{},
+ Layout<Shape<_8, Int<kBlockKGmem>>,
+ Stride<Int<kBlockKGmem>, _1>>{}));
+ using SmemLayoutOSTS = decltype(tile_to_shape(SmemLayoutAtomO{}, select<0, 2>(TileShape_MNK{})));
+ using SmemLayoutO = std::conditional_t<ArchTag::kMinComputeCapability >= 90, SmemLayoutOTMA, SmemLayoutOSTS>;
+
+ using ShapeO = cute::Shape<int32_t, int32_t, int32_t, int32_t, int32_t>; // (seqlen_q, d, head, batch, num_splits)
+ using StrideO = cute::Stride<int64_t, _1, int64_t, int64_t, int64_t>;
+ using StrideLSE = cute::Stride<_1, int64_t, int64_t, int64_t>; // (seqlen_q, head, batch, num_splits)
+ // ((qhead_per_khead, seqlen_q), d, nheads_kv, batch, num_splits)
+ using ShapeOPacked = std::conditional_t<!PackGQA, ShapeO, cute::Shape<cute::Shape<int32_t, int32_t>, int32_t, int32_t, int32_t, int32_t>>;
+ using StrideOPacked = std::conditional_t<!PackGQA, StrideO, cute::Stride<cute::Stride<int64_t, int64_t>, _1, int64_t, int64_t, int64_t>>;
+ // ((qhead_per_khead, seqlen_q), nheads_kv, batch, num_splits)
+ using ShapeLSEPacked = std::conditional_t<!PackGQA, cute::Shape<int32_t, int32_t, int32_t, int32_t>, cute::Shape<cute::Shape<int32_t, int32_t>, int32_t, int32_t, int32_t>>;
+ using StrideLSEPacked = std::conditional_t<!PackGQA, StrideLSE, cute::Stride<cute::Stride<int64_t, _1>, int64_t, int64_t, int64_t>>;
+
+ using CopyOpR2S = std::conditional_t<
+ ArchTag::kMinComputeCapability >= 90,
+ // cute::SM90_U32x4_STSM_N if Element size is 2 bytes (fp16, bf16)
+ decltype(cutlass::epilogue::collective::detail::sm90_get_smem_store_op_for_accumulator<StrideO, Element>()),
+ AutoVectorizingCopyWithAssumedAlignment<128>
+ >;
+ using SmemCopyAtomO = Copy_Atom<CopyOpR2S, Element>;
+
+ // static constexpr size_t SmemAlignmentO = cutlass::detail::alignment_for_swizzle(SmemLayoutO{});
+ // static_assert(SmemAlignmentO >= 128, "Require at least 128B alignment");
+ // struct TensorStorage : cute::aligned_struct<SmemAlignmentO> {
+ // cute::array_aligned<Element, Use_smem ? cute::cosize_v<SmemLayoutO> : 0, SmemAlignmentO> smem_o;
+ // };
+ struct TensorStorage : cute::aligned_struct<128> {
+ cute::array_aligned<Element, Use_smem ? cute::cosize_v<SmemLayoutO> : 0> smem_o;
+ };
+
+ using TMA_O = std::conditional_t<
+ Use_TMA_O,
+ decltype(make_tma_copy(
+ GmemTiledCopyOTMA{},
+ make_tensor(make_gmem_ptr(static_cast<Element*>(nullptr)), ShapeO{}, StrideO{}),
+ SmemLayoutOTMA{},
+ select<0, 2>(TileShape_MNK{}),
+ _1{})), // no mcast for O
+ std::nullptr_t
+ >;
+
+ // Host side kernel arguments
+ struct Arguments {
+ Element* ptr_O;
+ ShapeO const shape_O;
+ StrideO const stride_O;
+ float* ptr_LSE;
+ StrideLSE const stride_LSE;
+ int32_t const nheads_kv;
+ int const* cu_seqlens = nullptr;
+ int const* seqused = nullptr;
+ };
+
+ // Device side kernel params
+ struct Params {
+ Element* ptr_O;
+ ShapeO const shape_O;
+ StrideO const stride_O;
+ ShapeOPacked const shape_O_packed;
+ StrideOPacked const stride_O_packed;
+ float* ptr_LSE;
+ StrideLSE const stride_LSE;
+ ShapeLSEPacked const shape_LSE_packed;
+ StrideLSEPacked const stride_LSE_packed;
+ cutlass::FastDivmod qhead_per_khead_divmod;
+ TMA_O tma_store_O;
+ int const* cu_seqlens = nullptr;
+ int const* seqused = nullptr;
+ };
+
+ static Params
+ to_underlying_arguments(Arguments const& args) {
+ Tensor mO = make_tensor(make_gmem_ptr(args.ptr_O), args.shape_O, args.stride_O);
+ TMA_O tma_store_O = [&]{
+ if constexpr (Use_TMA_O) {
+ return make_tma_copy(GmemTiledCopyOTMA{}, mO, SmemLayoutO{}, select<0, 2>(TileShape_MNK{}), _1{}); // no mcast
+ } else {
+ return nullptr;
+ }
+ }();
+ // If PackGQA, reshape O to be ((qhead_per_khead, seqlen_q), head_size, nhead_k, batch_size, num_splits)
+ int const qhead_per_khead = !PackGQA ? 1 : cute::ceil_div(get<2>(args.shape_O), args.nheads_kv);
+ auto const shape_O_packed = cute::conditional_return<!PackGQA>(
+ args.shape_O,
+ make_shape(make_shape(qhead_per_khead, get<0>(args.shape_O)), get<1>(args.shape_O), args.nheads_kv, get<3>(args.shape_O), get<4>(args.shape_O))
+ );
+ auto const stride_O_packed = cute::conditional_return<!PackGQA>(
+ args.stride_O,
+ make_stride(make_stride(get<2>(args.stride_O), get<0>(args.stride_O)), get<1>(args.stride_O), get<2>(args.stride_O) * qhead_per_khead, get<3>(args.stride_O), get<4>(args.stride_O))
+ );
+ // If PackGQA, Reshape LSE to be ((qhead_per_khead, seqlen_q), nhead_k, batch_size, num_splits)
+ auto const shape_LSE_packed = cute::conditional_return<!PackGQA>(
+ select<0, 2, 3, 4>(args.shape_O),
+ make_shape(make_shape(qhead_per_khead, get<0>(args.shape_O)), args.nheads_kv, get<3>(args.shape_O), get<4>(args.shape_O))
+ );
+ auto const stride_LSE_packed = cute::conditional_return<!PackGQA>(
+ args.stride_LSE,
+ make_stride(make_stride(get<1>(args.stride_LSE), get<0>(args.stride_LSE)), get<1>(args.stride_LSE) * qhead_per_khead, get<2>(args.stride_LSE), get<3>(args.stride_LSE))
+ );
+ return {args.ptr_O, args.shape_O, args.stride_O, shape_O_packed, stride_O_packed,
+ args.ptr_LSE, args.stride_LSE, shape_LSE_packed, stride_LSE_packed,
+ cutlass::FastDivmod(qhead_per_khead),
+ tma_store_O, args.cu_seqlens, args.seqused};
+ }
+
+ /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best performance
+ CUTLASS_DEVICE
+ static void prefetch_tma_descriptors(Params const& params) {
+ if constexpr (Use_TMA_O) {
+ cute::prefetch_tma_descriptor(params.tma_store_O.get_tma_descriptor());
+ }
+ }
+
+ template <typename SharedStorage, typename FrgTensorO, typename FrgTensorLSE, typename TiledMma>
+ CUTLASS_DEVICE void
+ store(Params const& params,
+ FrgTensorO const& tOrO,
+ FrgTensorLSE const& lse,
+ SharedStorage& shared_storage,
+ TiledMma tiled_mma,
+ int thread_idx,
+ cute::tuple<int32_t, int32_t, int32_t, int32_t> const& block_coord
+ ) {
+
+ auto [m_block, bidh, bidb, split_idx] = block_coord;
+ Tensor sO = make_tensor(make_smem_ptr(shared_storage.tensors.epilogue.smem_o.data()), SmemLayoutO{});
+ // Tensor sO_pi = cute::as_position_independent_swizzle_tensor(sO);
+
+ Tensor tOrO_out = make_tensor_like<Element>(tOrO);
+ flash::convert_type_out(tOrO, tOrO_out);
+ if constexpr (FP8PermuteCol && (sizeof(Element) == 2 || sizeof(Element) == 4)) { flash::permute_output_fp8_Vcolmajor(tOrO_out); }
+
+ // Make sure all WGs have finished reading V
+ // Technically we don't need this if we're not using smem, but the mainloop makes the assumption that
+ // all epilogue threads sync at least once during the epilogue (so that we can start loading Q with
+ // cp.async if we need).
+ flash::named_barrier_sync(NumEpilogueThreads, cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+
+ // Step 1: Write O from rmem -> smem
+ if constexpr (Use_smem) {
+ auto smem_tiled_copy_O = make_tiled_copy_C(SmemCopyAtomO{}, tiled_mma);
+ auto smem_thr_copy_O = smem_tiled_copy_O.get_thread_slice(thread_idx);
+ Tensor taccOrO = smem_thr_copy_O.retile_S(tOrO_out); // ((Atom,AtomNum), MMA_M, MMA_N)
+ Tensor taccOsO = smem_thr_copy_O.partition_D(sO); // ((Atom,AtomNum),PIPE_M,PIPE_N)
+ // Tensor taccOsO = smem_thr_copy_O.partition_D(sO_pi); // ((Atom,AtomNum),PIPE_M,PIPE_N)
+ cute::copy(smem_tiled_copy_O, taccOrO, taccOsO);
+ if constexpr (Use_TMA_O) {
+ cutlass::arch::fence_view_async_shared(); // ensure smem writes are visible to TMA
+ cutlass::arch::NamedBarrier::arrive(NumEpilogueThreads + cutlass::NumThreadsPerWarp,
+ cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+ } else {
+ flash::named_barrier_sync(NumEpilogueThreads, cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+ }
+ } else {
+ if constexpr (ArchTag::kMinComputeCapability >= 90) {
+ #pragma unroll
+ for (uint32_t cta_id = 0; cta_id < size(ClusterShape{}); ++cta_id) {
+ shared_storage.pipelines.barrier_O.arrive(cta_id);
+ }
+ }
+ }
+
+ flash::SeqlenInfo<Varlen, kBlockM> seqlen_info{bidb, size<0>(params.shape_O), params.cu_seqlens, params.seqused};
+ bool is_varlen = Varlen && params.cu_seqlens;
+ int offset_o = seqlen_info.offset;
+ int seqlen_o = seqlen_info.seqlen;
+
+ // Step 2: Write LSE from rmem -> gmem
+ auto thread_mma = tiled_mma.get_thread_slice(thread_idx);
+ // (MMA,MMA_M,MMA_K)
+ Tensor taccOcO = thread_mma.partition_C(cute::make_identity_tensor(select<0, 2>(TileShape_MNK{})));
+ static_assert(decltype(size<0, 0>(taccOcO))::value == 2);
+ static_assert(decltype(size<0, 1>(taccOcO))::value == 2);
+ Tensor taccOcO_rowcol = make_tensor(taccOcO.data(), flash::convert_layout_acc_rowcol(taccOcO.layout()));
+ Tensor taccOcO_row = taccOcO_rowcol(_, _0{});
+ CUTE_STATIC_ASSERT_V(size(lse) == size(taccOcO_row)); // MMA_M
+
+ using PackGQAt = flash::PackGQAManager<get<0>(TileShape_MNK{}), get<2>(TileShape_MNK{}), NumEpilogueThreads, Element>;
+
+ Tensor mLSE = make_tensor(make_gmem_ptr(params.ptr_LSE + offset_o * get<0>(params.stride_LSE)), params.shape_LSE_packed, params.stride_LSE_packed)(_, bidh, !is_varlen ? bidb : 0, split_idx);
+ // if (thread_idx == 0) { printf("Before LSE write, m_block: %d, bidh: %d, bidb: %d, split_idx: %d, offset_o: %d, seqlen_o: %d\n", m_block, bidh, bidb, split_idx, offset_o, seqlen_o); print(mLSE); printf("\n"); }
+ if constexpr (!PackGQA) {
+ #pragma unroll
+ for (int mi = 0; mi < size(lse); ++mi) {
+ int const row = m_block * kBlockM + get<0>(taccOcO_row(mi));
+ if (get<1>(taccOcO_row(_0{})) == 0 && row < seqlen_o) { mLSE(row) = lse(mi); }
+ }
+ } else {
+ PackGQAt::store_LSE(mLSE, lse, tiled_mma, params.qhead_per_khead_divmod, thread_idx, seqlen_o, m_block);
+ }
+
+ // Step 3: Write O from smem -> gmem
+ if constexpr (Use_TMA_O) {
+ Tensor mO = params.tma_store_O.get_tma_tensor(params.shape_O)(_, _, bidh, bidb, split_idx);
+ Tensor gO = local_tile(mO, select<0, 2>(TileShape_MNK{}), make_coord(m_block, _0{})); // (M, K)
+ auto block_tma_O = params.tma_store_O.get_slice(_0{});
+ Tensor tOgO = block_tma_O.partition_D(gO); // (TMA, TMA_M, TMA_K)
+ Tensor tOsO = block_tma_O.partition_S(sO); // (TMA, TMA_M, TMA_K)
+ int warp_idx_sync = __shfl_sync(0xffffffff, thread_idx / cutlass::NumThreadsPerWarp, 0);
+ if (warp_idx_sync == NumEpilogueThreads / cutlass::NumThreadsPerWarp - 1) {
+ cutlass::arch::NamedBarrier::sync(NumEpilogueThreads + cutlass::NumThreadsPerWarp,
+ cutlass::arch::ReservedNamedBarriers::EpilogueBarrier);
+ if (cute::elect_one_sync()) {
+ cute::copy(params.tma_store_O, tOsO, tOgO);
+ tma_store_arrive();
+ tma_store_wait<0>();
+ #pragma unroll
+ for (uint32_t cta_id = 0; cta_id < size(ClusterShape{}); ++cta_id) {
+ shared_storage.pipelines.barrier_O.arrive(cta_id);
+ }
+ }
+ }
+ } else { // Don't use TMA in Varlen case since we don't want to overwrite the output of another sequence
+ Tensor mO = make_tensor(make_gmem_ptr(params.ptr_O + offset_o * get<0>(params.stride_O)), params.shape_O_packed, params.stride_O_packed)(_, _, bidh, !is_varlen ? bidb : 0, split_idx);
+ Tensor gO = local_tile(mO, select<0, 2>(TileShape_MNK{}), make_coord(m_block, _0{})); // (M, K)
+ // if (thread_idx == 0) { printf("Before O write, m_block: %d, bidh: %d, bidb: %d, split_idx: %d, offset_o: %d, seqlen_o: %d, mO_addr = %p, addr diff = %d\n", m_block, bidh, bidb, split_idx, offset_o, seqlen_o, mO.data(), reinterpret_cast<int>(&mO(0)) - reinterpret_cast<int>(params.ptr_O)); }
+ if constexpr (Use_smem) {
+ GmemTiledCopyO gmem_tiled_copy_O;
+ auto gmem_thr_copy_O = gmem_tiled_copy_O.get_thread_slice(thread_idx);
+ Tensor tOsO = gmem_thr_copy_O.partition_S(sO); // ((Atom,AtomNum),ATOM_M,ATOM_N)
+ // Tensor tOsO = gmem_thr_copy_O.partition_S(sO_pi); // ((Atom,AtomNum),ATOM_M,ATOM_N)
+ Tensor tOrO = make_fragment_like(tOsO);
+ cute::copy(gmem_tiled_copy_O, tOsO, tOrO);
+ if constexpr (ArchTag::kMinComputeCapability >= 90) {
+ cutlass::arch::fence_view_async_shared(); // ensure smem reads are done before next TMA to smem_v
+ #pragma unroll
+ for (uint32_t cta_id = 0; cta_id < size(ClusterShape{}); ++cta_id) {
+ shared_storage.pipelines.barrier_O.arrive(cta_id);
+ }
+ }
+ if constexpr (!PackGQA) {
+ // (BLK_M,BLK_K) -> (blk_m,blk_k)
+ Tensor tOcO = gmem_thr_copy_O.partition_D(cute::make_identity_tensor(select<0, 2>(TileShape_MNK{})));
+ Tensor tOpO = make_tensor<bool>(make_shape(size<2>(tOsO)));
+ #pragma unroll
+ for (int k = 0; k < size(tOpO); ++k) { tOpO(k) = get<1>(tOcO(_0{}, _0{}, k)) < get<1>(params.shape_O); }
+ Tensor tOgO = gmem_thr_copy_O.partition_D(gO);
+ // Clear_OOB_K must be false since we don't want to write zeros to gmem
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/false>(
+ gmem_tiled_copy_O, tOrO, tOgO, tOcO, tOpO, seqlen_o - m_block * kBlockM
+ );
+ } else {
+ // If PackGQA, we split the work of compute O_ptr among threads in the same row
+ PackGQAt::store_O(mO, tOrO, params.qhead_per_khead_divmod, thread_idx, seqlen_o, m_block);
+ }
+ } else {
+ // We already arrived on barrier_O earlier
+ if constexpr (!PackGQA) {
+ static constexpr int kGmemElemsPerStoreDirect = 2;
+ cute::Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element> gmem_copy_direct;
+ // Reshape acc from ((2, 2, V), MMA_M, MMA_N) to (nrow=(2, MMA_M), ncol=(2, V, MMA_N))
+ Tensor tOrO_rowcol = make_tensor(tOrO_out.data(), flash::convert_layout_acc_rowcol(tOrO.layout()));
+ Tensor tOrO_copy = cute::tiled_divide(tOrO_rowcol, Shape<_1, Int<kGmemElemsPerStoreDirect>>{});
+ Tensor tOgO = thread_mma.partition_C(gO);
+ Tensor tOgO_rowcol = make_tensor(tOgO.data(), flash::convert_layout_acc_rowcol(tOgO.layout()));
+ Tensor tOgO_copy = cute::tiled_divide(tOgO_rowcol, Shape<_1, Int<kGmemElemsPerStoreDirect>>{});
+ Tensor taccOcO_col = taccOcO_rowcol(_0{}, _);
+ #pragma unroll
+ for (int m = 0; m < size(taccOcO_row); ++m) {
+ if (get<0>(taccOcO_row(m)) < seqlen_o - m_block * kBlockM) {
+ #pragma unroll
+ for (int k = 0; k < size(taccOcO_col) / kGmemElemsPerStoreDirect; ++k) {
+ if (get<1>(taccOcO_col(k * kGmemElemsPerStoreDirect)) < get<1>(params.shape_O)) {
+ cute::copy(gmem_copy_direct, tOrO_copy(_, m, k), tOgO_copy(_, m, k));
+ }
+ }
+ }
+ }
+ } else {
+ PackGQAt::store_O_direct(mO, tOrO_out, tiled_mma, params.qhead_per_khead_divmod, thread_idx, seqlen_o, m_block);
+ }
+ }
+ }
+ }
+
+ CUTLASS_DEVICE void
+ store_tail() {
+ // Don't need to do tma_store_wait<0>() here since we already did in @store
+ }
+
+ // Write 0 to output and -inf to LSE
+ template <bool Clear_O=true>
+ CUTLASS_DEVICE void
+ store_zero(
+ Params const& params,
+ int thread_idx,
+ cute::tuple<int32_t, int32_t, int32_t, int32_t> const& block_coord
+ ) {
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ auto [m_block, bidh, bidb, split_idx] = block_coord;
+ flash::SeqlenInfo<Varlen, kBlockM> seqlen_info{bidb, size<0>(params.shape_O), params.cu_seqlens, params.seqused};
+ bool const is_varlen = Varlen && params.cu_seqlens;
+ int offset_o = seqlen_info.offset;
+ int seqlen_o = seqlen_info.seqlen;
+ int qhead_per_khead = !PackGQA ? 1 : params.qhead_per_khead_divmod.divisor;
+ Tensor mO = make_tensor(make_gmem_ptr(params.ptr_O + offset_o * get<0>(params.stride_O)), params.shape_O_packed, params.stride_O_packed)(_, _, bidh, !is_varlen ? bidb : 0, split_idx);
+ Tensor mLSE = make_tensor(make_gmem_ptr(params.ptr_LSE + offset_o * get<0>(params.stride_LSE)), params.shape_LSE_packed, params.stride_LSE_packed)(_, bidh, !is_varlen ? bidb : 0, split_idx);
+ Tensor gLSE = local_tile(mLSE, Shape<Int<kBlockM>>{}, make_coord(m_block));
+
+ static_assert(kBlockM <= NumEpilogueThreads);
+ if (thread_idx < kBlockM) {
+ const int row = m_block * kBlockM + thread_idx;
+ if constexpr (!PackGQA) {
+ if (row < seqlen_o) { mLSE(row) = -INFINITY; }
+ } else {
+ if (row < seqlen_o * qhead_per_khead) {
+ int m_idx, h_idx;
+ m_idx = params.qhead_per_khead_divmod.divmod(h_idx, row);
+ // mLSE shape shape ((qhead_per_khead, seqlen_q)) and it's unhappy with just 1 "make_coord"
+ mLSE(make_coord(make_coord(h_idx, m_idx))) = -INFINITY;
+ }
+ }
+ }
+
+ if constexpr (!Clear_O) { return; }
+
+ GmemTiledCopyO gmem_tiled_copy_O;
+ auto gmem_thr_copy_O = gmem_tiled_copy_O.get_thread_slice(thread_idx);
+ Tensor tOcO = gmem_thr_copy_O.partition_D(cute::make_identity_tensor(select<0, 2>(TileShape_MNK{})));
+ if constexpr (!PackGQA) {
+ Tensor tOpO = make_tensor<bool>(make_shape(size<2>(tOcO)));
+ #pragma unroll
+ for (int k = 0; k < size(tOpO); ++k) { tOpO(k) = get<1>(tOcO(_0{}, _0{}, k)) < get<1>(params.shape_O); }
+ Tensor gO = local_tile(mO, select<0, 2>(TileShape_MNK{}), make_coord(m_block, _0{})); // (M, K)
+ Tensor tOgO = gmem_thr_copy_O.partition_D(gO);
+ Tensor tOrO = make_fragment_like(tOgO);
+ cute::clear(tOrO);
+ // Clear_OOB_K must be false since we don't want to write zeros to gmem
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/false>(
+ gmem_tiled_copy_O, tOrO, tOgO, tOcO, tOpO, seqlen_o - m_block * kBlockM
+ );
+ } else {
+ // If PackGQA, we split the work of compute O_ptr among threads in the same row
+ using PackGQAt = flash::PackGQAManager<get<0>(TileShape_MNK{}), get<2>(TileShape_MNK{}), NumEpilogueThreads, Element>;
+ Tensor tOrO = make_tensor<Element>(make_shape(Shape<_1, Int<kGmemElemsPerStore>>{}, size<1>(tOcO), size<2>(tOcO)));
+ cute::clear(tOrO);
+ PackGQAt::store_O(mO, tOrO, params.qhead_per_khead_divmod, thread_idx, seqlen_o, m_block);
+ }
+
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash.h b/src/solarwm/backends/minimax_h3/hopper_src/flash.h
new file mode 100644
index 0000000..4559a13
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash.h
@@ -0,0 +1,205 @@
+/******************************************************************************
+ * Copyright (c) 2023, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cuda.h>
+#include <vector>
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+struct Qkv_params {
+ using index_t = int64_t;
+ // The QKV matrices.
+ void *__restrict__ q_ptr;
+ void *__restrict__ k_ptr;
+ void *__restrict__ v_ptr;
+
+ // The stride between rows of the Q, K and V matrices.
+ index_t q_batch_stride;
+ index_t k_batch_stride;
+ index_t v_batch_stride;
+ index_t q_row_stride;
+ index_t k_row_stride;
+ index_t v_row_stride;
+ index_t q_head_stride;
+ index_t k_head_stride;
+ index_t v_head_stride;
+ index_t v_dim_stride;
+
+ // The number of heads.
+ int h, h_k;
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+struct Flash_fwd_params : public Qkv_params {
+ using index_t = int64_t;
+
+ // The O matrix (output).
+ void * __restrict__ o_ptr;
+ void * __restrict__ oaccum_ptr;
+
+ // The stride between rows of O.
+ index_t o_batch_stride;
+ index_t o_row_stride;
+ index_t o_head_stride;
+
+ // The pointer to the softmax sum.
+ void * __restrict__ softmax_lse_ptr;
+ void * __restrict__ softmax_lseaccum_ptr;
+
+ // For FP8 scaling
+ float * __restrict__ q_descale_ptr;
+ float * __restrict__ k_descale_ptr;
+ float * __restrict__ v_descale_ptr;
+ index_t q_descale_batch_stride;
+ index_t q_descale_head_stride;
+ index_t k_descale_batch_stride;
+ index_t k_descale_head_stride;
+ index_t v_descale_batch_stride;
+ index_t v_descale_head_stride;
+
+ // The dimensions.
+ int b, seqlen_q, seqlen_k, seqlen_knew, d, seqlen_q_rounded, seqlen_k_rounded, d_rounded, rotary_dim;
+ int total_q, total_k, total_knew;
+ int b_k; // When having KV cache and with cache_batch_idx, K & V might have larger batch size than Q
+
+ // The scaling factors for the kernel.
+ float scale_softmax;
+ float softcap;
+
+ // array of length b+1 holding starting offset of each sequence.
+ int * __restrict__ cu_seqlens_q;
+ int * __restrict__ cu_seqlens_k;
+ int * __restrict__ cu_seqlens_knew;
+ int * __restrict__ leftpad_k;
+
+ // If provided, the actual length of each q/k sequence.
+ int *__restrict__ seqused_q;
+ int *__restrict__ seqused_k;
+
+ // The stride between rows of Oaccum.
+ index_t oaccum_split_stride;
+ index_t oaccum_batch_stride;
+ index_t oaccum_row_stride;
+ index_t oaccum_head_stride;
+
+ // The stride between rows of LSEaccum.
+ index_t lseaccum_split_stride;
+ index_t lseaccum_batch_stride;
+ index_t lseaccum_head_stride;
+
+ // The K_new and V_new matrices.
+ void * __restrict__ knew_ptr;
+ void * __restrict__ vnew_ptr;
+
+ // The stride between rows of the Q, K and V matrices.
+ index_t knew_batch_stride;
+ index_t vnew_batch_stride;
+ index_t knew_row_stride;
+ index_t vnew_row_stride;
+ index_t knew_head_stride;
+ index_t vnew_head_stride;
+
+ // The cos and sin matrices for rotary embedding.
+ void * __restrict__ rotary_cos_ptr;
+ void * __restrict__ rotary_sin_ptr;
+
+ // The indices to index into the KV cache.
+ int * __restrict__ kv_batch_idx;
+
+ // Paged KV cache
+ int * __restrict__ page_table;
+ index_t page_table_batch_stride;
+ int page_size;
+ int num_pages;
+
+ // The dropout probability (probability of keeping an activation).
+ float p_dropout;
+ // uint32_t p_dropout_in_uint;
+ // uint16_t p_dropout_in_uint16_t;
+ uint8_t p_dropout_in_uint8_t;
+
+ // Scale factor of 1 / (1 - p_dropout).
+ float rp_dropout;
+
+ // Local window size
+ int window_size_left, window_size_right;
+ int sink_token_length;
+
+ // Pointer to the RNG seed (idx 0) and offset (idx 1).
+ uint64_t * rng_state;
+
+ bool is_bf16;
+ bool is_fp32;
+ bool is_e4m3;
+ bool is_causal;
+ bool is_local;
+
+ bool is_rotary_interleaved;
+
+ int num_splits; // For split-KV version
+ bool pack_gqa;
+
+ int * __restrict__ tile_count_semaphore;
+
+ int arch;
+ int num_sm;
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+struct Flash_bwd_params : public Flash_fwd_params {
+ using index_t = int64_t;
+
+ // The dO and dQKV matrices.
+ void *__restrict__ do_ptr;
+ void *__restrict__ dq_ptr;
+ void *__restrict__ dk_ptr;
+ void *__restrict__ dv_ptr;
+
+ // To accumulate dQ
+ void *__restrict__ dq_accum_ptr;
+ void *__restrict__ dk_accum_ptr;
+ void *__restrict__ dv_accum_ptr;
+
+ // // To accumulate dK and dV in case we're splitting the bwd along seqlen_q
+ // dimension void *__restrict__ dk_accum_ptr; void *__restrict__
+ // dv_accum_ptr;
+
+ // The stride between rows of the dO, dQ, dK and dV matrices.
+ index_t do_batch_stride;
+ index_t do_row_stride;
+ index_t do_head_stride;
+ index_t dq_batch_stride;
+ index_t dk_batch_stride;
+ index_t dv_batch_stride;
+ index_t dq_row_stride;
+ index_t dk_row_stride;
+ index_t dv_row_stride;
+ index_t dq_head_stride;
+ index_t dk_head_stride;
+ index_t dv_head_stride;
+
+ // The pointer to the softmax d sum.
+ void *__restrict__ dsoftmax_sum;
+ void *__restrict__ softmax_lse_log2_ptr;
+
+ int *__restrict__ dq_semaphore;
+ int *__restrict__ dk_semaphore;
+ int *__restrict__ dv_semaphore;
+
+ bool deterministic;
+ index_t dq_accum_split_stride;
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <int Arch, typename T, int Headdim, bool Split, bool PagedKV, bool Has_softcap, bool PackGQA>
+void run_mha_fwd_(Flash_fwd_params &params, cudaStream_t stream);
+template <int Arch, typename T, int Headdim, bool Has_softcap>
+void run_mha_bwd_(Flash_bwd_params &params, cudaStream_t stream);
+template <typename T, typename Tpartial, int Headdim>
+void run_mha_fwd_combine_(Flash_fwd_params &params, cudaStream_t stream);
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_api.cpp b/src/solarwm/backends/minimax_h3/hopper_src/flash_api.cpp
new file mode 100644
index 0000000..82643d9
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_api.cpp
@@ -0,0 +1,1342 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+// Include these 2 headers instead of torch/extension.h since we don't need all of the torch headers.
+#include <torch/python.h>
+#include <torch/nn/functional.h>
+#include <torch/version.h> // For TORCH_VERSION* macros
+#include <ATen/cuda/CUDAContext.h>
+#include <c10/cuda/CUDAGuard.h>
+
+#include <cutlass/numeric_types.h>
+
+#include "flash.h"
+#include "static_switch.h"
+#include "tile_size.h"
+#include "heuristics.h"
+
+// Copied from https://github.com/pytorch/pytorch/commit/7931eee5c5ebcdf468bff4d308510b03355cd909
+// This is so that we can pass in torch.dtype as a parameter to the function.
+#if TORCH_VERSION_MAJOR < 2 || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR < 4)
+
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+namespace pybind11::detail {
+
+ template <>
+ struct type_caster<at::ScalarType> {
+ public:
+ // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes)
+ PYBIND11_TYPE_CASTER(at::ScalarType, _("torch.dtype"));
+ // PYBIND11_TYPE_CASTER defines a member field called value. at::ScalarType
+ // cannot be default-initialized, we provide this constructor to explicitly
+ // initialize that field. The value doesn't matter as it will be overwritten
+ // after a successful call to load.
+ type_caster() : value(at::kFloat) {}
+ bool load(handle src, bool) {
+ PyObject* obj = src.ptr();
+ if (THPDtype_Check(obj)) {
+ value = reinterpret_cast<THPDtype*>(obj)->scalar_type;
+ return true;
+ }
+ return false;
+ }
+ static handle cast(
+ const at::ScalarType& src,
+ return_value_policy /* policy */,
+ handle /* parent */) {
+ return Py_NewRef(torch::getTHPDtype(src));
+ }
+ };
+
+} // namespace pybind11::detail
+
+#endif
+
+#define CHECK_DEVICE(x) TORCH_CHECK(x.is_cuda(), #x " must be on CUDA")
+#define CHECK_SHAPE(x, ...) TORCH_CHECK(x.sizes() == torch::IntArrayRef({__VA_ARGS__}), #x " must have shape (" #__VA_ARGS__ ")")
+#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
+
+void set_params_fprop(Flash_fwd_params &params,
+ // sizes
+ const size_t b,
+ const size_t seqlen_q,
+ const size_t seqlen_k,
+ const size_t seqlen_q_rounded,
+ const size_t seqlen_k_rounded,
+ const size_t h,
+ const size_t h_k,
+ const size_t d,
+ const size_t d_rounded,
+ // device pointers
+ const at::Tensor q,
+ const at::Tensor k,
+ const at::Tensor v,
+ at::Tensor out,
+ void *cu_seqlens_q_d,
+ void *cu_seqlens_k_d,
+ void *seqused_q,
+ void *seqused_k,
+ void *softmax_lse_d,
+ float p_dropout,
+ float softmax_scale,
+ int window_size_left,
+ int window_size_right,
+ const float softcap=0.f,
+ const int sm_margin=0) {
+
+ // Reset the parameters
+ params = {};
+
+ params.is_bf16 = q.dtype() == torch::kBFloat16;
+ params.is_e4m3 = q.dtype() == torch::kFloat8_e4m3fn;
+
+ // Set the pointers and strides.
+ params.q_ptr = q.data_ptr();
+ params.k_ptr = k.data_ptr();
+ params.v_ptr = v.data_ptr();
+ // All stride are in elements, not bytes.
+ params.q_row_stride = q.stride(-3);
+ params.k_row_stride = k.stride(-3);
+ params.v_row_stride = v.stride(-3);
+ params.q_head_stride = q.stride(-2);
+ params.k_head_stride = k.stride(-2);
+ params.v_head_stride = v.stride(-2);
+ params.v_dim_stride = v.stride(-1);
+ params.o_ptr = out.data_ptr();
+ params.o_row_stride = out.stride(-3);
+ params.o_head_stride = out.stride(-2);
+
+ if (cu_seqlens_q_d == nullptr) {
+ params.q_batch_stride = q.stride(0);
+ params.o_batch_stride = out.stride(0);
+ }
+ if (cu_seqlens_k_d == nullptr) {
+ params.k_batch_stride = k.stride(0);
+ params.v_batch_stride = v.stride(0);
+ }
+
+ params.cu_seqlens_q = static_cast<int *>(cu_seqlens_q_d);
+ params.cu_seqlens_k = static_cast<int *>(cu_seqlens_k_d);
+ params.seqused_q = static_cast<int *>(seqused_q);
+ params.seqused_k = static_cast<int *>(seqused_k);
+
+ // Softmax sum
+ params.softmax_lse_ptr = softmax_lse_d;
+
+ // Set the dimensions.
+ params.b = b;
+ params.h = h;
+ params.h_k = h_k;
+ params.seqlen_q = seqlen_q;
+ params.seqlen_k = seqlen_k;
+ params.seqlen_q_rounded = seqlen_q_rounded;
+ params.seqlen_k_rounded = seqlen_k_rounded;
+ params.d = d;
+ params.d_rounded = d_rounded;
+
+ // Set the different scale values.
+ params.scale_softmax = softmax_scale;
+ params.softcap = softcap;
+
+ // Set this to probability of keeping an element to simplify things.
+ params.p_dropout = 1.f - p_dropout;
+ // Convert p from float to int so we don't have to convert the random uint to float to compare.
+ // [Minor] We want to round down since when we do the comparison we use <= instead of <
+ // params.p_dropout_in_uint = uint32_t(std::floor(params.p_dropout * 4294967295.0));
+ // params.p_dropout_in_uint16_t = uint16_t(std::floor(params.p_dropout * 65535.0));
+ params.p_dropout_in_uint8_t = uint8_t(std::floor(params.p_dropout * 255.0));
+ params.rp_dropout = 1.f / params.p_dropout;
+ TORCH_CHECK(p_dropout < 1.f);
+ #ifdef FLASHATTENTION_DISABLE_DROPOUT
+ TORCH_CHECK(p_dropout == 0.0f, "This flash attention build does not support dropout.");
+ #endif
+
+ // Causal is the special case where window_size_right == 0 and window_size_left < 0.
+ // Local is the more general case where window_size_right >= 0 or window_size_left >= 0.
+ params.is_causal = window_size_left < 0 && window_size_right == 0;
+ params.is_local = (window_size_left >= 0 || window_size_right >= 0) && !params.is_causal;
+
+ // TODO: check this
+ if (window_size_left < 0 && window_size_right >= 0) { window_size_left = seqlen_k - 1; }
+ if (window_size_left >= 0 && window_size_right < 0) { window_size_right = seqlen_q - 1; }
+ params.window_size_left = window_size_left;
+ params.window_size_right = window_size_right;
+
+ params.arch = at::cuda::getCurrentDeviceProperties()->major * 10 + at::cuda::getCurrentDeviceProperties()->minor;
+ params.num_sm = at::cuda::getCurrentDeviceProperties()->multiProcessorCount - sm_margin;
+
+ #ifdef FLASHATTENTION_DISABLE_LOCAL
+ TORCH_CHECK(!params.is_local, "This flash attention build does not support local attention.");
+ #endif
+}
+
+void set_params_dgrad(Flash_bwd_params &params,
+ // sizes
+ const size_t b,
+ const size_t seqlen_q,
+ const size_t seqlen_k,
+ const size_t seqlen_q_rounded,
+ const size_t seqlen_k_rounded,
+ const size_t h,
+ const size_t h_k,
+ const size_t d,
+ const size_t d_rounded,
+ // device pointers
+ const at::Tensor q,
+ const at::Tensor k,
+ const at::Tensor v,
+ const at::Tensor out,
+ const at::Tensor dout,
+ at::Tensor dq,
+ at::Tensor dk,
+ at::Tensor dv,
+ void *cu_seqlens_q_d,
+ void *cu_seqlens_k_d,
+ void *seqused_q,
+ void *seqused_k,
+ void *dq_accum_d,
+ void *dk_accum_d,
+ void *dv_accum_d,
+ void *softmax_lse_d,
+ void *dsoftmax_sum_d,
+ float p_dropout,
+ float softmax_scale,
+ int window_size_left,
+ int window_size_right,
+ const float softcap=0.f,
+ bool deterministic=false,
+ int const sm_margin=0) {
+
+ set_params_fprop(params,
+ b, seqlen_q, seqlen_k, seqlen_q_rounded, seqlen_k_rounded, h, h_k, d, d_rounded,
+ q, k, v, out,
+ cu_seqlens_q_d,
+ cu_seqlens_k_d,
+ seqused_q,
+ seqused_k,
+ softmax_lse_d,
+ p_dropout,
+ softmax_scale,
+ window_size_left,
+ window_size_right,
+ softcap,
+ sm_margin);
+
+ // Set the pointers and strides.
+ params.do_ptr = dout.data_ptr();
+ params.do_row_stride = dout.stride(-3);
+ params.do_head_stride = dout.stride(-2);
+ params.dq_ptr = dq.data_ptr();
+ params.dk_ptr = dk.data_ptr();
+ params.dv_ptr = dv.data_ptr();
+ params.dq_row_stride = dq.stride(-3);
+ params.dk_row_stride = dk.stride(-3);
+ params.dv_row_stride = dv.stride(-3);
+ params.dq_head_stride = dq.stride(-2);
+ params.dk_head_stride = dk.stride(-2);
+ params.dv_head_stride = dv.stride(-2);
+
+ if (cu_seqlens_q_d == nullptr) {
+ params.do_batch_stride = dout.stride(0);
+ params.dq_batch_stride = dq.stride(0);
+ params.dk_batch_stride = dk.stride(0);
+ params.dv_batch_stride = dv.stride(0);
+ }
+
+ params.dq_accum_ptr = dq_accum_d;
+ params.dk_accum_ptr = dk_accum_d;
+ params.dv_accum_ptr = dv_accum_d;
+
+ // Softmax sum
+ params.dsoftmax_sum = dsoftmax_sum_d;
+
+ params.deterministic = deterministic;
+}
+
+void run_mha_fwd(Flash_fwd_params &params, cudaStream_t stream) {
+ // HEADDIM_SWITCH(params.d, [&] {
+ // run_mha_fwd_<cutlass::half_t, kHeadSize>(params, stream);
+ // });
+ TORCH_CHECK(params.num_splits >= 1);
+ ARCH_SWITCH(params.arch, Arch, [&] {
+ SPLIT_SWITCH(params.num_splits > 1, Split, [&] {
+ PAGEDKV_SWITCH(params.page_table, PagedKV, [&] {
+ PACKGQA_SWITCH(params.pack_gqa, PackGQA_, [&] {
+ // Always enable PackGQA for Sm8x or PagedKV or Split to reduce compilation
+ static constexpr bool PackGQA = PackGQA_ || Arch < 90 || PagedKV || Split;
+ SOFTCAP_SWITCH(params.softcap > 0.0, Has_softcap, [&] {
+ if (!params.is_e4m3) {
+ if (params.is_bf16) {
+ #ifndef FLASHATTENTION_DISABLE_HDIM64
+ if (params.d <= 64) { return run_mha_fwd_<Arch, cutlass::bfloat16_t, 64, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM96
+ if (params.d <= 96) { return run_mha_fwd_<Arch, cutlass::bfloat16_t, 96, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM128
+ if (params.d <= 128) { return run_mha_fwd_<Arch, cutlass::bfloat16_t, 128, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM192
+ if (params.d <= 192) { return run_mha_fwd_<Arch, cutlass::bfloat16_t, 192, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM256
+ if (params.d <= 256) { return run_mha_fwd_<Arch, cutlass::bfloat16_t, 256, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ } else {
+ #ifndef FLASHATTENTION_DISABLE_FP16
+ #ifndef FLASHATTENTION_DISABLE_HDIM64
+ if (params.d <= 64) { return run_mha_fwd_<Arch, cutlass::half_t, 64, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM96
+ if (params.d <= 96) { return run_mha_fwd_<Arch, cutlass::half_t, 96, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM128
+ if (params.d <= 128) { return run_mha_fwd_<Arch, cutlass::half_t, 128, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM192
+ if (params.d <= 192) { return run_mha_fwd_<Arch, cutlass::half_t, 192, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM256
+ if (params.d <= 256) { return run_mha_fwd_<Arch, cutlass::half_t, 256, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #else
+ TORCH_CHECK(false, "This flash attention build does not support FP16.");
+ #endif
+ }
+ } else {
+ #ifndef FLASHATTENTION_DISABLE_FP8
+ #ifndef FLASHATTENTION_DISABLE_HDIM64
+ if (params.d <= 64) { return run_mha_fwd_<90, cutlass::float_e4m3_t, 64, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM96
+ if (params.d <= 96) { return run_mha_fwd_<90, cutlass::float_e4m3_t, 96, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM128
+ if (params.d <= 128) { return run_mha_fwd_<90, cutlass::float_e4m3_t, 128, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM192
+ if (params.d <= 192) { return run_mha_fwd_<90, cutlass::float_e4m3_t, 192, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM256
+ if (params.d <= 256) { return run_mha_fwd_<90, cutlass::float_e4m3_t, 256, Split, PagedKV, Has_softcap, PackGQA>(params, stream); }
+ #endif
+ #else
+ TORCH_CHECK(false, "This flash attention build does not support FP8.");
+ #endif
+ }
+ });
+ });
+ });
+ });
+ });
+}
+
+void run_mha_fwd_combine(Flash_fwd_params &params, cudaStream_t stream) {
+ #ifndef FLASHATTENTION_DISABLE_SPLIT
+ // If hdim is 96 or 192, it's faster to round them to 128 or 256 respectively
+ // so that kBlockM is smaller and we have more parallelism.
+ if (params.is_fp32) {
+ if (params.d <= 64) {
+ run_mha_fwd_combine_<float, float, 64>(params, stream);
+ } else if (params.d <= 128) {
+ run_mha_fwd_combine_<float, float, 128>(params, stream);
+ } else {
+ run_mha_fwd_combine_<float, float, 256>(params, stream);
+ }
+ } else if (params.is_bf16) {
+ if (params.d <= 64) {
+ run_mha_fwd_combine_<cutlass::bfloat16_t, float, 64>(params, stream);
+ } else if (params.d <= 128) {
+ run_mha_fwd_combine_<cutlass::bfloat16_t, float, 128>(params, stream);
+ } else {
+ run_mha_fwd_combine_<cutlass::bfloat16_t, float, 256>(params, stream);
+ }
+ } else {
+ if (params.d <= 64) {
+ run_mha_fwd_combine_<cutlass::half_t, float, 64>(params, stream);
+ } else if (params.d <= 128) {
+ run_mha_fwd_combine_<cutlass::half_t, float, 128>(params, stream);
+ } else {
+ run_mha_fwd_combine_<cutlass::half_t, float, 256>(params, stream);
+ }
+ }
+ #else
+ TORCH_CHECK(false, "This flash attention build does not support combine kernels.");
+ #endif
+}
+
+inline bool get_pack_gqa(Flash_fwd_params const& params) {
+ // Always enable PackGQA for Sm8x or PagedKV or Split to reduce compilation and binary size.
+ // Has little effect on speed.
+ if (params.arch < 90 || params.page_table || params.num_splits > 1) { return true; }
+ #ifdef FLASHATTENTION_DISABLE_PACKGQA
+ return false;
+ #else
+ // params.page_table must already be set
+ if (params.h == params.h_k) { return false; }
+ // This needs to match the kernel configs
+ auto kBlockMN_kernel_args_sm90 = tile_size_fwd_sm90(params.d_rounded, params.is_causal, params.is_local, params.is_e4m3 ? 1 : 2 /*element_size*/, false /*v_colmajor*/, params.page_table, params.softcap > 0.f);
+ int const kBlockM = std::get<0>(kBlockMN_kernel_args_sm90);
+ return should_pack_gqa(params.cu_seqlens_q || params.seqused_q, params.seqlen_q, params.h / params.h_k, kBlockM);
+ #endif
+}
+
+inline int get_num_splits(Flash_fwd_params const& params) {
+ #ifdef FLASHATTENTION_DISABLE_SPLIT
+ return 1;
+ #else
+ // Always enable PackGQA for Split
+ // params.page_table must already be set
+ // This needs to match the kernel configs
+ bool varlen = params.cu_seqlens_q || params.cu_seqlens_k || params.seqused_q || params.seqused_k || params.leftpad_k;
+ auto kBlockMN_kernel_args_sm90 = tile_size_fwd_sm90(params.d_rounded, params.is_causal, params.is_local, params.is_e4m3 ? 1 : 2 /*element_size*/, false /*v_colmajor*/, params.page_table, params.softcap > 0.f);
+ // Strictly speaking we need to pass in (varlen && params.num_splits > 1) but num_splits
+ // has not been set here. It's OK though because we might just underestimate kBlockN a bit
+ auto kBlockMN_kernel_args_sm8x = tile_size_fwd_sm8x(params.arch == 86 || params.arch == 89, params.d_rounded, params.is_causal, params.is_local, params.is_e4m3 ? 1 : 2 /*element_size*/, params.page_table, varlen, params.softcap > 0.f, params.knew_ptr);
+ int const kBlockM = params.arch >= 90 ? std::get<0>(kBlockMN_kernel_args_sm90) : std::get<0>(kBlockMN_kernel_args_sm8x);
+ int const kBlockN = params.arch >= 90 ? std::get<1>(kBlockMN_kernel_args_sm90) : std::get<1>(kBlockMN_kernel_args_sm8x);
+ int seqlen_q_packgqa = params.seqlen_q * (params.h / params.h_k);
+ // If is_local, we're not going to load all of seqlen_k
+ int const seqlen_k_loaded = !params.is_local
+ ? params.seqlen_k
+ : std::max(0, std::min(params.seqlen_k, params.window_size_right + params.window_size_left + 1 + kBlockM));
+ int const num_n_blocks = (seqlen_k_loaded + kBlockN - 1) / kBlockN;
+ int const num_m_blocks = (seqlen_q_packgqa + kBlockM - 1) / kBlockM;
+ return num_splits_heuristic(params.b * (!params.pack_gqa ? params.h : params.h_k) * num_m_blocks, params.num_sm, num_n_blocks, 128);
+ // return num_splits_heuristic(params.b * params.h_k * num_m_blocks, params.b * params.h_k,
+ // params.num_sm, num_n_blocks, 128, params.d_rounded);
+ #endif
+}
+
+inline int get_max_headdim() {
+ #ifndef FLASHATTENTION_DISABLE_HDIM256
+ return 256;
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM192
+ return 192;
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM128
+ return 128;
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM96
+ return 96;
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM64
+ return 64;
+ #endif
+ return 0;
+}
+
+inline int round_up_headdim(int head_size) {
+ #ifndef FLASHATTENTION_DISABLE_HDIM64
+ if (head_size <= 64) { return 64; }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM96
+ if (head_size <= 96) { return 96; }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM128
+ if (head_size <= 128) { return 128; }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM192
+ if (head_size <= 192) { return 192; }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM256
+ if (head_size <= 256) { return 256; }
+ #endif
+ return 256;
+}
+
+// b: batch_size
+// b_k: batch_size_k
+// s_q: seqlen_q
+// s_k: seqlen_k
+// s_k_new: seqlen_k_new
+// h: num_heads
+// h_k: num_heads_k
+// d: head_size
+std::vector<at::Tensor>
+mha_fwd(at::Tensor &q, // (b, s_q, h, d) or (total_q, h, d) if there is cu_seqlens_q
+ const at::Tensor &k, // (b_k, s_k, h_k, d) or (total_k, h_k, d) if there is cu_seqlens_k or (num_pages, page_size, h_k, d) if there is page_table.
+ const at::Tensor &v, // (b_k, s_k, h_k, d) or (total_k, h_k, d) if there is cu_seqlens_k or (num_pages, page_size, h_k, d) if there is page_table.
+ std::optional<const at::Tensor> &k_new_, // (b, s_k_new, h_k, d) or (total_k_new, h_k, d) if there is cu_seqlens_k_new
+ std::optional<const at::Tensor> &v_new_, // (b, s_k_new, h_k, d) or (total_k_new, h_k, d) if there is cu_seqlens_k_new
+ std::optional<at::Tensor> &out_, // (b, s_q, h, d) or (total_q, h, d) if there is cu_seqlens_q
+ std::optional<const at::Tensor> &cu_seqlens_q_, // b+1
+ std::optional<const at::Tensor> &cu_seqlens_k_, // b+1
+ std::optional<const at::Tensor> &cu_seqlens_k_new_, // b+1
+ std::optional<const at::Tensor> &seqused_q_, // b. If given, only this many elements of each batch element's queries and outputs are used.
+ std::optional<const at::Tensor> &seqused_k_, // b. If given, only this many elements of each batch element's keys are used.
+ std::optional<int> max_seqlen_q_,
+ // TODO: check if we need max_seqlen_k
+ std::optional<int> max_seqlen_k_,
+ std::optional<const at::Tensor> &page_table_, // (b_k, max_num_pages_per_seq)
+ std::optional<const at::Tensor> &kv_batch_idx_, // b. indices to index into the KV cache
+ std::optional<const at::Tensor> &leftpad_k_, // b
+ std::optional<const at::Tensor> &rotary_cos_, // seqlen_ro x (rotary_dim / 2)
+ std::optional<const at::Tensor> &rotary_sin_, // seqlen_ro x (rotary_dim / 2)
+ std::optional<at::Tensor> &q_descale_, // (b, h_k), not (b, h)
+ std::optional<at::Tensor> &k_descale_, // (b, h_k)
+ std::optional<at::Tensor> &v_descale_, // (b, h_k)
+ float const softmax_scale,
+ bool is_causal,
+ int window_size_left,
+ int window_size_right,
+ int sink_token_length,
+ float const softcap,
+ bool const is_rotary_interleaved, // if true, rotary combines indices 0 & 1, else indices 0 & rotary_dim / 2
+ int num_splits,
+ std::optional<bool> pack_gqa_,
+ int const sm_margin
+ ) {
+
+ auto dprops = at::cuda::getCurrentDeviceProperties();
+ bool is_sm8x = dprops->major >= 8;
+ TORCH_CHECK(is_sm8x, "FlashAttention only supports Ampere GPUs or newer.");
+
+ auto q_type = q.scalar_type();
+ TORCH_CHECK(q_type == at::ScalarType::Half || q_type == at::ScalarType::BFloat16 || q_type == at::ScalarType::Float8_e4m3fn,
+ "FlashAttention only supports fp16, bf16, and fp8_e4m3 data type");
+ if (dprops->major < 9) {
+ TORCH_CHECK(q_type == at::ScalarType::Half || q_type == at::ScalarType::BFloat16,
+ "FlashAttention on Ampere/Ada cards only supports fp16 and bf16 data type");
+ }
+ TORCH_CHECK(k.scalar_type() == q_type, "query and key must have the same dtype");
+ TORCH_CHECK(v.scalar_type() == q_type, "query and value must have the same dtype");
+
+ CHECK_DEVICE(q); CHECK_DEVICE(k); CHECK_DEVICE(v);
+
+ TORCH_CHECK(q.stride(-1) == 1, "Input tensor must have contiguous last dimension");
+ TORCH_CHECK(k.stride(-1) == 1, "Input tensor must have contiguous last dimension");
+ TORCH_CHECK(v.stride(-1) == 1, "Input tensor must have contiguous last dimension");
+
+ at::Tensor page_table;
+ const bool paged_KV = page_table_.has_value();
+ if (paged_KV) {
+ page_table = page_table_.value();
+ CHECK_DEVICE(page_table);
+ TORCH_CHECK(page_table.dtype() == torch::kInt32, "page_table must have dtype torch.int32");
+ TORCH_CHECK(page_table.stride(-1) == 1, "page_table must have contiguous last dimension");
+ }
+
+ at::Tensor cu_seqlens_q;
+ bool const is_varlen_q = cu_seqlens_q_.has_value();
+ if (is_varlen_q) {
+ cu_seqlens_q = cu_seqlens_q_.value();
+ CHECK_DEVICE(cu_seqlens_q); CHECK_CONTIGUOUS(cu_seqlens_q);
+ TORCH_CHECK(cu_seqlens_q.dtype() == torch::kInt32, "cu_seqlens_q must have dtype torch.int32");
+ TORCH_CHECK(max_seqlen_q_.has_value(), "max_seqlen_q must be provided if cu_seqlens_q is provided");
+ }
+ at::Tensor cu_seqlens_k;
+ bool const is_varlen_k = cu_seqlens_k_.has_value();
+ if (is_varlen_k) {
+ cu_seqlens_k = cu_seqlens_k_.value();
+ CHECK_DEVICE(cu_seqlens_k); CHECK_CONTIGUOUS(cu_seqlens_k);
+ TORCH_CHECK(cu_seqlens_k.dtype() == torch::kInt32, "cu_seqlens_k must have dtype torch.int32");
+ TORCH_CHECK(max_seqlen_k_.has_value(), "max_seqlen_k must be provided if cu_seqlens_k is provided");
+ TORCH_CHECK(!paged_KV, "If cu_seqlens_k is passed in, then page table is not supported");
+ TORCH_CHECK(!kv_batch_idx_.has_value(), "If cu_seqlens_k is passed in, then page table is not supported");
+ }
+ // This is what we will template on
+ bool const is_varlen = is_varlen_q || is_varlen_k || seqused_q_.has_value() || seqused_k_.has_value() || leftpad_k_.has_value();
+ #ifdef FLASHATTENTION_DISABLE_VARLEN
+ TORCH_CHECK(!is_varlen, "This flash attention build does not support varlen.");
+ #endif
+
+ auto const sizes = q.sizes();
+ const int batch_size = !is_varlen_q ? sizes[0] : cu_seqlens_q.size(0) - 1;
+ int seqlen_q = !is_varlen_q ? sizes[1] : max_seqlen_q_.value();
+ int total_q = !is_varlen_q ? batch_size * sizes[1] : sizes[0];
+ int num_heads = q.size(-2);
+ int const head_size = q.size(-1);
+ int const max_num_pages_per_seq = !paged_KV ? 0 : page_table.size(1);
+ int const num_pages = !paged_KV ? 0 : k.size(0);
+ int const page_size = !paged_KV ? 1 : k.size(1);
+ int const seqlen_k = !is_varlen_k ? (!paged_KV ? k.size(1) : max_num_pages_per_seq * page_size) : max_seqlen_k_.value();
+ int const total_k = !is_varlen_k ? batch_size * k.size(1) : k.size(0);
+ int const num_heads_k = k.size(-2);
+ int const batch_size_k = !paged_KV ? (!is_varlen_k ? k.size(0) : cu_seqlens_k.size(0) - 1) : page_table.size(0);
+ if (!kv_batch_idx_.has_value()) {
+ TORCH_CHECK(batch_size == batch_size_k, "batch_size must be equal to batch_size_k");
+ }
+ int const max_headdim = get_max_headdim();
+ TORCH_CHECK(head_size <= max_headdim, "FlashAttention forward only supports head dimension at most " + std::to_string(max_headdim));
+ TORCH_CHECK(num_heads % num_heads_k == 0, "Number of heads in key/value must divide number of heads in query");
+
+ // This needs to go before kBlockM & kBlockN since we rely on the correct window_size and is_causal to set kBlockM
+ // TODO: check this
+ if (window_size_left >= seqlen_k - 1) { window_size_left = -1; }
+ if (window_size_right >= seqlen_q - 1) { window_size_right = -1; }
+ if (is_causal) { window_size_right = 0; }
+ // There's a case where is_causal=false, window_size=(-1, 0). Then set_params_fprop will set params.is_causal=true.
+ // If we don't have is_causal here matching params.is_causal, we might get the wrong kBlockM.
+ is_causal = window_size_left < 0 && window_size_right == 0;
+
+ if (!is_varlen_q) {
+ CHECK_SHAPE(q, batch_size, seqlen_q, num_heads, head_size);
+ } else {
+ CHECK_SHAPE(q, total_q, num_heads, head_size);
+ CHECK_SHAPE(cu_seqlens_q, batch_size + 1);
+ }
+ if (!paged_KV) {
+ if (!is_varlen_k) {
+ CHECK_SHAPE(k, batch_size_k, seqlen_k, num_heads_k, head_size);
+ CHECK_SHAPE(v, batch_size_k, seqlen_k, num_heads_k, head_size);
+ } else {
+ CHECK_SHAPE(k, total_k, num_heads_k, head_size);
+ CHECK_SHAPE(v, total_k, num_heads_k, head_size);
+ CHECK_SHAPE(cu_seqlens_k, batch_size + 1);
+ }
+ } else {
+ CHECK_SHAPE(k, num_pages, page_size, num_heads_k, head_size);
+ CHECK_SHAPE(v, num_pages, page_size, num_heads_k, head_size);
+ CHECK_SHAPE(page_table, batch_size_k, max_num_pages_per_seq);
+ }
+
+ if (seqused_q_.has_value()){
+ auto seqused_q = seqused_q_.value();
+ TORCH_CHECK(seqused_q.dtype() == torch::kInt32, "seqused_q must have dtype int32");
+ CHECK_DEVICE(seqused_q); CHECK_CONTIGUOUS(seqused_q);
+ CHECK_SHAPE(seqused_q, batch_size);
+ }
+ if (seqused_k_.has_value()) {
+ auto seqused_k = seqused_k_.value();
+ TORCH_CHECK(seqused_k.dtype() == torch::kInt32, "seqused_k must have dtype int32");
+ CHECK_DEVICE(seqused_k); CHECK_CONTIGUOUS(seqused_k);
+ CHECK_SHAPE(seqused_k, batch_size);
+ }
+
+ int const alignment = q_type == torch::kFloat8_e4m3fn ? 16 : 8;
+ TORCH_CHECK(head_size % alignment == 0, "head_size should be a multiple of " + std::to_string(alignment));
+
+ auto opts = q.options();
+ auto out_type = q_type == at::ScalarType::Float8_e4m3fn ? at::ScalarType::BFloat16 : q_type;
+ at::Tensor out;
+ if (out_.has_value()) {
+ out = out_.value();
+ TORCH_CHECK(out.scalar_type() == out_type, "For FP16/BF16 input, output must have the same dtype as inputs. For FP8 input, output must have dtype BF16");
+ CHECK_DEVICE(out);
+ TORCH_CHECK(out.stride(-1) == 1, "Output tensor must have contiguous last dimension");
+ if (!is_varlen_q) {
+ CHECK_SHAPE(out, batch_size, seqlen_q, num_heads, head_size);
+ } else {
+ CHECK_SHAPE(out, total_q, num_heads, head_size);
+ }
+ } else {
+ out = torch::empty_like(q, opts.dtype(out_type));
+ }
+
+ auto round_multiple = [](int x, int m) { return (x + m - 1) / m * m; };
+ int const head_size_rounded = round_up_headdim(head_size);
+ int const seqlen_q_rounded = round_multiple(seqlen_q, 128);
+ int const seqlen_k_rounded = round_multiple(seqlen_k, 128);
+
+ // Otherwise the kernel will be launched from cuda:0 device
+ // Cast to char to avoid compiler warning about narrowing
+ at::cuda::CUDAGuard device_guard{(char)q.get_device()};
+
+ at::Tensor softmax_lse;
+ if (!is_varlen_q) {
+ softmax_lse = torch::empty({batch_size, num_heads, seqlen_q}, opts.dtype(at::kFloat));
+ } else {
+ softmax_lse = torch::empty({num_heads, total_q}, opts.dtype(at::kFloat));
+ }
+
+ Flash_fwd_params params;
+ set_params_fprop(params,
+ batch_size,
+ seqlen_q, seqlen_k,
+ seqlen_q_rounded, seqlen_k_rounded,
+ num_heads, num_heads_k,
+ head_size, head_size_rounded,
+ q, k, v, out,
+ !is_varlen_q ? nullptr : cu_seqlens_q.data_ptr(),
+ !is_varlen_k ? nullptr : cu_seqlens_k.data_ptr(),
+ seqused_q_.has_value() ? seqused_q_.value().data_ptr() : nullptr,
+ seqused_k_.has_value() ? seqused_k_.value().data_ptr() : nullptr,
+ softmax_lse.data_ptr(),
+ /*p_dropout=*/0.f,
+ softmax_scale,
+ window_size_left,
+ window_size_right,
+ softcap,
+ sm_margin);
+ params.total_q = total_q;
+ params.total_k = total_k;
+ params.sink_token_length = sink_token_length;
+ params.b_k = batch_size_k;
+
+ if (paged_KV) {
+ params.page_table = page_table.data_ptr<int>();
+ params.page_table_batch_stride = page_table.stride(0);
+ }
+ params.page_size = page_size;
+ params.num_pages = num_pages;
+
+ params.num_splits = num_splits <= 0 ? get_num_splits(params) : num_splits;
+ params.pack_gqa = pack_gqa_.has_value() ? pack_gqa_.value() : get_pack_gqa(params);
+
+ if (k_new_.has_value()) {
+ at::Tensor k_new, v_new;
+ TORCH_CHECK(v_new_.has_value(), "If k_new is supplied, v_new must also be passed in");
+ TORCH_CHECK(seqused_k_.has_value(), "If k_new is supplied, seqlens_k must also be passed in");
+ TORCH_CHECK(seqlen_q <= seqlen_k, "If k_new is supplied, it must have seqlen <= the seqlen of the KV cache");
+ at::Tensor cu_seqlens_k_new;
+ bool const is_varlen_k_new = cu_seqlens_k_new_.has_value();
+ if (is_varlen_k_new) {
+ cu_seqlens_k_new = cu_seqlens_k_new_.value();
+ CHECK_DEVICE(cu_seqlens_k_new); CHECK_CONTIGUOUS(cu_seqlens_k_new);
+ TORCH_CHECK(cu_seqlens_k_new.dtype() == torch::kInt32, "cu_seqlens_k_new must have dtype torch.int32");
+ }
+ k_new = k_new_.value();
+ v_new = v_new_.value();
+ TORCH_CHECK(k_new.dtype() == q_type, "k_new must have the same dtype as query");
+ TORCH_CHECK(v_new.dtype() == q_type, "v_new must have the same dtype as query");
+ CHECK_DEVICE(k_new); CHECK_DEVICE(v_new);
+ TORCH_CHECK(k_new.stride(-1) == 1, "k_new tensor must have contiguous last dimension");
+ TORCH_CHECK(v_new.stride(-1) == 1, "v_new tensor must have contiguous last dimension");
+ // We don't need max_seqlen_k_new, so seqlen_k_new can be whatever when is_varlen_k_new
+ int seqlen_k_new = !is_varlen_k_new ? k_new.size(1) : 0;
+ int total_k_new = !is_varlen_k_new ? batch_size * k_new.size(1): k_new.size(0);
+ if (!is_varlen_k_new) {
+ CHECK_SHAPE(k_new, batch_size, seqlen_k_new, num_heads_k, head_size);
+ CHECK_SHAPE(v_new, batch_size, seqlen_k_new, num_heads_k, head_size);
+ } else {
+ CHECK_SHAPE(k_new, total_k_new, num_heads_k, head_size);
+ CHECK_SHAPE(v_new, total_k_new, num_heads_k, head_size);
+ CHECK_SHAPE(cu_seqlens_k_new, batch_size + 1);
+ }
+ params.seqlen_knew = seqlen_k_new;
+ params.total_knew = total_k_new;
+ params.knew_ptr = k_new.data_ptr();
+ params.vnew_ptr = v_new.data_ptr();
+ // All stride are in elements, not bytes.
+ params.knew_row_stride = k_new.stride(-3);
+ params.vnew_row_stride = v_new.stride(-3);
+ params.knew_head_stride = k_new.stride(-2);
+ params.vnew_head_stride = v_new.stride(-2);
+ if (!is_varlen_k_new) {
+ params.knew_batch_stride = k_new.stride(0);
+ params.vnew_batch_stride = v_new.stride(0);
+ }
+ if (is_varlen_k_new) {
+ params.cu_seqlens_knew = static_cast<int*>(cu_seqlens_k_new.data_ptr());
+ }
+ }
+
+ if (leftpad_k_.has_value()) {
+ auto leftpad_k = leftpad_k_.value();
+ TORCH_CHECK(leftpad_k.dtype() == torch::kInt32, "leftpad_k must have dtype int32");
+ CHECK_DEVICE(leftpad_k); CHECK_CONTIGUOUS(leftpad_k);
+ CHECK_SHAPE(leftpad_k, batch_size);
+ params.leftpad_k = static_cast<int *>(leftpad_k.data_ptr());
+ }
+
+ if (rotary_cos_.has_value()) {
+ TORCH_CHECK(k_new_.has_value(), "If rotary cos/sin are provided, new key / value to be appended to KV cache must also be provided");
+ auto rotary_cos = rotary_cos_.value();
+ CHECK_DEVICE(rotary_cos); CHECK_CONTIGUOUS(rotary_cos);
+ params.rotary_dim = rotary_cos.size(1) * 2;
+ TORCH_CHECK(params.rotary_dim <= head_size, "rotary_dim must be <= headdim");
+ TORCH_CHECK(params.rotary_dim % 16 == 0, "Only rotary dimensions divisible by 16 are currently supported");
+ const int seqlen_ro = rotary_cos.size(0);
+ if (paged_KV) {
+ TORCH_CHECK(seqlen_ro >= seqlen_k, "cos/sin seqlen must be at least the seqlen of KV cache");
+ }
+ CHECK_SHAPE(rotary_cos, seqlen_ro, params.rotary_dim / 2);
+ TORCH_CHECK(rotary_cos.scalar_type() == q_type, "rotary_cos must have the same dtype as query");
+
+ TORCH_CHECK(rotary_sin_.has_value(), "If rotary cos is provided, rotary sin must also be provided");
+ auto rotary_sin = rotary_sin_.value();
+ CHECK_DEVICE(rotary_sin); CHECK_CONTIGUOUS(rotary_sin);
+ CHECK_SHAPE(rotary_sin, seqlen_ro, params.rotary_dim / 2);
+ TORCH_CHECK(rotary_sin.scalar_type() == q_type, "rotary_cos must have the same dtype as query");
+ params.rotary_cos_ptr = rotary_cos.data_ptr();
+ params.rotary_sin_ptr = rotary_sin.data_ptr();
+ params.is_rotary_interleaved = is_rotary_interleaved;
+ } else {
+ params.rotary_dim = 0;
+ }
+
+ if (kv_batch_idx_.has_value()) {
+ auto kv_batch_idx = kv_batch_idx_.value();
+ CHECK_DEVICE(kv_batch_idx); CHECK_CONTIGUOUS(kv_batch_idx);
+ TORCH_CHECK(kv_batch_idx.scalar_type() == torch::kInt32, "kv_batch_idx must have dtype int32");
+ params.kv_batch_idx = reinterpret_cast<int *>(kv_batch_idx.data_ptr());
+ }
+
+ at::Tensor out_accum, softmax_lse_accum;
+ auto outaccum_type = at::ScalarType::Float;
+ if (params.num_splits > 1) {
+ TORCH_CHECK(params.num_splits <= 256, "num_splits > 256 not supported");
+ if (!is_varlen_q) {
+ out_accum = torch::empty({params.num_splits, batch_size, num_heads, seqlen_q, head_size}, opts.dtype(outaccum_type));
+ softmax_lse_accum = torch::empty({params.num_splits, batch_size, num_heads, seqlen_q}, opts.dtype(at::kFloat));
+ params.oaccum_batch_stride = out_accum.stride(1);
+ params.lseaccum_batch_stride = softmax_lse_accum.stride(1);
+ } else {
+ out_accum = torch::empty({params.num_splits, num_heads, total_q, head_size}, opts.dtype(outaccum_type));
+ softmax_lse_accum = torch::empty({params.num_splits, num_heads, total_q}, opts.dtype(at::kFloat));
+ }
+ params.is_fp32 = false;
+ params.oaccum_ptr = out_accum.data_ptr();
+ params.softmax_lseaccum_ptr = softmax_lse_accum.data_ptr();
+ params.oaccum_split_stride = out_accum.stride(0);
+ params.oaccum_row_stride = out_accum.stride(-2);
+ params.oaccum_head_stride = out_accum.stride(-3);
+ params.lseaccum_split_stride = softmax_lse_accum.stride(0);
+ params.lseaccum_head_stride = softmax_lse_accum.stride(-2);
+ }
+
+ at::Tensor tile_count_semaphore;
+ // We don't use the persistent scheduler if Split and not Varlen
+ bool const persistent_scheduler = params.arch >= 90
+ ? (((params.is_causal || params.is_local) && (params.num_splits == 1)) || is_varlen)
+ : ((params.is_causal && !is_varlen) || (is_varlen && params.num_splits > 1));
+ if (persistent_scheduler) {
+ tile_count_semaphore = torch::zeros({1}, opts.dtype(torch::kInt32));
+ params.tile_count_semaphore = tile_count_semaphore.data_ptr<int>();
+ } else {
+ params.tile_count_semaphore = nullptr;
+ }
+
+ if (q_type == at::ScalarType::Float8_e4m3fn) {
+ if (q_descale_.has_value()) {
+ auto q_descale = q_descale_.value();
+ CHECK_DEVICE(q_descale);
+ CHECK_SHAPE(q_descale, batch_size, num_heads_k);
+ params.q_descale_ptr = q_descale.data_ptr<float>();
+ params.q_descale_batch_stride = q_descale.stride(0);
+ params.q_descale_head_stride = q_descale.stride(1);
+ } else {
+ params.q_descale_ptr = nullptr;
+ }
+ if (k_descale_.has_value()) {
+ auto k_descale = k_descale_.value();
+ CHECK_DEVICE(k_descale);
+ CHECK_SHAPE(k_descale, batch_size, num_heads_k);
+ params.k_descale_ptr = k_descale.data_ptr<float>();
+ params.k_descale_batch_stride = k_descale.stride(0);
+ params.k_descale_head_stride = k_descale.stride(1);
+ } else {
+ params.k_descale_ptr = nullptr;
+ }
+ if (v_descale_.has_value()) {
+ auto v_descale = v_descale_.value();
+ CHECK_DEVICE(v_descale);
+ CHECK_SHAPE(v_descale, batch_size, num_heads_k);
+ params.v_descale_ptr = v_descale.data_ptr<float>();
+ params.v_descale_batch_stride = v_descale.stride(0);
+ params.v_descale_head_stride = v_descale.stride(1);
+ } else {
+ params.v_descale_ptr = nullptr;
+ }
+ }
+
+ #ifdef FLASHATTENTION_DISABLE_LOCAL
+ TORCH_CHECK(!params.is_local, "This flash attention build does not support local attention.");
+ #endif
+ #ifdef FLASHATTENTION_DISABLE_SOFTCAP
+ TORCH_CHECK(params.softcap == 0.0, "This flash attention build does not support tanh softcapping.");
+ #endif
+ #ifdef FLASHATTENTION_DISABLE_SPLIT
+ TORCH_CHECK(params.num_splits == 1, "This flash attention build does not support splits.");
+ #endif
+ #ifdef FLASHATTENTION_DISABLE_PACKGQA
+ TORCH_CHECK(!params.pack_gqa || params.arch < 90 || params.page_table || params.num_splits > 1, "This flash attention build does not support pack_gqa.");
+ #endif
+ #ifdef FLASHATTENTION_DISABLE_PAGEDKV
+ TORCH_CHECK(!paged_KV, "This flash attention build does not support paged KV.");
+ #endif
+ #ifdef FLASHATTENTION_DISABLE_APPENDKV
+ TORCH_CHECK(!k_new_.has_value(), "This flash attention build does not support appending KV.");
+ #endif
+
+ if (total_q > 0 && (total_k + params.total_knew) > 0 && num_heads_k > 0) {
+ auto stream = at::cuda::getCurrentCUDAStream().stream();
+ run_mha_fwd(params, stream);
+ if (params.num_splits > 1) {
+ if (out_type == at::ScalarType::BFloat16) {
+ // Since we want output in BF16. Otherwise fwd_combine will output to FP16
+ params.is_bf16 = true;
+ }
+ // Unless there's seqused_q, for the purpose of attn_combine, we can just treat it as batch=1
+ // and seqlen = total_q, and don't need to dispatch to Varlen there.
+ // if (is_varlen_q && !seqused_q_.has_value()) {
+ if (is_varlen_q) {
+ params.b = 1;
+ params.seqlen_q = total_q;
+ }
+ run_mha_fwd_combine(params, stream);
+ }
+ } else if (total_q > 0 && num_heads_k > 0) {
+ // If seqlen_k == 0, then we have an empty tensor. We need to set the output to 0.
+ out.zero_();
+ softmax_lse.fill_(std::numeric_limits<float>::infinity());
+ }
+
+ // return {out, softmax_lse};
+ return {out, softmax_lse, out_accum, softmax_lse_accum};
+}
+
+void run_mha_bwd(Flash_bwd_params &params, cudaStream_t stream) {
+ #ifndef FLASHATTENTION_DISABLE_BACKWARD
+ // FP16_SWITCH(!params.is_bf16, [&] {
+ // HEADDIM_SWITCH(params.d, [&] {
+ // run_mha_bwd_<elem_type, kHeadDim>(params, stream);
+ // });
+ // });
+ ARCH_SWITCH(params.arch, Arch, [&] {
+ SOFTCAP_SWITCH(params.softcap > 0.f, Has_softcap, [&] {
+ if (!params.is_bf16) {
+ #ifndef FLASHATTENTION_DISABLE_FP16
+ #ifndef FLASHATTENTION_DISABLE_HDIM64
+ if (params.d <= 64) { return run_mha_bwd_<Arch, cutlass::half_t, 64, Has_softcap>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM96
+ if (params.d <= 96) { return run_mha_bwd_<Arch, cutlass::half_t, 96, Has_softcap>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM128
+ if (params.d <= 128) { return run_mha_bwd_<Arch, cutlass::half_t, 128, Has_softcap>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM192
+ if (params.d <= 192) { return run_mha_bwd_<Arch, cutlass::half_t, 192, Has_softcap>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM256
+ if (params.d <= 256) { return run_mha_bwd_<Arch, cutlass::half_t, 256, Has_softcap>(params, stream); }
+ #endif
+ #else
+ TORCH_CHECK(false, "This flash attention build does not support FP16.");
+ #endif
+ } else {
+ #ifndef FLASHATTENTION_DISABLE_HDIM64
+ if (params.d <= 64) { return run_mha_bwd_<Arch, cutlass::bfloat16_t, 64, Has_softcap>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM96
+ if (params.d <= 96) { return run_mha_bwd_<Arch, cutlass::bfloat16_t, 96, Has_softcap>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM128
+ if (params.d <= 128) { return run_mha_bwd_<Arch, cutlass::bfloat16_t, 128, Has_softcap>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM192
+ if (params.d <= 192) { return run_mha_bwd_<Arch, cutlass::bfloat16_t, 192, Has_softcap>(params, stream); }
+ #endif
+ #ifndef FLASHATTENTION_DISABLE_HDIM256
+ if (params.d <= 256) { return run_mha_bwd_<Arch, cutlass::bfloat16_t, 256, Has_softcap>(params, stream); }
+ #endif
+ }
+ });
+ });
+ #endif
+}
+
+
+// b: batch_size
+// s_q: seqlen_q
+// s_k: seqlen_k
+// h: num_heads
+// h_k: num_heads_k
+// d: head_size
+std::vector<at::Tensor> mha_bwd(
+ const at::Tensor &dout, // (b, s_q, h, d) or (total_q, h, d) if there is cu_seqlens_q
+ const at::Tensor &q, // (b, s_q, h, d) or (total_q, h, d) if there is cu_seqlens_q
+ const at::Tensor &k, // (b, s_k, h_k, d) or (total_k, h_k, d) if there is cu_seqlens_k
+ const at::Tensor &v, // (b, s_k, h_k, d) or (total_k, h_k, d) if there is cu_seqlens_k
+ const at::Tensor &out, // (b, s_q, h, d) or (total_q, h, d) if there is cu_seqlens_q
+ const at::Tensor &softmax_lse, // (b, h, s_q) or (h, total_q) if there is cu_seqlens_q
+ std::optional<at::Tensor> &dq_, // (b, s_q, h, d) or (total_q, h, d) if there is cu_seqlens_q
+ std::optional<at::Tensor> &dk_, // (b, s_k, h_k, d) or (total_k, h_k, d) if there is cu_seqlens_k
+ std::optional<at::Tensor> &dv_, // (b, s_k, h_k, d) or (total_k, h_k, d) if there is cu_seqlens_k
+ std::optional<const at::Tensor> &cu_seqlens_q_, // b+1
+ std::optional<const at::Tensor> &cu_seqlens_k_, // b+1
+ std::optional<const at::Tensor> &seqused_q_, // b. If given, only this many elements of each batch element's queries and outputs are used.
+ std::optional<const at::Tensor> &seqused_k_, // b. If given, only this many elements of each batch element's keys are used.
+ std::optional<int> max_seqlen_q_,
+ std::optional<int> max_seqlen_k_,
+ float const softmax_scale,
+ bool is_causal,
+ int window_size_left,
+ int window_size_right,
+ int const sink_token_length,
+ float const softcap,
+ bool const deterministic,
+ int const sm_margin) {
+
+ #ifdef FLASHATTENTION_DISABLE_BACKWARD
+ TORCH_CHECK(false, "This flash attention build does not support backward.");
+ #endif
+
+ auto dprops = at::cuda::getCurrentDeviceProperties();
+ bool is_sm8x = dprops->major >= 8;
+ TORCH_CHECK(is_sm8x, "FlashAttention only supports Ampere GPUs or newer.");
+
+ auto q_type = q.dtype();
+ TORCH_CHECK(q_type == torch::kFloat16 || q_type == torch::kBFloat16,
+ "FlashAttention only support fp16 and bf16 data type");
+ TORCH_CHECK(k.dtype() == q_type, "query and key must have the same dtype");
+ TORCH_CHECK(v.dtype() == q_type, "query and value must have the same dtype");
+ TORCH_CHECK(out.dtype() == q_type, "query and out must have the same dtype");
+ TORCH_CHECK(dout.dtype() == q_type, "query and dout must have the same dtype");
+
+ CHECK_DEVICE(q); CHECK_DEVICE(k); CHECK_DEVICE(v);
+ CHECK_DEVICE(out); CHECK_DEVICE(dout); CHECK_DEVICE(softmax_lse);
+
+ TORCH_CHECK(q.stride(-1) == 1, "Input tensor must have contiguous last dimension");
+ TORCH_CHECK(k.stride(-1) == 1, "Input tensor must have contiguous last dimension");
+ TORCH_CHECK(v.stride(-1) == 1, "Input tensor must have contiguous last dimension");
+ TORCH_CHECK(out.stride(-1) == 1, "out tensor must have contiguous last dimension");
+ TORCH_CHECK(dout.stride(-1) == 1, "dout tensor must have contiguous last dimension");
+
+ at::Tensor cu_seqlens_q;
+ bool const is_varlen_q = cu_seqlens_q_.has_value();
+ if (is_varlen_q) {
+ cu_seqlens_q = cu_seqlens_q_.value();
+ CHECK_DEVICE(cu_seqlens_q); CHECK_CONTIGUOUS(cu_seqlens_q);
+ TORCH_CHECK(cu_seqlens_q.dtype() == torch::kInt32, "cu_seqlens_q must have dtype torch.int32");
+ TORCH_CHECK(max_seqlen_q_.has_value(), "max_seqlen_q must be provided if cu_seqlens_q is provided");
+ }
+ at::Tensor cu_seqlens_k;
+ bool const is_varlen_k = cu_seqlens_k_.has_value();
+ if (is_varlen_k) {
+ cu_seqlens_k = cu_seqlens_k_.value();
+ CHECK_DEVICE(cu_seqlens_k); CHECK_CONTIGUOUS(cu_seqlens_k);
+ TORCH_CHECK(cu_seqlens_k.dtype() == torch::kInt32, "cu_seqlens_k must have dtype torch.int32");
+ TORCH_CHECK(max_seqlen_k_.has_value(), "max_seqlen_k must be provided if cu_seqlens_k is provided");
+ }
+ // This is what we will template on
+ bool const is_varlen = is_varlen_q || is_varlen_k || seqused_q_.has_value() || seqused_k_.has_value();
+ #ifdef FLASHATTENTION_DISABLE_VARLEN
+ TORCH_CHECK(!is_varlen, "This flash attention build does not support varlen.");
+ #endif
+
+ auto const sizes = q.sizes();
+ int const batch_size = !is_varlen_q ? sizes[0] : cu_seqlens_q.size(0) - 1;
+ int const seqlen_q = !is_varlen_q ? sizes[1] : max_seqlen_q_.value();
+ int const total_q = !is_varlen_q ? batch_size * sizes[1] : sizes[0];
+ int const num_heads = q.size(-2);
+ int const head_size = q.size(-1);
+ int const seqlen_k = !is_varlen_k ? k.size(1) : max_seqlen_k_.value();
+ int const total_k = !is_varlen_k ? batch_size * k.size(1) : k.size(0);
+ int const num_heads_k = k.size(-2);
+ TORCH_CHECK(head_size % 8 == 0, "head_size should be a multiple of 8");
+ int const max_headdim = get_max_headdim();
+ TORCH_CHECK(head_size <= max_headdim, "FlashAttention forward only supports head dimension at most " + std::to_string(max_headdim));
+ TORCH_CHECK(num_heads % num_heads_k == 0, "Number of heads in key/value must divide number of heads in query");
+
+ // This needs to go before kBlockM & kBlockN since we rely on the correct window_size and is_causal to set kBlockM
+ if (window_size_left >= seqlen_k - 1) { window_size_left = -1; }
+ if (window_size_right >= seqlen_q - 1) { window_size_right = -1; }
+ if (is_causal) { window_size_right = 0; }
+ // There's a case where is_causal=false, window_size=(-1, 0). Then set_params_bprop will set params.is_causal=true.
+ // If we don't have is_causal here matching params.is_causal, we might get the wrong kBlockM (and cause IMA).
+ is_causal = window_size_left < 0 && window_size_right == 0;
+
+ int const arch = at::cuda::getCurrentDeviceProperties()->major * 10 + at::cuda::getCurrentDeviceProperties()->minor;
+ int const head_size_rounded = round_up_headdim(head_size);
+ // Very important that these match the kernel configs
+ bool const is_local = (window_size_left >= 0 || window_size_right >= 0) && !is_causal;
+ int const kBlockM_sm90 = head_size_rounded <= 64 ? (is_causal && softcap > 0.0 ? 96 : 128)
+ : (head_size_rounded <= 96 ? 64
+ : (head_size_rounded <= 128 ? (is_causal || is_local || softcap > 0.0 ? 64 : 80)
+ : 64));
+ int const kBlockM_sm80 = head_size_rounded <= 64 ? 128 : 64;
+ int const kBlockM_sm86 = head_size_rounded <= 192 ? 64 : 32;
+ int const kBlockM = arch >= 90 ? kBlockM_sm90 : (arch == 86 || arch == 89 ? kBlockM_sm86 : kBlockM_sm80);
+ int const kBlockN_sm90 = head_size_rounded <= 128
+ ? 128
+ : (head_size_rounded <= 192 ? 96 : 80);
+ int const kBlockN_sm80 = head_size_rounded <= 128
+ ? 128
+ : (head_size_rounded <= 192 ? 80 : 64);
+ int const kBlockN_sm86 = head_size_rounded <= 64 ? 128
+ : (head_size_rounded <= 96 ? 128
+ : (head_size_rounded <= 128 ? 96
+ : (head_size_rounded <= 192 ? 64 : 64)));
+ int const kBlockN = arch >= 90 ? kBlockN_sm90 : (arch == 86 || arch == 89 ? kBlockN_sm86 : kBlockN_sm80);
+ auto round_multiple = [](int x, int m) { return (x + m - 1) / m * m; };
+ int const seqlen_q_rounded = round_multiple(seqlen_q, kBlockM);
+ int const seqlen_k_rounded = round_multiple(seqlen_k, kBlockN);
+ int const total_q_padded_rounded = round_multiple(total_q + batch_size * kBlockM, kBlockM);
+ int const total_k_padded_rounded = round_multiple(total_k + batch_size * kBlockN, kBlockN);
+
+ if (!is_varlen_q) {
+ CHECK_SHAPE(q, batch_size, seqlen_q, num_heads, head_size);
+ CHECK_SHAPE(out, batch_size, seqlen_q, num_heads, head_size);
+ CHECK_SHAPE(dout, batch_size, seqlen_q, num_heads, head_size);
+ } else {
+ CHECK_SHAPE(q, total_q, num_heads, head_size);
+ CHECK_SHAPE(out, total_q, num_heads, head_size);
+ CHECK_SHAPE(dout, total_q, num_heads, head_size);
+ CHECK_SHAPE(cu_seqlens_q, batch_size + 1);
+ }
+ if (!is_varlen_k) {
+ CHECK_SHAPE(k, batch_size, seqlen_k, num_heads_k, head_size);
+ CHECK_SHAPE(v, batch_size, seqlen_k, num_heads_k, head_size);
+ } else {
+ CHECK_SHAPE(k, total_k, num_heads_k, head_size);
+ CHECK_SHAPE(v, total_k, num_heads_k, head_size);
+ CHECK_SHAPE(cu_seqlens_k, batch_size + 1);
+ }
+
+ if (seqused_q_.has_value()){
+ auto seqused_q = seqused_q_.value();
+ TORCH_CHECK(seqused_q.dtype() == torch::kInt32, "seqused_q must have dtype int32");
+ CHECK_DEVICE(seqused_q); CHECK_CONTIGUOUS(seqused_q);
+ CHECK_SHAPE(seqused_q, batch_size);
+ }
+ if (seqused_k_.has_value()){
+ auto seqused_k = seqused_k_.value();
+ TORCH_CHECK(seqused_k.dtype() == torch::kInt32, "seqused_k must have dtype int32");
+ CHECK_DEVICE(seqused_k); CHECK_CONTIGUOUS(seqused_k);
+ CHECK_SHAPE(seqused_k, batch_size);
+ }
+
+ at::Tensor dq, dk, dv;
+ if (dq_.has_value()) {
+ dq = dq_.value();
+ TORCH_CHECK(dq.dtype() == q_type, "dq must have the same dtype as q");
+ CHECK_DEVICE(dq);
+ TORCH_CHECK(dq.stride(-1) == 1, "dq must have contiguous last dimension");
+ if (!is_varlen_q) {
+ CHECK_SHAPE(dq, batch_size, seqlen_q, num_heads, head_size);
+ } else {
+ CHECK_SHAPE(dq, total_q, num_heads, head_size);
+ }
+ } else {
+ dq = torch::empty_like(q);
+ }
+ if (dk_.has_value()) {
+ dk = dk_.value();
+ TORCH_CHECK(dk.dtype() == q_type, "dk must have the same dtype as q");
+ CHECK_DEVICE(dk);
+ TORCH_CHECK(dk.stride(-1) == 1, "dk must have contiguous last dimension");
+ if (!is_varlen_k) {
+ CHECK_SHAPE(dk, batch_size, seqlen_k, num_heads_k, head_size);
+ } else {
+ CHECK_SHAPE(dk, total_k, num_heads_k, head_size);
+ }
+ } else {
+ dk = torch::empty_like(k);
+ }
+ if (dv_.has_value()) {
+ dv = dv_.value();
+ TORCH_CHECK(dv.dtype() == q_type, "dv must have the same dtype as q");
+ CHECK_DEVICE(dv);
+ TORCH_CHECK(dv.stride(-1) == 1, "dv must have contiguous last dimension");
+ if (!is_varlen_k) {
+ CHECK_SHAPE(dv, batch_size, seqlen_k, num_heads_k, head_size);
+ } else {
+ CHECK_SHAPE(dv, total_k, num_heads_k, head_size);
+ }
+ } else {
+ dv = torch::empty_like(v);
+ }
+
+ // Otherwise the kernel will be launched from cuda:0 device
+ // Cast to char to avoid compiler warning about narrowing
+ at::cuda::CUDAGuard device_guard{(char)q.get_device()};
+
+ auto opts = q.options();
+ // Need softmax_d to have total_q_padded_rounded since we want its address to be aligned by 16/8 bytes for TMA / LDG.64
+ at::Tensor softmax_d, softmax_lse_log2;
+ if (!is_varlen) {
+ // Need softmax_d to have seqlen_q_rounded since we want its address to be aligned by 16/8 bytes for TMA / LDG.64
+ softmax_d = torch::empty({batch_size, num_heads, seqlen_q_rounded}, opts.dtype(at::kFloat));
+ softmax_lse_log2 = torch::empty({batch_size, num_heads, seqlen_q_rounded}, opts.dtype(at::kFloat));
+ } else {
+ softmax_d = torch::empty({num_heads, total_q_padded_rounded}, opts.dtype(at::kFloat));
+ softmax_lse_log2 = torch::empty({num_heads, total_q_padded_rounded}, opts.dtype(at::kFloat));
+ }
+ at::Tensor dq_accum, dk_accum, dv_accum;
+ if (!is_varlen) {
+ dq_accum = torch::empty({batch_size, num_heads, seqlen_q_rounded * head_size_rounded}, opts.dtype(at::kFloat));
+ } else {
+ dq_accum = torch::empty({num_heads, total_q_padded_rounded * head_size_rounded}, opts.dtype(at::kFloat));
+ }
+ if (num_heads_k != num_heads) { // MQA / GQA
+ if (!is_varlen) {
+ dk_accum = torch::zeros({batch_size, num_heads_k, seqlen_k_rounded * head_size_rounded}, opts.dtype(at::kFloat));
+ dv_accum = torch::zeros({batch_size, num_heads_k, seqlen_k_rounded * head_size_rounded}, opts.dtype(at::kFloat));
+ } else {
+ dk_accum = torch::zeros({num_heads_k, total_k_padded_rounded, head_size_rounded}, opts.dtype(at::kFloat));
+ dv_accum = torch::zeros({num_heads_k, total_k_padded_rounded, head_size_rounded}, opts.dtype(at::kFloat));
+ }
+ }
+
+ Flash_bwd_params params;
+ set_params_dgrad(params,
+ batch_size,
+ seqlen_q, seqlen_k,
+ seqlen_q_rounded, seqlen_k_rounded,
+ num_heads, num_heads_k,
+ head_size, head_size_rounded,
+ q, k, v, out,
+ dout, dq, dk, dv,
+ !is_varlen_q ? nullptr : cu_seqlens_q.data_ptr(),
+ !is_varlen_k ? nullptr : cu_seqlens_k.data_ptr(),
+ seqused_q_.has_value() ? seqused_q_.value().data_ptr() : nullptr,
+ seqused_k_.has_value() ? seqused_k_.value().data_ptr() : nullptr,
+ dq_accum.data_ptr(),
+ num_heads_k != num_heads ? dk_accum.data_ptr() : nullptr,
+ num_heads_k != num_heads ? dv_accum.data_ptr() : nullptr,
+ softmax_lse.data_ptr(),
+ softmax_d.data_ptr(),
+ /*p_dropout=*/0.f,
+ softmax_scale,
+ window_size_left,
+ window_size_right,
+ softcap,
+ deterministic,
+ sm_margin);
+ params.total_q = total_q;
+ params.total_k = total_k;
+ params.softmax_lse_log2_ptr = softmax_lse_log2.data_ptr();
+ params.sink_token_length = sink_token_length;
+
+ // auto tile_count_semaphore = (params.is_causal || params.is_local) ? torch::zeros({1}, opts.dtype(torch::kInt32)) : torch::empty({1}, opts.dtype(torch::kInt32));
+ // params.tile_count_semaphore = tile_count_semaphore.data_ptr<int>();
+ // Will be zero'ed out in the backward preprocess kernel
+ at::Tensor dq_semaphore = torch::empty({(seqlen_q + kBlockM - 1) / kBlockM, batch_size, num_heads}, opts.dtype(torch::kInt32));
+ params.dq_semaphore = dq_semaphore.data_ptr<int>();
+ if (num_heads_k != num_heads && params.deterministic) {
+ // TODO: do we need to zero them out?
+ at::Tensor dk_semaphore = torch::empty({(seqlen_k + kBlockN - 1) / kBlockN, batch_size, num_heads_k}, opts.dtype(torch::kInt32));
+ at::Tensor dv_semaphore = torch::empty({(seqlen_k + kBlockN - 1) / kBlockN, batch_size, num_heads_k}, opts.dtype(torch::kInt32));
+ params.dk_semaphore = dk_semaphore.data_ptr<int>();
+ params.dv_semaphore = dv_semaphore.data_ptr<int>();
+ }
+
+ #ifdef FLASHATTENTION_DISABLE_LOCAL
+ TORCH_CHECK(!params.is_local, "This flash attention build does not support local attention.");
+ #endif
+ #ifdef FLASHATTENTION_DISABLE_SOFTCAP
+ TORCH_CHECK(params.softcap == 0.0, "This flash attention build does not support tanh softcapping.");
+ #endif
+
+ if (total_q > 0 && total_k > 0 && num_heads_k > 0) {
+ auto stream = at::cuda::getCurrentCUDAStream().stream();
+ run_mha_bwd(params, stream);
+ } else if (total_k > 0 && num_heads_k > 0) {
+ // If seqlen_q == 0, then we have an empty tensor. We need to set the output to 0.
+ dk.zero_();
+ dv.zero_();
+ softmax_d.zero_();
+ } else if (total_q > 0 && num_heads_k > 0) {
+ dq.zero_();
+ softmax_d.zero_();
+ }
+
+ return { dq, dk, dv, softmax_d, softmax_lse_log2, dq_accum, dk_accum, dv_accum };
+}
+
+std::vector<at::Tensor>
+mha_combine(const at::Tensor &out_partial, // num_splits x batch_size x seqlen x num_heads x head_size
+ const at::Tensor &lse_partial, // num_splits x batch_size x seqlen x num_heads
+ std::optional<at::Tensor> out_, // batch_size x seqlen x num_heads x head_size
+ std::optional<at::ScalarType> out_dtype_
+ ) {
+
+ auto dprops = at::cuda::getCurrentDeviceProperties();
+ bool is_sm8x = dprops->major >= 8;
+ TORCH_CHECK(is_sm8x, "Attention combine function only supports Ampere GPUs or newer.");
+
+ auto out_partial_type = out_partial.scalar_type();
+ TORCH_CHECK(out_partial_type == at::ScalarType::Float, "Attention combine function only support fp32 data type");
+ TORCH_CHECK(lse_partial.scalar_type() == at::ScalarType::Float, "Attention combine function only support fp32 data type");
+
+ CHECK_DEVICE(out_partial); CHECK_DEVICE(lse_partial);
+
+ TORCH_CHECK(out_partial.stride(-1) == 1, "Input tensor must have contiguous last dimension");
+ TORCH_CHECK(lse_partial.stride(-2) == 1, "LSE tensor must be contiguous in the seqlen dimension");
+
+ const auto sizes = out_partial.sizes();
+
+ const int num_splits = sizes[0];
+ const int batch_size = sizes[1];
+ const int seqlen = sizes[2];
+ const int num_heads = sizes[3];
+ const int head_size_og = sizes[4];
+ TORCH_CHECK(head_size_og <= 256, "FlashAttention combine only supports head dimension at most 256");
+ TORCH_CHECK(num_splits <= 256, "FlashAttention combine only supports num_splits at most 256");
+
+ CHECK_SHAPE(out_partial, num_splits, batch_size, seqlen, num_heads, head_size_og);
+ CHECK_SHAPE(lse_partial, num_splits, batch_size, seqlen, num_heads);
+
+ int const alignment = 4;
+ at::Tensor out_partial_padded;
+ auto pad = [](at::Tensor x, int alignment) {
+ return x.size(-1) % alignment == 0 ? x : torch::nn::functional::pad(x, torch::nn::functional::PadFuncOptions({0, alignment - x.size(-1) % alignment}));
+ };
+ out_partial_padded = pad(out_partial, alignment);
+
+ auto round_multiple = [](int x, int m) { return (x + m - 1) / m * m; };
+ const int head_size = round_multiple(head_size_og, alignment);
+
+ auto opts = out_partial.options();
+ at::ScalarType out_type = out_dtype_.value_or(out_partial.scalar_type());
+ TORCH_CHECK(out_type == at::ScalarType::Float || out_type == at::ScalarType::BFloat16 || out_type == at::ScalarType::Half, "Output type must be FP32, FP16 or BF16");
+ at::Tensor out;
+ if (out_.has_value()) {
+ out = out_.value();
+ TORCH_CHECK(out.scalar_type() == out_type);
+ CHECK_DEVICE(out);
+ TORCH_CHECK(out.stride(-1) == 1, "Output tensor must have contiguous last dimension");
+ CHECK_SHAPE(out, batch_size, seqlen, num_heads, head_size_og);
+ if (head_size_og % alignment != 0) {
+ out = torch::empty({batch_size, seqlen, num_heads, head_size}, opts.dtype(out_type));
+ }
+ } else {
+ out = torch::empty({batch_size, seqlen, num_heads, head_size}, opts.dtype(out_type));
+ }
+
+ // Otherwise the kernel will be launched from cuda:0 device
+ // Cast to char to avoid compiler warning about narrowing
+ at::cuda::CUDAGuard device_guard{(char)out_partial.get_device()};
+
+ auto softmax_lse = torch::empty({batch_size, num_heads, seqlen}, opts.dtype(at::kFloat)).transpose(1, 2);
+
+ Flash_fwd_params params {}; // Need to reset the params to set everything to zero
+ params.is_fp32 = out_type == at::ScalarType::Float;
+ params.is_bf16 = out_type == at::ScalarType::BFloat16;
+ params.oaccum_ptr = out_partial_padded.data_ptr();
+ params.softmax_lseaccum_ptr = lse_partial.data_ptr();
+ params.o_ptr = out.data_ptr();
+ params.softmax_lse_ptr = softmax_lse.data_ptr();
+ params.b = batch_size;
+ params.h = num_heads;
+ params.seqlen_q = seqlen;
+ params.d = head_size;
+ params.num_splits = num_splits;
+ params.oaccum_split_stride = out_partial_padded.stride(0);
+ params.oaccum_row_stride = out_partial_padded.stride(2);
+ params.oaccum_head_stride = out_partial_padded.stride(3);
+ params.oaccum_batch_stride = out_partial_padded.stride(1);
+ params.lseaccum_split_stride = lse_partial.stride(0);
+ params.lseaccum_head_stride = lse_partial.stride(3);
+ params.lseaccum_batch_stride = lse_partial.stride(1);
+ params.o_row_stride = out.stride(1);
+ params.o_head_stride = out.stride(2);
+ params.o_batch_stride = out.stride(0);
+
+ if (seqlen > 0 && batch_size > 0) {
+ auto stream = at::cuda::getCurrentCUDAStream().stream();
+ run_mha_fwd_combine(params, stream);
+ }
+
+ at::Tensor out_padded = out;
+ if (head_size_og % alignment != 0) {
+ out = out.index({"...", torch::indexing::Slice(torch::indexing::None, head_size_og)});
+ // if (out_.has_value()) { out_.value().copy_(out); }
+ }
+
+ return {out, softmax_lse};
+}
+
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
+ m.doc() = "FlashAttention";
+ m.def("fwd", &mha_fwd, "Forward pass");
+ m.def("bwd", &mha_bwd, "Backward pass");
+ m.def("fwd_combine", &mha_combine, "Combine partial attention outputs");
+}
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_hdim128_bf16_sm90.cu b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_hdim128_bf16_sm90.cu
new file mode 100644
index 0000000..54fc817
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_hdim128_bf16_sm90.cu
@@ -0,0 +1,12 @@
+// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+// Splitting the different template instantiations to different files to speed up compilation.
+// This file is auto-generated. See "generate_kernels.py"
+
+#include "flash_bwd_launch_template.h"
+
+#ifndef FLASHATTENTION_DISABLE_HDIM128
+template<>
+void run_mha_bwd_<90, cutlass::bfloat16_t, 128, false>(Flash_bwd_params &params, cudaStream_t stream) {
+ run_mha_bwd_hdim128<90, cutlass::bfloat16_t, false>(params, stream);
+}
+#endif
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_kernel_sm80.h b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_kernel_sm80.h
new file mode 100644
index 0000000..b4fe262
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_kernel_sm80.h
@@ -0,0 +1,174 @@
+/******************************************************************************
+ * Copyright (c) 2024, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cute/tensor.hpp"
+
+#include <cutlass/cutlass.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+#include <cutlass/kernel_hardware_info.h>
+
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <class CollectiveMainloop_, class CollectiveEpilogue_, class TileScheduler_>
+class FlashAttnBwdSm80 {
+
+public:
+
+ // Type Aliases
+ static constexpr bool Is_causal = CollectiveMainloop_::Is_causal;
+ static constexpr bool Is_local = CollectiveMainloop_::Is_local;
+ static_assert(CollectiveMainloop_::Varlen == CollectiveEpilogue_::Varlen);
+ static constexpr bool Varlen = CollectiveMainloop_::Varlen;
+
+ // Mainloop derived types
+ using CollectiveMainloop = CollectiveMainloop_;
+ using TileShape_MNK = typename CollectiveMainloop::TileShape_MNK;
+ using TiledMmaSdP = typename CollectiveMainloop::TiledMmaSdP;
+ using TiledMmadKV = typename CollectiveMainloop::TiledMmadKV;
+ using ArchTag = typename CollectiveMainloop::ArchTag;
+ using MainloopArguments = typename CollectiveMainloop::Arguments;
+ using MainloopParams = typename CollectiveMainloop::Params;
+ static constexpr bool dKV_swapAB = CollectiveMainloop::dKV_swapAB;
+
+ // Epilogue derived types
+ using CollectiveEpilogue = CollectiveEpilogue_;
+ using EpilogueArguments = typename CollectiveEpilogue::Arguments;
+ using EpilogueParams = typename CollectiveEpilogue::Params;
+
+ static_assert(ArchTag::kMinComputeCapability >= 80);
+
+ using TileScheduler = TileScheduler_;
+ using TileSchedulerArguments = typename flash::TileSchedulerArguments;
+ using TileSchedulerParams = typename TileScheduler::Params;
+
+ static constexpr uint32_t NumThreads = CUTE_STATIC_V(size(TiledMmaSdP{}));
+ static constexpr uint32_t MaxThreadsPerBlock = CUTE_STATIC_V(size(TiledMmaSdP{}));
+ static constexpr uint32_t MinBlocksPerMultiprocessor = 1;
+
+ // Kernel level shared memory storage
+ struct SharedStorage {
+ struct TensorStorage : cute::aligned_struct<128> {
+ union {
+ typename CollectiveMainloop::TensorStorage mainloop;
+ typename CollectiveEpilogue::TensorStorage epilogue;
+ };
+ } tensors;
+
+ alignas(16) typename TileScheduler::SharedStorage smem_scheduler;
+
+ };
+
+ static constexpr int SharedStorageSize = sizeof(SharedStorage);
+
+ // Device side arguments
+ struct Arguments {
+ MainloopArguments mainloop{};
+ EpilogueArguments epilogue{};
+ cutlass::KernelHardwareInfo hw_info{};
+ TileSchedulerArguments scheduler{};
+ };
+
+ // Kernel entry point API
+ struct Params {
+ MainloopParams mainloop{};
+ EpilogueParams epilogue{};
+ cutlass::KernelHardwareInfo hw_info{};
+ TileSchedulerParams scheduler{};
+ };
+
+ //
+ // Methods
+ //
+
+ // Convert to underlying arguments. In this case, a simple copy for the aliased type.
+ static
+ Params
+ to_underlying_arguments(Arguments const& args) {
+ CUTLASS_TRACE_HOST("to_underlying_arguments():");
+
+ // Get SM count if needed, otherwise use user supplied SM count
+ int sm_count = args.hw_info.sm_count;
+ if (sm_count <= 0) {
+ CUTLASS_TRACE_HOST(" WARNING: Arguments do not include a valid SM count.\n"
+ " For optimal performance, populate the arguments KernelHardwareInfo struct with the SM count.");
+ sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(args.hw_info.device_id);
+ }
+
+ CUTLASS_TRACE_HOST("to_underlying_arguments(): Setting persistent grid SM count to " << sm_count);
+
+ cutlass::KernelHardwareInfo hw_info{args.hw_info.device_id, sm_count};
+ return {
+ CollectiveMainloop::to_underlying_arguments(args.mainloop),
+ CollectiveEpilogue::to_underlying_arguments(args.epilogue),
+ hw_info,
+ TileScheduler::to_underlying_arguments(args.scheduler)
+ };
+ }
+
+ // Computes the kernel launch grid shape based on runtime parameters
+ static dim3
+ get_grid_shape(Params const& params) {
+ return TileScheduler::get_grid_shape(params.scheduler, params.hw_info.sm_count);
+ }
+
+ static dim3
+ get_block_shape() {
+ return dim3(MaxThreadsPerBlock, 1, 1);
+ }
+
+ CUTLASS_DEVICE
+ void
+ operator()(Params const& params, char* smem_buf) {
+
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+
+ SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(smem_buf);
+
+ CollectiveMainloop collective_mainloop;
+ CollectiveEpilogue collective_epilogue;
+
+ TileScheduler scheduler(reinterpret_cast<typename TileScheduler::SharedStorage*>(&shared_storage.smem_scheduler));
+ // Initialize matmul objects.
+ TiledMmadKV tiled_mma_dKV;
+
+ scheduler.init_consumer();
+
+ int warp_idx = cutlass::canonical_warp_idx_sync();
+ CUTLASS_PRAGMA_NO_UNROLL
+ for (auto work_tile_info = warp_idx == 0 ? scheduler.template get_initial_work</*IsProducerWarp=*/true>(params.scheduler) : scheduler.template get_initial_work</*IsProducerWarp=*/false>(params.scheduler);
+ work_tile_info.is_valid(params.scheduler);
+ work_tile_info = warp_idx == 0 ? scheduler.template get_next_work</*IsProducerWarp=*/true>(params.scheduler, work_tile_info) : scheduler.template get_next_work</*IsProducerWarp=*/false>(params.scheduler, work_tile_info)) {
+
+ auto block_coord_ = work_tile_info.get_block_coord(params.scheduler);
+ auto [n_block, bidh, bidb, _ /*split_idx*/] = block_coord_;
+ cute::tuple<int32_t, int32_t, int32_t> block_coord = {n_block, bidh, bidb};
+
+ // dK and dV output accumulator.
+ Tensor tdKrdK = partition_fragment_C(tiled_mma_dKV, select<!dKV_swapAB ? 1 : 2, !dKV_swapAB? 2 : 1>(TileShape_MNK{}));
+ Tensor tdVrdV = partition_fragment_C(tiled_mma_dKV, select<!dKV_swapAB ? 1 : 2, !dKV_swapAB? 2 : 1>(TileShape_MNK{}));
+ bool tile_valid = collective_mainloop.mma(
+ params.mainloop, tdKrdK, tdVrdV, threadIdx.x, block_coord,
+ shared_storage);
+ scheduler.prefetch_next_work(params.scheduler, work_tile_info);
+ if (tile_valid) {
+ collective_epilogue.store(params.epilogue, tdKrdK, tdVrdV, shared_storage, tiled_mma_dKV,
+ threadIdx.x, block_coord);
+ } else {
+ collective_epilogue.store_zero(params.epilogue, threadIdx.x, block_coord);
+ }
+ }
+
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_kernel_sm90.h b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_kernel_sm90.h
new file mode 100644
index 0000000..7aa32a8
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_kernel_sm90.h
@@ -0,0 +1,285 @@
+
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cute/tensor.hpp"
+
+#include <cutlass/cutlass.h>
+#include <cutlass/arch/reg_reconfig.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+#include <cutlass/numeric_conversion.h>
+#include <cutlass/kernel_hardware_info.h>
+#include "cutlass/pipeline/pipeline.hpp"
+
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <class CollectiveMainloop_, class CollectiveEpilogue_, class TileScheduler_>
+class FlashAttnBwdSm90 {
+
+public:
+
+ // Type Aliases
+ static constexpr bool Is_causal = CollectiveMainloop_::Is_causal;
+ static constexpr bool Is_local = CollectiveMainloop_::Is_local;
+ static_assert(CollectiveMainloop_::Varlen == CollectiveEpilogue_::Varlen);
+ static constexpr bool Varlen = CollectiveMainloop_::Varlen;
+
+ // Mainloop derived types
+ using CollectiveMainloop = CollectiveMainloop_;
+ using TileShape_MNK = typename CollectiveMainloop::TileShape_MNK;
+ using TiledMmaSdP = typename CollectiveMainloop::TiledMmaSdP;
+ using TiledMmadKV = typename CollectiveMainloop::TiledMmadKV;
+ using ArchTag = typename CollectiveMainloop::ArchTag;
+ using ClusterShape = typename CollectiveMainloop::ClusterShape;
+ using MainloopArguments = typename CollectiveMainloop::Arguments;
+ using MainloopParams = typename CollectiveMainloop::Params;
+ static constexpr bool dKV_swapAB = CollectiveMainloop::dKV_swapAB;
+
+ // Epilogue derived types
+ using CollectiveEpilogue = CollectiveEpilogue_;
+ using EpilogueArguments = typename CollectiveEpilogue::Arguments;
+ using EpilogueParams = typename CollectiveEpilogue::Params;
+
+ static_assert(ArchTag::kMinComputeCapability >= 90);
+
+ using TileScheduler = TileScheduler_;
+ using TileSchedulerArguments = typename flash::TileSchedulerArguments;
+ using TileSchedulerParams = typename TileScheduler::Params;
+
+ static constexpr uint32_t NumLoadWarpGroups = 1;
+ static constexpr uint32_t NumMmaWarpGroups = CUTE_STATIC_V(size(TiledMmaSdP{})) / cutlass::NumThreadsPerWarpGroup;
+ static constexpr uint32_t MaxThreadsPerBlock = CUTE_STATIC_V(size(TiledMmaSdP{})) + (NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup);
+ static constexpr uint32_t MinBlocksPerMultiprocessor = 1;
+ static_assert(NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3);
+
+ /// Register requirement for Load and Math WGs
+ static constexpr uint32_t LoadRegisterRequirement = NumMmaWarpGroups == 2 ? 24 : 32;
+ static constexpr uint32_t MmaRegisterRequirement = NumMmaWarpGroups == 2 ? 240 : 160;
+ // If you want to print from the producer warp, you'd need to increase the number of registers
+ // Otherwise you'll get CUDA error.
+ // static constexpr uint32_t LoadRegisterRequirement = 40;
+ // static constexpr uint32_t MmaRegisterRequirement = NumMmaWarpGroups == 2 ? 232 : 152;
+
+ // Kernel level shared memory storage
+ struct SharedStorage {
+ struct TensorStorage : cute::aligned_struct<128> {
+ union {
+ typename CollectiveMainloop::TensorStorage mainloop;
+ typename CollectiveEpilogue::TensorStorage epilogue;
+ };
+ } tensors;
+
+ struct PipelineStorage : cute::aligned_struct<16> {
+ alignas(16) cutlass::arch::ClusterTransactionBarrier barrier_KV;
+ alignas(16) typename CollectiveMainloop::MainloopPipeline::SharedStorage pipeline_q;
+ alignas(16) typename CollectiveMainloop::MainloopPipeline_dO::SharedStorage pipeline_do;
+ alignas(16) typename TileScheduler::SharedStorage smem_scheduler;
+ } pipelines;
+
+ };
+
+ static constexpr int SharedStorageSize = sizeof(SharedStorage);
+
+ // Device side arguments
+ struct Arguments {
+ MainloopArguments mainloop{};
+ EpilogueArguments epilogue{};
+ cutlass::KernelHardwareInfo hw_info{};
+ TileSchedulerArguments scheduler{};
+ };
+
+ // Kernel entry point API
+ struct Params {
+ MainloopParams mainloop{};
+ EpilogueParams epilogue{};
+ cutlass::KernelHardwareInfo hw_info{};
+ TileSchedulerParams scheduler{};
+ };
+
+ //
+ // Methods
+ //
+
+ // Convert to underlying arguments. In this case, a simple copy for the aliased type.
+ static
+ Params
+ to_underlying_arguments(Arguments const& args) {
+ CUTLASS_TRACE_HOST("to_underlying_arguments():");
+
+ // Get SM count if needed, otherwise use user supplied SM count
+ int sm_count = args.hw_info.sm_count;
+ if (sm_count <= 0) {
+ CUTLASS_TRACE_HOST(" WARNING: Arguments do not include a valid SM count.\n"
+ " For optimal performance, populate the arguments KernelHardwareInfo struct with the SM count.");
+ sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(args.hw_info.device_id);
+ }
+
+ CUTLASS_TRACE_HOST("to_underlying_arguments(): Setting persistent grid SM count to " << sm_count);
+
+ cutlass::KernelHardwareInfo hw_info{args.hw_info.device_id, sm_count};
+ return {
+ CollectiveMainloop::to_underlying_arguments(args.mainloop),
+ CollectiveEpilogue::to_underlying_arguments(args.epilogue),
+ hw_info,
+ TileScheduler::to_underlying_arguments(args.scheduler)
+ };
+ }
+
+ // Computes the kernel launch grid shape based on runtime parameters
+ static dim3
+ get_grid_shape(Params const& params) {
+ return TileScheduler::get_grid_shape(params.scheduler, params.hw_info.sm_count);
+ }
+
+ static dim3
+ get_block_shape() {
+ return dim3(MaxThreadsPerBlock, 1, 1);
+ }
+
+ CUTLASS_DEVICE
+ void
+ operator()(Params const& params, char* smem_buf) {
+
+ static constexpr int NumMmaThreads = NumMmaWarpGroups * cutlass::NumThreadsPerWarpGroup;
+ static constexpr int NumCopyThreads = NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup;
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+
+ using MainloopPipeline = typename CollectiveMainloop::MainloopPipeline;
+ using PipelineParams = typename MainloopPipeline::Params;
+ using PipelineState = typename MainloopPipeline::PipelineState;
+ using MainloopPipeline_dO = typename CollectiveMainloop::MainloopPipeline_dO;
+ using PipelineParams_dO = typename MainloopPipeline_dO::Params;
+ using PipelineState_dO = typename MainloopPipeline_dO::PipelineState;
+ static constexpr bool Q_dO_same_stages = std::is_same_v<MainloopPipeline, MainloopPipeline_dO>;
+
+ SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(smem_buf);
+
+ int const lane_predicate = cute::elect_one_sync();
+ int const warp_idx = cutlass::canonical_warp_idx_sync();
+
+ // Issue Tma Descriptor Prefetch from a single thread
+ if (warp_idx == 0 && lane_predicate) {
+ CollectiveMainloop::prefetch_tma_descriptors(params.mainloop);
+ CollectiveEpilogue::prefetch_tma_descriptors(params.epilogue);
+ }
+
+ // Obtain warp index
+ int const warp_group_thread_idx = threadIdx.x % cutlass::NumThreadsPerWarpGroup;
+
+ PipelineParams pipeline_params;
+ pipeline_params.transaction_bytes = CollectiveMainloop::TmaTransactionBytesQ + CollectiveMainloop::TmaTransactionBytesLSE;
+ int warp_group_idx = cutlass::canonical_warp_group_idx();
+ pipeline_params.role = warp_group_idx == 0
+ ? MainloopPipeline::ThreadCategory::Producer
+ : MainloopPipeline::ThreadCategory::Consumer;
+ pipeline_params.is_leader = warp_group_thread_idx == 0;
+ pipeline_params.num_consumers = NumMmaThreads;
+
+ if (warp_idx == 0 && lane_predicate) {
+ shared_storage.pipelines.barrier_KV.init(1 /*numThreads*/);
+ }
+ // We're counting on pipeline_q to call cutlass::arch::fence_barrier_init();
+ MainloopPipeline pipeline_q(shared_storage.pipelines.pipeline_q, pipeline_params, ClusterShape{});
+ auto role_dO = warp_group_idx == 0
+ ? MainloopPipeline_dO::ThreadCategory::Producer
+ : MainloopPipeline_dO::ThreadCategory::Consumer;
+ PipelineParams_dO pipeline_params_dO {pipeline_params.transaction_bytes, role_dO, pipeline_params.is_leader, pipeline_params.num_consumers};
+ MainloopPipeline_dO pipeline_do(shared_storage.pipelines.pipeline_do, cute::conditional_return<Q_dO_same_stages>(pipeline_params, pipeline_params_dO), ClusterShape{});
+
+ CollectiveMainloop collective_mainloop;
+ CollectiveEpilogue collective_epilogue;
+
+ // We need this to guarantee that the Pipeline init is visible to all producers and consumer blocks in the Cluster
+ if constexpr (size(ClusterShape{}) > 1) {
+ cute::cluster_arrive_relaxed();
+ cute::cluster_wait();
+ } else {
+ __syncthreads();
+ }
+
+ if (warp_group_idx == 0) { // Producer
+ cutlass::arch::warpgroup_reg_dealloc<LoadRegisterRequirement>();
+
+ int warp_idx_in_warpgroup = __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0);
+ if (warp_idx_in_warpgroup == 0) { // Load K, V, and do TMA on Q and dO
+ PipelineState smem_pipe_write = cutlass::make_producer_start_state<MainloopPipeline>();
+ PipelineState_dO smem_pipe_write_do = cutlass::make_producer_start_state<MainloopPipeline_dO>();
+
+ TileScheduler scheduler(reinterpret_cast<typename TileScheduler::SharedStorage*>(&shared_storage.pipelines.smem_scheduler));
+ for (auto work_tile_info = scheduler.template get_initial_work</*IsProducerWarp=*/true>(params.scheduler);
+ work_tile_info.is_valid(params.scheduler);
+ work_tile_info = scheduler.template get_next_work</*IsProducerWarp=*/true>(params.scheduler, work_tile_info)) {
+ auto block_coord_ = work_tile_info.get_block_coord(params.scheduler);
+ auto [n_block, bidh, bidb, _ /*split_idx*/] = block_coord_;
+ cute::tuple<int32_t, int32_t, int32_t> block_coord = {n_block, bidh, bidb};
+ auto scheduler_prefetch = [&scheduler, &params, &work_tile_info]() {
+ scheduler.prefetch_next_work(params.scheduler, work_tile_info);
+ };
+ collective_mainloop.load(params.mainloop, pipeline_q, pipeline_do, smem_pipe_write,
+ smem_pipe_write_do, shared_storage, scheduler_prefetch, block_coord);
+ }
+ collective_mainloop.load_tail(pipeline_q, pipeline_do, smem_pipe_write, smem_pipe_write_do);
+ } else if (warp_idx_in_warpgroup == 1) {
+ TileScheduler scheduler(reinterpret_cast<typename TileScheduler::SharedStorage*>(&shared_storage.pipelines.smem_scheduler));
+ for (auto work_tile_info = scheduler.template get_initial_work</*IsProducerWarp=*/false>(params.scheduler);
+ work_tile_info.is_valid(params.scheduler);
+ work_tile_info = scheduler.template get_next_work</*IsProducerWarp=*/false>(params.scheduler, work_tile_info)) {
+ auto block_coord_ = work_tile_info.get_block_coord(params.scheduler);
+ auto [n_block, bidh, bidb, _ /*split_idx*/] = block_coord_;
+ cute::tuple<int32_t, int32_t, int32_t> block_coord = {n_block, bidh, bidb};
+ collective_mainloop.store_dq(params.mainloop, shared_storage, block_coord);
+ }
+ }
+ } else { // Consumer
+ cutlass::arch::warpgroup_reg_alloc<MmaRegisterRequirement>();
+
+ TileScheduler scheduler(reinterpret_cast<typename TileScheduler::SharedStorage*>(&shared_storage.pipelines.smem_scheduler));
+ // Initialize matmul objects.
+ TiledMmadKV tiled_mma_dKV;
+
+ PipelineState smem_pipe_read;
+ PipelineState_dO smem_pipe_read_do;
+
+ collective_mainloop.mma_init();
+ scheduler.init_consumer();
+
+ int work_idx = 0;
+ CUTLASS_PRAGMA_NO_UNROLL
+ for (auto work_tile_info = scheduler.template get_initial_work</*IsProducerWarp=*/false>(params.scheduler);
+ work_tile_info.is_valid(params.scheduler);
+ work_tile_info = scheduler.template get_next_work</*IsProducerWarp=*/false>(params.scheduler, work_tile_info)) {
+ auto block_coord_ = work_tile_info.get_block_coord(params.scheduler);
+ auto [n_block, bidh, bidb, _ /*split_idx*/] = block_coord_;
+ cute::tuple<int32_t, int32_t, int32_t> block_coord = {n_block, bidh, bidb};
+
+ // dK and dV output accumulator.
+ Tensor tdKrdK = partition_fragment_C(tiled_mma_dKV, select<!dKV_swapAB ? 1 : 2, !dKV_swapAB? 2 : 1>(TileShape_MNK{}));
+ Tensor tdVrdV = partition_fragment_C(tiled_mma_dKV, select<!dKV_swapAB ? 1 : 2, !dKV_swapAB? 2 : 1>(TileShape_MNK{}));
+ bool tile_valid = collective_mainloop.mma(
+ params.mainloop, pipeline_q, pipeline_do, smem_pipe_read, smem_pipe_read_do,
+ tdKrdK, tdVrdV, threadIdx.x - NumCopyThreads, work_idx, block_coord, shared_storage);
+ if (tile_valid) {
+ collective_epilogue.store(params.epilogue, tdKrdK, tdVrdV, shared_storage, tiled_mma_dKV,
+ threadIdx.x - NumCopyThreads, block_coord);
+ } else {
+ collective_epilogue.store_zero(params.epilogue, threadIdx.x - NumCopyThreads, block_coord);
+ }
+
+ }
+ collective_epilogue.store_tail();
+ }
+
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_launch_template.h b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_launch_template.h
new file mode 100644
index 0000000..635228e
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_launch_template.h
@@ -0,0 +1,377 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cute/tensor.hpp"
+
+#include "cutlass/device_kernel.h" // For device_kernel
+#include "cutlass/kernel_launch.h" // For kernel_launch
+#include "cutlass/cluster_launch.hpp" // For ClusterLauncher
+
+#include "static_switch.h"
+#include "flash.h"
+#include "flash_bwd_preprocess_kernel.h"
+#include "flash_bwd_postprocess_kernel.h"
+#include "tile_scheduler.hpp"
+#include "mainloop_bwd_sm90_tma_gmma_ws.hpp"
+#include "mainloop_bwd_sm80.hpp"
+#include "epilogue_bwd.hpp"
+#include "flash_bwd_kernel_sm90.h"
+#include "flash_bwd_kernel_sm80.h"
+
+using namespace cute;
+
+template <int Arch, int kHeadDim, int kBlockM, int kBlockN, typename Element,
+ bool Is_causal, bool Is_local, bool Has_softcap, bool Varlen, bool Deterministic, bool GQA,
+ int Stages_dO=2, int Stages_dS_or_QSm80=2,
+ bool SdP_swapAB=true, bool dKV_swapAB=false, bool dQ_swapAB=false,
+ int NumMmaWarpGroups=2, int AtomLayoutMSdP=1, int AtomLayoutNdKV=2, int AtomLayoutMdQ=1,
+ bool V_in_regs=false>
+void run_flash_bwd(Flash_bwd_params &params, cudaStream_t stream) {
+ static_assert(!(Is_causal && Is_local), "Is_causal and Is_local cannot be true at the same time.");
+ using ElementAccum = float;
+ using ArchTag = std::conditional_t<Arch >= 90, cutlass::arch::Sm90, cutlass::arch::Sm80>;
+
+ int const total_q_padded_rounded = cute::round_up(params.total_q + params.b * kBlockM, kBlockM);
+ int const total_k_padded_rounded = cute::round_up(params.total_k + params.b * kBlockN, kBlockN);
+ bool const is_varlen_q = params.cu_seqlens_q;
+ bool const is_varlen_k = params.cu_seqlens_k;
+ int seqlen_q = !is_varlen_q ? params.seqlen_q : params.total_q;
+ int seqlen_k = !is_varlen_k ? params.seqlen_k : params.total_k;
+ int seqlen_q_rounded = !is_varlen_q ? params.seqlen_q_rounded : total_q_padded_rounded;
+ int seqlen_k_rounded = !is_varlen_k ? params.seqlen_k_rounded : total_k_padded_rounded;
+ int batch_q = !is_varlen_q ? params.b : 1;
+ int batch_k = !is_varlen_k ? params.b : 1;
+
+ using TileShape_MK = cute::Shape<Int<kBlockM>, Int<kHeadDim>>;
+ using PreprocessKernel = flash::FlashAttnBwdPreprocess<TileShape_MK, Element, ElementAccum, ArchTag, /*Clear_dQaccum=*/true, Varlen>;
+ typename PreprocessKernel::Arguments preprocess_args {
+ static_cast<Element const*>(params.o_ptr),
+ {seqlen_q, params.d, params.h, batch_q}, // shape_O
+ {params.o_row_stride, _1{}, params.o_head_stride, !is_varlen_q ? params.o_batch_stride : 0}, // stride_O
+ static_cast<Element const*>(params.do_ptr),
+ {params.do_row_stride, _1{}, params.do_head_stride, !is_varlen_q ? params.do_batch_stride : 0}, // stride_dO
+ static_cast<float*>(params.dsoftmax_sum),
+ {seqlen_q_rounded, params.h, batch_q}, // shape_dPsum
+ {_1{}, seqlen_q_rounded, !is_varlen_q ? params.h * params.seqlen_q_rounded : 0}, // stride_dPsum
+ static_cast<float*>(params.softmax_lse_ptr),
+ {_1{}, seqlen_q, !is_varlen_q ? params.h * params.seqlen_q : 0}, // stride_LSE
+ static_cast<float*>(params.softmax_lse_log2_ptr),
+ {_1{}, seqlen_q_rounded, !is_varlen_q ? params.h * params.seqlen_q_rounded : 0}, // stride_LSE_log2
+ static_cast<ElementAccum*>(params.dq_accum_ptr),
+ {seqlen_q_rounded * params.d_rounded, params.h, batch_q}, // shape_dQaccum
+ {_1{}, seqlen_q_rounded * params.d_rounded, !is_varlen_q ? params.d_rounded * seqlen_q_rounded * params.h : 0}, // stride_dQaccum
+ params.b,
+ params.dq_semaphore,
+ params.cu_seqlens_q,
+ params.seqused_q
+ };
+ typename PreprocessKernel::Params preprocess_params = PreprocessKernel::to_underlying_arguments(preprocess_args);
+ int num_m_block = cute::ceil_div(params.seqlen_q, kBlockM);
+ dim3 grid_m(num_m_block, params.h, params.b);
+ cutlass::kernel_launch<PreprocessKernel>(grid_m, PreprocessKernel::MaxThreadsPerBlock, PreprocessKernel::SharedStorageSize, stream, preprocess_params, false /*launch_with_pdl*/);
+ CHECK_CUDA_KERNEL_LAUNCH();
+
+ using TileShape_MNK = cute::Shape<Int<kBlockM>, Int<kBlockN>, Int<kHeadDim>>;
+ using ClusterShape = cute::Shape<_1, Int<1>, _1>; // Currently doesn't not support cluster
+ // Stages_dS_or_QSm80 is Stages_dS if Sm90 and Stages if Sm80
+ static constexpr int Stages = Arch >= 90 ? 2 : Stages_dS_or_QSm80;
+ static constexpr int Stages_dS = Arch >= 90 ? Stages_dS_or_QSm80 : 1;
+ using CollectiveMainloop = std::conditional_t<
+ Arch >= 90,
+ flash::CollectiveMainloopBwdSm90<Stages, Stages_dO, Stages_dS, ClusterShape, TileShape_MNK, Element, ElementAccum, cutlass::arch::Sm90,
+ Is_causal, Is_local, Has_softcap, Varlen, Deterministic,
+ SdP_swapAB, dKV_swapAB, dQ_swapAB, NumMmaWarpGroups, AtomLayoutMSdP, AtomLayoutNdKV, AtomLayoutMdQ, V_in_regs>,
+ flash::CollectiveMainloopBwdSm80<Stages, Stages_dO, TileShape_MNK, Element, ElementAccum, cutlass::arch::Sm80,
+ Is_causal, Is_local, Has_softcap, Varlen, Deterministic,
+ SdP_swapAB, dKV_swapAB, dQ_swapAB, NumMmaWarpGroups, AtomLayoutMSdP, AtomLayoutNdKV, AtomLayoutMdQ, V_in_regs>
+ >;
+ using CollectiveEpilogue = std::conditional_t<
+ !GQA,
+ flash::CollectiveEpilogueBwd<TileShape_MNK, Element, ArchTag, CollectiveMainloop::NumMmaThreads, Varlen, dKV_swapAB, NumMmaWarpGroups * (Arch >= 90 ? 1 : cutlass::NumWarpsPerWarpGroup) / AtomLayoutNdKV>,
+ flash::CollectiveEpilogueBwdGQA<TileShape_MNK, ElementAccum, ArchTag, CollectiveMainloop::NumMmaThreads, Varlen, Deterministic>
+ >;
+ using Scheduler = flash::SingleTileScheduler<Varlen, false /*Split*/, false /*PackGQA*/, kBlockN>;
+ using AttnKernel = std::conditional_t<
+ Arch >= 90,
+ flash::enable_sm90_or_later<flash::FlashAttnBwdSm90<CollectiveMainloop, CollectiveEpilogue, Scheduler>>,
+ flash::enable_sm80_to_sm89<flash::FlashAttnBwdSm80<CollectiveMainloop, CollectiveEpilogue, Scheduler>>
+ >;
+
+ typename CollectiveMainloop::Arguments mainloop_args {
+ static_cast<Element const*>(params.q_ptr),
+ {seqlen_q, params.d, params.h, batch_q}, // shape_Q
+ {params.q_row_stride, _1{}, params.q_head_stride, !is_varlen_q ? params.q_batch_stride : 0}, // stride_Q
+ static_cast<Element const*>(params.k_ptr),
+ {seqlen_k, params.d, params.h_k, batch_k}, // shape_K
+ {params.k_row_stride, _1{}, params.k_head_stride, !is_varlen_k ? params.k_batch_stride : 0}, // stride_K
+ static_cast<Element const*>(params.v_ptr),
+ {params.v_row_stride, _1{}, params.v_head_stride, !is_varlen_k ? params.v_batch_stride : 0}, // stride_V
+ static_cast<Element const*>(params.do_ptr),
+ {params.do_row_stride, _1{}, params.do_head_stride, !is_varlen_q ? params.do_batch_stride : 0}, // stride_dO
+ static_cast<ElementAccum*>(params.dq_accum_ptr),
+ {seqlen_q_rounded * params.d_rounded, params.h, batch_q}, // shape_dQaccum
+ {_1{}, seqlen_q_rounded * params.d_rounded, !is_varlen_q ? params.d_rounded * params.seqlen_q_rounded * params.h : 0}, // stride_dQaccum
+ static_cast<float*>(params.softmax_lse_log2_ptr),
+ {seqlen_q_rounded, params.h, batch_q}, // shape_LSE
+ {_1{}, seqlen_q_rounded, !is_varlen_q ? params.h * params.seqlen_q_rounded : 0}, // stride_LSE_log2
+ static_cast<float*>(params.dsoftmax_sum),
+ {_1{}, seqlen_q_rounded, !is_varlen_q ? params.h * params.seqlen_q_rounded : 0}, // stride_dPsum
+ params.scale_softmax,
+ params.window_size_left, params.window_size_right, params.sink_token_length,
+ params.softcap,
+ params.b,
+ params.dq_semaphore,
+ params.cu_seqlens_q, params.cu_seqlens_k,
+ params.seqused_q, params.seqused_k
+ };
+ // The case work with GQA is ugly but idk how to fix it.
+ typename CollectiveEpilogue::Arguments epilogue_args {
+ static_cast<typename CollectiveEpilogue::Element*>(!GQA ? params.dk_ptr : params.dk_accum_ptr),
+ [&] {
+ if constexpr (!GQA) {
+ return typename CollectiveEpilogue::ShapedKV {seqlen_k, params.d, params.h, batch_k}; // shape_dK
+ } else {
+ return typename CollectiveEpilogue::ShapedKV {seqlen_k_rounded * params.d_rounded, params.h_k, batch_k}; // shape_dKaccum
+ }
+ }(),
+ [&] {
+ if constexpr (!GQA) {
+ return typename CollectiveEpilogue::StridedKV {params.dk_row_stride, _1{}, params.dk_head_stride, !is_varlen_k ? params.dk_batch_stride : 0}; // stride_dK
+ } else {
+ return typename CollectiveEpilogue::StridedKV {_1{}, params.d_rounded * seqlen_k_rounded, !is_varlen_k ? params.h_k * params.d_rounded * params.seqlen_k_rounded : 0}; // stride_dKaccum
+ }
+ }(),
+ static_cast<typename CollectiveEpilogue::Element*>(!GQA ? params.dv_ptr : params.dv_accum_ptr),
+ [&] {
+ if constexpr (!GQA) {
+ return typename CollectiveEpilogue::StridedKV {params.dv_row_stride, _1{}, params.dv_head_stride, !is_varlen_k ? params.dv_batch_stride : 0}; // stride_dV
+ } else {
+ return typename CollectiveEpilogue::StridedKV {_1{}, params.d_rounded * seqlen_k_rounded, !is_varlen_k ? params.h_k * params.d_rounded * params.seqlen_k_rounded : 0}; // stride_dVaccum
+ }
+ }(),
+ params.h,
+ params.dk_semaphore,
+ params.dv_semaphore,
+ params.cu_seqlens_k,
+ params.seqused_k,
+ };
+
+ int num_blocks_n = cutlass::ceil_div(params.seqlen_k, get<1>(TileShape_MNK{}));
+ num_blocks_n = cutlass::round_up(num_blocks_n, size<1>(ClusterShape{}));
+ typename flash::TileSchedulerArguments scheduler_args {
+ num_blocks_n, params.h, params.b, 1 /*num_splits*/,
+ params.h / params.h_k,
+ params.seqlen_k,
+ params.seqlen_q, params.d, sizeof(Element),
+ params.tile_count_semaphore, params.cu_seqlens_k, params.seqused_k
+ };
+
+ int device;
+ cudaGetDevice(&device);
+ typename AttnKernel::Params kernel_params = AttnKernel::to_underlying_arguments({
+ mainloop_args, epilogue_args, {device, params.num_sm}, scheduler_args
+ });
+
+ dim3 grid_dims = AttnKernel::get_grid_shape(kernel_params);
+ dim3 block_dims = AttnKernel::get_block_shape();
+ int smem_size = AttnKernel::SharedStorageSize;
+ // int smem_size_q = sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_q));
+ // int smem_size_do = sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_do));
+ // int smem_size_ds = sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_ds));
+ // int smem_size_dqacc = [&] {
+ // if constexpr (Arch >= 90) {
+ // return sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_dqacc));
+ // } else {
+ // return 0;
+ // }
+ // }();
+ // int smem_size_k = sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_k));
+ // int smem_size_v = sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_v));
+ // int smem_size_lse = sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_lse));
+ // int smem_size_dpsum = sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_dpsum));
+ // printf("smem_size = %d, q = %d, k = %d, v = %d, do = %d, ds = %d, dqacc = %d, lse = %d, dpsum = %d\n", smem_size, smem_size_q, smem_size_k, smem_size_v, smem_size_do, smem_size_ds, smem_size_dqacc, smem_size_lse, smem_size_dpsum);
+ if constexpr (size(ClusterShape{}) > 1) {
+ void const* kernel = (void const*) cutlass::device_kernel<AttnKernel>;
+ if (smem_size >= 48 * 1024) {
+ CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
+ }
+ dim3 cluster_dims(size<0>(ClusterShape{}), size<1>(ClusterShape{}), size<2>(ClusterShape{}));
+ cutlass::ClusterLauncher::launch(
+ grid_dims, cluster_dims, block_dims, smem_size, stream, kernel, kernel_params, false /*launch_with_pdl*/);
+ } else {
+ if (smem_size >= 48 * 1024) {
+ CHECK_CUDA(cudaFuncSetAttribute(cutlass::device_kernel<AttnKernel>, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
+ }
+ cutlass::kernel_launch<AttnKernel>(grid_dims, block_dims, smem_size, stream, kernel_params, false /*launch_with_pdl*/);
+ }
+ CHECK_CUDA_KERNEL_LAUNCH();
+
+ using PostprocessKernel = flash::FlashAttnBwdPostprocessConvertdQ<TileShape_MK, Element, ElementAccum, ArchTag,
+ AttnKernel::CollectiveMainloop::NumMmaThreads,
+ typename AttnKernel::CollectiveMainloop::TiledMmadQ,
+ AttnKernel::CollectiveMainloop::dQ_swapAB
+ >;
+ typename PostprocessKernel::Arguments postprocess_args {
+ static_cast<ElementAccum const*>(params.dq_accum_ptr),
+ {seqlen_q_rounded * params.d_rounded, params.h, batch_q}, // shape_dQaccum
+ {_1{}, seqlen_q_rounded * params.d_rounded, !is_varlen_q ? params.d_rounded * params.seqlen_q_rounded * params.h : 0}, // stride_dQaccum
+ static_cast<Element*>(params.dq_ptr),
+ {seqlen_q, params.d, params.h, batch_q}, // shape_dQ
+ {params.dq_row_stride, _1{}, params.dq_head_stride, params.dq_batch_stride}, // stride_dQ
+ params.scale_softmax,
+ params.cu_seqlens_q,
+ params.seqused_q
+ };
+ typename PostprocessKernel::Params postprocess_params = PostprocessKernel::to_underlying_arguments(postprocess_args);
+ int num_m_block_postprocess = cute::ceil_div(params.seqlen_q, get<0>(TileShape_MK{}));
+ dim3 grid_m_postprocess(num_m_block_postprocess, params.h, params.b);
+ int smem_size_postprocess = PostprocessKernel::SharedStorageSize;
+ if (smem_size_postprocess >= 48 * 1024) {
+ CHECK_CUDA(cudaFuncSetAttribute(cutlass::device_kernel<PostprocessKernel>, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size_postprocess));
+ }
+ cutlass::kernel_launch<PostprocessKernel>(grid_m_postprocess, PostprocessKernel::MaxThreadsPerBlock, smem_size_postprocess, stream, postprocess_params, false /*launch_with_pdl*/);
+ CHECK_CUDA_KERNEL_LAUNCH();
+
+ if constexpr (GQA) {
+ using TileShape_NK = cute::Shape<Int<kBlockN>, Int<kHeadDim>>;
+ using PostprocessKerneldKV = flash::FlashAttnBwdPostprocessConvertdQ<TileShape_NK, Element, ElementAccum, ArchTag,
+ AttnKernel::CollectiveEpilogue::NumEpilogueThreads,
+ typename AttnKernel::CollectiveMainloop::TiledMmadKV,
+ AttnKernel::CollectiveMainloop::dKV_swapAB
+ >;
+ typename PostprocessKerneldKV::Arguments postprocess_dK_args {
+ static_cast<ElementAccum const*>(params.dk_accum_ptr),
+ {seqlen_k_rounded * params.d_rounded, params.h_k, batch_k}, // shape_dKaccum
+ {_1{}, seqlen_k_rounded * params.d_rounded, !is_varlen_k ? params.d_rounded * params.seqlen_k_rounded * params.h_k : 0}, // stride_dKaccum
+ static_cast<Element*>(params.dk_ptr),
+ {seqlen_k, params.d, params.h_k, batch_k}, // shape_dK
+ {params.dk_row_stride, _1{}, params.dk_head_stride, params.dk_batch_stride}, // stride_dK
+ 1.f,
+ params.cu_seqlens_k,
+ params.seqused_k
+ };
+ typename PostprocessKerneldKV::Params postprocess_dK_params = PostprocessKerneldKV::to_underlying_arguments(postprocess_dK_args);
+ typename PostprocessKerneldKV::Arguments postprocess_dV_args {
+ static_cast<ElementAccum const*>(params.dv_accum_ptr),
+ {seqlen_k_rounded * params.d_rounded, params.h_k, batch_k}, // shape_dVaccum
+ {_1{}, seqlen_k_rounded * params.d_rounded, !is_varlen_k ? params.d_rounded * params.seqlen_k_rounded * params.h_k : 0}, // stride_dVaccum
+ static_cast<Element*>(params.dv_ptr),
+ {seqlen_k, params.d, params.h_k, batch_k}, // shape_dV
+ {params.dv_row_stride, _1{}, params.dv_head_stride, params.dv_batch_stride}, // stride_dV
+ 1.f,
+ params.cu_seqlens_k,
+ params.seqused_k
+ };
+ typename PostprocessKerneldKV::Params postprocess_dV_params = PostprocessKerneldKV::to_underlying_arguments(postprocess_dV_args);
+ int num_n_block_postprocess = cute::ceil_div(params.seqlen_k, get<0>(TileShape_NK{}));
+ dim3 grid_n_postprocess(num_n_block_postprocess, params.h_k, params.b);
+ int smem_size_postprocess = PostprocessKerneldKV::SharedStorageSize;
+ if (smem_size_postprocess >= 48 * 1024) {
+ CHECK_CUDA(cudaFuncSetAttribute(cutlass::device_kernel<PostprocessKerneldKV>, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size_postprocess));
+ }
+ cutlass::kernel_launch<PostprocessKerneldKV>(grid_n_postprocess, PostprocessKerneldKV::MaxThreadsPerBlock, smem_size_postprocess, stream, postprocess_dK_params, false /*launch_with_pdl*/);
+ CHECK_CUDA_KERNEL_LAUNCH();
+ cutlass::kernel_launch<PostprocessKerneldKV>(grid_n_postprocess, PostprocessKerneldKV::MaxThreadsPerBlock, smem_size_postprocess, stream, postprocess_dV_params, false /*launch_with_pdl*/);
+ CHECK_CUDA_KERNEL_LAUNCH();
+ }
+
+}
+
+template<int Arch, typename T, int kBlockM, int kBlockN, int kHeadDim, bool Is_causal, bool Is_local, bool Has_softcap,
+ int Stages_dO=2, int Stages_dS_or_QSm80=2,
+ bool SdP_swapAB=true, bool dKV_swapAB=false, bool dQ_swapAB=false,
+ int NumMmaWarpGroups=2, int AtomLayoutMSdP=1, int AtomLayoutNdKV=2, int AtomLayoutMdQ=1,
+ bool V_in_regs=false>
+void run_mha_bwd_dispatch(Flash_bwd_params &params, cudaStream_t stream) {
+ VARLEN_SWITCH(params.cu_seqlens_q != nullptr || params.cu_seqlens_k != nullptr, Varlen, [&] {
+ BOOL_SWITCH(params.h != params.h_k, GQA, [&] {
+// BOOL_SWITCH(params.deterministic, Deterministic, [&] {
+ // run_flash_bwd<kHeadDim, kBlockM, kBlockN, T, Is_causal, Is_local, Has_softcap, Varlen, false, GQA, Stages_dO, Stages_dS_or_QSm80, SdP_swapAB, dKV_swapAB, dQ_swapAB, NumMmaWarpGroups, AtomLayoutMSdP, AtomLayoutNdKV, AtomLayoutMdQ>(params, stream);
+ run_flash_bwd<Arch, kHeadDim, kBlockM, kBlockN, T, Is_causal, Is_local, Has_softcap, Varlen /*Varlen*/, false /*Deterministic*/, GQA, Stages_dO, Stages_dS_or_QSm80, SdP_swapAB, dKV_swapAB, dQ_swapAB, NumMmaWarpGroups, AtomLayoutMSdP, AtomLayoutNdKV, AtomLayoutMdQ, V_in_regs>(params, stream);
+// });
+ });
+ });
+}
+
+
+template<int Arch, typename T, bool Has_softcap>
+void run_mha_bwd_hdim64(Flash_bwd_params &params, cudaStream_t stream) {
+ CAUSAL_LOCAL_SWITCH(params.is_causal, params.is_local, Is_causal, Is_local, [&] {
+ if constexpr (Arch >= 90) {
+ if constexpr (Is_causal && Has_softcap) {
+ // register spill with 128 x 128
+ run_mha_bwd_dispatch<Arch, T, 96, 128, 64, Is_causal, Is_local, Has_softcap, 2, 2, true, false, true, 2, 1, 2, 2, false>(params, stream);
+ } else {
+ // With ShuffleStats we no longer have register spilling when Has_softcap and using 128 x 128 block.
+ run_mha_bwd_dispatch<Arch, T, 128, 128, 64, Is_causal, Is_local, Has_softcap, 2, 2, true, false, false, 2, 1, 2, 2, false>(params, stream);
+ }
+ } else if constexpr (Arch == 86 || Arch == 89) {
+ run_mha_bwd_dispatch<Arch, T, 64, 128, 64, Is_causal, Is_local, Has_softcap, 2, 2, false, false, false, 2, 2, 4, 2, true>(params, stream);
+ // run_mha_bwd_dispatch<Arch, T, 96, 96, 64, Is_causal, Is_local, Has_softcap, 1, 2, false, true, true, 2, 2, 4, 4, false>(params, stream);
+ // run_mha_bwd_dispatch<Arch, T, 80, 128, 64, Is_causal, Is_local, Has_softcap, 1, 2, true, false, true, 2, 2, 4, 2, true>(params, stream);
+ // run_mha_bwd_dispatch<Arch, T, 96, 128, 64, Is_causal, Is_local, Has_softcap, 1, 2, true, false, true, 2, 1, 8, 4, false>(params, stream);
+ } else {
+ run_mha_bwd_dispatch<Arch, T, 128, 128, 64, Is_causal, Is_local, Has_softcap, 2, 2, false, false, false, 2, 4, 4, 4, false>(params, stream);
+ }
+ });
+}
+
+template<int Arch, typename T, bool Has_softcap>
+void run_mha_bwd_hdim96(Flash_bwd_params &params, cudaStream_t stream) {
+ CAUSAL_LOCAL_SWITCH(params.is_causal, params.is_local, Is_causal, Is_local, [&] {
+ if constexpr (Arch >= 90) {
+ run_mha_bwd_dispatch<Arch, T, 64, 128, 96, Is_causal, Is_local, Has_softcap, 2, 2, true, false, false, 2, 1, 2, 1, true>(params, stream);
+ } else if constexpr (Arch == 86 || Arch == 89) {
+ run_mha_bwd_dispatch<Arch, T, 64, 128, 96, Is_causal, Is_local, Has_softcap, 1, 2, false, false, false, 2, 2, 4, 2, true>(params, stream);
+ } else {
+ run_mha_bwd_dispatch<Arch, T, 64, 128, 96, Is_causal, Is_local, Has_softcap, 2, 2, false, false, false, 2, 2, 4, 2, false>(params, stream);
+ }
+ });
+}
+
+template<int Arch, typename T, bool Has_softcap>
+void run_mha_bwd_hdim128(Flash_bwd_params &params, cudaStream_t stream) {
+ CAUSAL_LOCAL_SWITCH(params.is_causal, params.is_local, Is_causal, Is_local, [&] {
+ if constexpr (Arch >= 90) {
+ if constexpr (Is_causal || Is_local || Has_softcap) {
+ run_mha_bwd_dispatch<Arch, T, 64, 128, 128, Is_causal, Is_local, Has_softcap, 2, 2, true, false, false, 2, 1, 2, 1, false>(params, stream);
+ } else {
+ run_mha_bwd_dispatch<Arch, T, 80, 128, 128, Is_causal, Is_local, Has_softcap, 2, 2, true, false, true, 2, 1, 2, 1, false>(params, stream);
+ }
+ } else if constexpr (Arch == 86 || Arch == 89) {
+ run_mha_bwd_dispatch<Arch, T, 64, 96, 128, Is_causal, Is_local, Has_softcap, 1, 2, false, false, false, 2, 2, 2, 2, true>(params, stream);
+ } else {
+ run_mha_bwd_dispatch<Arch, T, 64, 128, 128, Is_causal, Is_local, Has_softcap, 2, 2, false, false, false, 2, 2, 2, 2, false>(params, stream);
+ }
+ });
+}
+
+template<int Arch, typename T, bool Has_softcap>
+void run_mha_bwd_hdim192(Flash_bwd_params &params, cudaStream_t stream) {
+ CAUSAL_LOCAL_SWITCH(params.is_causal, params.is_local, Is_causal, Is_local, [&] {
+ if constexpr (Arch >= 90) {
+ run_mha_bwd_dispatch<Arch, T, 64, 96, 192, Is_causal, Is_local, Has_softcap, 1, 1, false, true, false, 3, 1, 1, 1, false>(params, stream);
+ } else if constexpr (Arch == 86 || Arch == 89) {
+ run_mha_bwd_dispatch<Arch, T, 64, 64, 192, Is_causal, Is_local, Has_softcap, 1, 1, false, false, false, 2, 2, 2, 2, true>(params, stream);
+ } else {
+ run_mha_bwd_dispatch<Arch, T, 64, 80, 192, Is_causal, Is_local, Has_softcap, 1, 2, false, true, false, 2, 4, 2, 2, false>(params, stream);
+ }
+ });
+}
+
+template<int Arch, typename T, bool Has_softcap>
+void run_mha_bwd_hdim256(Flash_bwd_params &params, cudaStream_t stream) {
+ CAUSAL_LOCAL_SWITCH(params.is_causal, params.is_local, Is_causal, Is_local, [&] {
+ if constexpr (Arch >= 90) {
+ run_mha_bwd_dispatch<Arch, T, 64, 80, 256, Is_causal, Is_local, Has_softcap, 1, 1, false, true, true, 2, 1, 1, 1, false>(params, stream);
+ } else if constexpr (Arch == 86 || Arch == 89) {
+ run_mha_bwd_dispatch<Arch, T, 32, 64, 256, Is_causal, Is_local, Has_softcap, 1, 1, false, false, false, 2, 2, 2, 1, true>(params, stream);
+ // run_mha_bwd_dispatch<Arch, T, 64, 32, 256, Is_causal, Is_local, Has_softcap, 1, 1, false, false, false, 2, 4, 1, 2, true>(params, stream);
+ } else {
+ run_mha_bwd_dispatch<Arch, T, 64, 64, 256, Is_causal, Is_local, Has_softcap, 1, 1, false, false, false, 2, 4, 2, 2, false>(params, stream);
+ }
+ });
+}
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_postprocess_kernel.h b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_postprocess_kernel.h
new file mode 100644
index 0000000..c91e261
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_postprocess_kernel.h
@@ -0,0 +1,256 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cute/tensor.hpp"
+
+#include <cutlass/cutlass.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+#include <cutlass/numeric_conversion.h>
+#include "cutlass/arch/barrier.h"
+
+#include "seqlen.h"
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <class TileShape_MK_, class Element, class ElementAccum, class ArchTag_, int kNThreads, class TiledMma, bool dQ_swapAB>
+class FlashAttnBwdPostprocessConvertdQ {
+
+public:
+
+ // Type Aliases
+ using TileShape_MK = TileShape_MK_;
+ using ArchTag = ArchTag_;
+
+ static_assert(ArchTag::kMinComputeCapability >= 75);
+ static constexpr bool IsSm90 = ArchTag::kMinComputeCapability >= 90;
+
+ static constexpr uint32_t MaxThreadsPerBlock = kNThreads;
+ static constexpr uint32_t MinBlocksPerMultiprocessor = 2;
+
+ static constexpr int kBlockM = get<0>(TileShape_MK{});
+ static constexpr int kHeadDim = get<1>(TileShape_MK{});
+ static_assert(!IsSm90 || kNThreads % cutlass::NumThreadsPerWarpGroup == 0, "kNThreads must be a multiple of NumThreadsPerWarpGroup");
+ static constexpr int NumdQWarpGgroups = kNThreads / cutlass::NumThreadsPerWarpGroup;
+ using R2SLayoutAtomdQaccum = std::conditional_t<
+ IsSm90,
+ Layout<Shape<Int<cutlass::NumThreadsPerWarpGroup>, Int<NumdQWarpGgroups>>>,
+ Layout<Shape<Int<kNThreads>>>
+ >;
+ using R2STiledCopydQaccum = decltype(make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementAccum>{}, R2SLayoutAtomdQaccum{},
+ Layout<Shape<Int<IsSm90 ? 4 : 1>>>{})); // Val layout, 1 or 4 vals per read
+ using G2SLayoutAtomdQaccum = Layout<Shape<Int<kNThreads>>>;
+ // UniversalCopy instead of AutoVectorizingCopyWithAssumedAlignment as the latter generates cp.async instructions
+ using G2STiledCopydQaccum = decltype(make_tiled_copy(Copy_Atom<UniversalCopy<uint128_t>, ElementAccum>{}, G2SLayoutAtomdQaccum{},
+ Layout<Shape<_4>>{})); // Val layout, 4 vals per read
+ // We don't do bound checking for the gmem -> smem load so we just assert here.
+ static_assert(IsSm90 || (kBlockM * kHeadDim) % (kNThreads * 4) == 0);
+ static constexpr int SmemdQaccumSize = size(TileShape_MK{});
+ using SmemLayoutdQaccumFlat = Layout<Shape<Int<SmemdQaccumSize>>>;
+ using SmemLayoutdQaccum = std::conditional_t<
+ IsSm90,
+ Layout<Shape<Int<kBlockM * kHeadDim / NumdQWarpGgroups>, Int<NumdQWarpGgroups>>>,
+ Layout<Shape<Int<kBlockM * kHeadDim>>>
+ >;
+
+ // We can't just use kHeadDim here. E.g. if MMA shape is 64 x 96 but split across 2 WGs,
+ // then setting kBlockKSmem to 32 will cause "Static shape_div failure".
+ // We want to treat it as 64 x 48, so kBlockKSmem should be 16.
+ static constexpr int MmaShapeN = get<1>(typename TiledMma::AtomShape_MNK{});
+ static constexpr int kBlockKSmem = MmaShapeN % 64 == 0 ? 64 : (MmaShapeN % 32 == 0 ? 32 : 16);
+ static constexpr int kSwizzle = kBlockKSmem == 64 ? 3 : (kBlockKSmem == 32 ? 2 : 1);
+ using SmemLayoutAtomdQ =
+ decltype(composition(Swizzle<kSwizzle, 3, 3>{},
+ Layout<Shape<Int<8>, Int<kBlockKSmem>>,
+ Stride<Int<kBlockKSmem>, _1>>{}));
+ using SmemLayoutdQ = decltype(tile_to_shape(SmemLayoutAtomdQ{}, TileShape_MK{}));
+ using SmemLayoutdQt =
+ decltype(cute::composition(SmemLayoutdQ{},
+ make_layout(make_shape(get<1>(TileShape_MK{}), get<0>(TileShape_MK{})),
+ make_stride(Int<get<0>(TileShape_MK{})>{}, _1{}))));
+
+ using SmemCopyAtomdQ = Copy_Atom<
+ std::conditional_t<
+ IsSm90,
+ std::conditional_t<!dQ_swapAB, cute::SM90_U32x4_STSM_N, cute::SM90_U16x8_STSM_T>,
+ AutoVectorizingCopyWithAssumedAlignment<128>
+ >,
+ Element>;
+
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(Element);
+ static_assert(kHeadDim % kGmemElemsPerLoad == 0, "Headdim must be a multiple of kGmemElemsPerLoad");
+ static constexpr int kGmemThreadsPerRow = cutlass::gcd(kHeadDim / kGmemElemsPerLoad, int(MaxThreadsPerBlock));
+ static_assert(MaxThreadsPerBlock % kGmemThreadsPerRow == 0, "MaxThreadsPerBlock must be a multiple of kGmemThreadsPerRow");
+ using GmemLayoutAtom = Layout<Shape <Int<MaxThreadsPerBlock / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+ using GmemTiledCopy = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per load
+
+ struct SharedStorage : cute::aligned_struct<128> {
+ cute::array_aligned<ElementAccum, cute::cosize_v<SmemLayoutdQaccum>> smem_dqacc;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutdQ>> smem_dq;
+ alignas(16) cutlass::arch::ClusterTransactionBarrier barrier_dQaccum;
+ };
+
+ static constexpr int SharedStorageSize = sizeof(SharedStorage);
+
+ using ShapedQ = cute::Shape<int32_t, int32_t, int32_t, int32_t>; // (seqlen_q, d, head, batch)
+ using StridedQ = cute::Stride<int64_t, _1, int64_t, int64_t>;
+ using ShapedQaccum = cute::Shape<int32_t, int32_t, int32_t>; // (seqlen_q * d, head, batch)
+ using StridedQaccum = cute::Stride<_1, int64_t, int64_t>;
+
+ // Device side arguments
+ struct Arguments {
+ ElementAccum const* ptr_dQaccum;
+ ShapedQaccum const shape_dQaccum;
+ StridedQaccum const stride_dQaccum;
+ Element* ptr_dQ;
+ ShapedQ const shape_dQ;
+ StridedQ const stride_dQ;
+ float const softmax_scale;
+ int const* cu_seqlens = nullptr;
+ int const* seqused = nullptr;
+ };
+
+ // Kernel entry point API
+ struct Params {
+ ElementAccum const* ptr_dQaccum;
+ ShapedQaccum const shape_dQaccum;
+ StridedQaccum const stride_dQaccum;
+ Element* ptr_dQ;
+ ShapedQ const shape_dQ;
+ StridedQ const stride_dQ;
+ float const softmax_scale;
+ int const* cu_seqlens = nullptr;
+ int const* seqused = nullptr;
+ };
+
+ // Convert to underlying arguments. In this case, a simple copy for the aliased type.
+ static
+ Params
+ to_underlying_arguments(Arguments const& args) {
+ return {
+ args.ptr_dQaccum,
+ args.shape_dQaccum,
+ args.stride_dQaccum,
+ args.ptr_dQ,
+ args.shape_dQ,
+ args.stride_dQ,
+ args.softmax_scale,
+ args.cu_seqlens,
+ args.seqused
+ };
+ }
+
+ CUTLASS_DEVICE
+ void
+ operator()(Params const& params, char* smem_buf) {
+
+ static constexpr int kBlockM = get<0>(TileShape_MK{});
+ SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(smem_buf);
+
+ Tensor sdQaccum = make_tensor(make_smem_ptr(shared_storage.smem_dqacc.data()), SmemLayoutdQaccum{});
+ Tensor sdQaccum_flat = make_tensor(make_smem_ptr(shared_storage.smem_dqacc.data()), SmemLayoutdQaccumFlat{});
+ Tensor sdQ = make_tensor(make_smem_ptr(shared_storage.smem_dq.data()), SmemLayoutdQ{});
+ Tensor sdQt = make_tensor(make_smem_ptr(shared_storage.smem_dq.data()), SmemLayoutdQt{});
+
+ int const thread_idx = threadIdx.x;
+ int const m_block = blockIdx.x;
+ int const bidh = blockIdx.y;
+ int const bidb = blockIdx.z;
+
+ flash::SeqlenInfo<true /*Varlen*/, kBlockM> seqlen_info(bidb, size<0>(params.shape_dQ), params.cu_seqlens, params.seqused);
+ bool const is_varlen = params.cu_seqlens;
+ if (is_varlen && m_block * kBlockM >= seqlen_info.seqlen) { return; }
+
+ // Step 1: load dQaccum from gmem to smem
+ Tensor mdQaccum = make_tensor(make_gmem_ptr(reinterpret_cast<ElementAccum const*>(params.ptr_dQaccum)),
+ params.shape_dQaccum, params.stride_dQaccum)(_, bidh, !is_varlen ? bidb : 0);
+ Tensor gdQaccum = local_tile(domain_offset(make_coord(seqlen_info.offset_padded * kHeadDim), mdQaccum), Shape<Int<kBlockM * kHeadDim>>{}, make_coord(m_block)); // (M * K)
+ if constexpr (IsSm90) { // Use BulkCopy
+ static constexpr uint32_t TmaTransactionBytesdQaccum = static_cast<uint32_t>(size(SmemLayoutdQaccumFlat{}) * cute::sizeof_bits_v<ElementAccum> / 8);
+ auto bulk_copy = Copy_Traits<SM90_BULK_COPY_AUTO>{};
+ // if (thread0()) { print(gdQaccum); printf("\n"); print(sdQaccum_flat); printf("\n"); }
+ if (thread_idx == 0) {
+ shared_storage.barrier_dQaccum.init(1 /*numThreads*/);
+ shared_storage.barrier_dQaccum.arrive_and_expect_tx(TmaTransactionBytesdQaccum);
+ copy(bulk_copy.with(*reinterpret_cast<uint64_t*>(&shared_storage.barrier_dQaccum)), gdQaccum, sdQaccum_flat);
+ }
+ __syncthreads();
+ shared_storage.barrier_dQaccum.wait(0);
+ } else {
+ G2STiledCopydQaccum g2s_tiled_copy_dQaccum;
+ auto g2s_thr_copy_dQaccum = g2s_tiled_copy_dQaccum.get_thread_slice(thread_idx);
+ Tensor tdQgdQaccumg2s = g2s_thr_copy_dQaccum.partition_S(gdQaccum);
+ Tensor tdQsdQaccumg2s = g2s_thr_copy_dQaccum.partition_D(sdQaccum);
+ cute::copy(g2s_tiled_copy_dQaccum, tdQgdQaccumg2s, tdQsdQaccumg2s);
+ __syncthreads();
+ }
+
+ // __syncthreads(); if (cute::thread0()) { print_tensor(sdQaccum); }
+
+ // Step 2: Load dQaccum from smem to register, then convert fp32 -> fp16/bf16
+ R2STiledCopydQaccum s2r_tiled_copy_dQaccum;
+ auto s2r_thr_copy_dQaccum = s2r_tiled_copy_dQaccum.get_thread_slice(thread_idx);
+ Tensor tdQsdQaccum = s2r_thr_copy_dQaccum.partition_S(sdQaccum);
+ TiledMma tiled_mma_dQ;
+ Tensor taccdQrdQaccum = partition_fragment_C(tiled_mma_dQ, select<!dQ_swapAB ? 0 : 1, !dQ_swapAB ? 1 : 0>(TileShape_MK{}));
+ // if (blockIdx.x == 0 && blockIdx.y == 0 && threadIdx.x == 1) { print(tiled_mma_dQ); printf("\n"); }
+ // if (blockIdx.x == 0 && blockIdx.y == 0 && threadIdx.x == 1) { print(tdQsdQaccum); }
+ // if (blockIdx.x == 0 && blockIdx.y == 0 && threadIdx.x == 1) { print(taccdQrdQaccum); }
+ CUTE_STATIC_ASSERT_V(size(taccdQrdQaccum) == size(tdQsdQaccum));
+ Tensor tdQrdQaccum = s2r_thr_copy_dQaccum.retile_D(taccdQrdQaccum);
+ cute::copy(s2r_tiled_copy_dQaccum, tdQsdQaccum, tdQrdQaccum);
+ #pragma unroll
+ for (int i = 0; i < size(taccdQrdQaccum); ++i) { taccdQrdQaccum(i) *= params.softmax_scale; }
+ // Convert tdQrdQ from fp32 to fp16
+ Tensor rdQ = make_tensor_like<Element>(taccdQrdQaccum);
+ flash::convert_type_out(taccdQrdQaccum, rdQ);
+
+ // Step 3: Copy dQ from register to smem
+ auto smem_tiled_copy_dQ = make_tiled_copy_C(SmemCopyAtomdQ{}, tiled_mma_dQ);
+ auto smem_thr_copy_dQ = smem_tiled_copy_dQ.get_thread_slice(thread_idx);
+ Tensor taccdQrdQ = smem_thr_copy_dQ.retile_S(rdQ); // ((Atom,AtomNum), MMA_N, MMA_N)
+ // if (cute::thread0()) { print(smem_tiled_copy_dQ); }
+ // if (cute::thread0()) { print(smem_thr_copy_dQ); }
+ // if (cute::thread0()) { print(sdQ); }
+ Tensor taccdQsdQ = smem_thr_copy_dQ.partition_D(cute::conditional_return<!dQ_swapAB>(sdQ, sdQt)); // ((Atom,AtomNum),PIPE_M,PIPE_N)
+ cute::copy(smem_tiled_copy_dQ, taccdQrdQ, taccdQsdQ);
+ __syncthreads();
+
+ // Step 4: Copy dQ from smem to register to prepare for coalesced write to gmem
+ Tensor mdQ = make_tensor(make_gmem_ptr(params.ptr_dQ), params.shape_dQ, params.stride_dQ)(_, _, bidh, !is_varlen ? bidb : 0);
+ Tensor gdQ = local_tile(domain_offset(make_coord(seqlen_info.offset, _0{}), mdQ), TileShape_MK{}, make_coord(m_block, _0{})); // (M, K)
+ GmemTiledCopy gmem_tiled_copy_dQ;
+ auto gmem_thr_copy_dQ = gmem_tiled_copy_dQ.get_thread_slice(thread_idx);
+ Tensor tdQsdQ = gmem_thr_copy_dQ.partition_S(sdQ); // ((Atom,AtomNum),ATOM_M,ATOM_N)
+ Tensor tdQgdQ = gmem_thr_copy_dQ.partition_D(gdQ);
+
+ Tensor tdQrdQ = make_fragment_like(tdQsdQ);
+ Tensor tdQcdQ = gmem_thr_copy_dQ.partition_D(cute::make_identity_tensor(TileShape_MK{}));
+ Tensor tdQpdQ = make_tensor<bool>(make_shape(size<2>(tdQgdQ)));
+ #pragma unroll
+ for (int k = 0; k < size(tdQpdQ); ++k) { tdQpdQ(k) = get<1>(tdQcdQ(_0{}, _0{}, k)) < get<1>(params.shape_dQ); }
+ // Need to check OOB when reading from smem if kBlockM isn't evenly tiled
+ static constexpr bool EvenM = kBlockM % CUTE_STATIC_V(size<0>(GmemLayoutAtom{})) == 0;
+ flash::copy</*Is_even_MN=*/EvenM, /*Is_even_K=*/true, /*Clear_OOB_MN=*/false>(
+ gmem_tiled_copy_dQ, tdQsdQ, tdQrdQ, tdQcdQ, tdQpdQ, kBlockM);
+
+ // Step 5: Copy dQ from register to gmem
+ // Clear_OOB_K must be false since we don't want to write zeros to gmem
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/false>(
+ gmem_tiled_copy_dQ, tdQrdQ, tdQgdQ, tdQcdQ, tdQpdQ, std::min(seqlen_info.seqlen - m_block * kBlockM, kBlockM)
+ );
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_preprocess_kernel.h b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_preprocess_kernel.h
new file mode 100644
index 0000000..85e877f
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_bwd_preprocess_kernel.h
@@ -0,0 +1,252 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cute/tensor.hpp"
+
+#include <cutlass/cutlass.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+#include <cutlass/numeric_conversion.h>
+
+#include "seqlen.h"
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <class TileShape_MK_, class Element, class ElementAccum, class ArchTag_, bool Clear_dQaccum, bool Varlen>
+class FlashAttnBwdPreprocess {
+
+public:
+
+ // Type Aliases
+ using TileShape_MK = TileShape_MK_;
+ using ArchTag = ArchTag_;
+
+ static_assert(std::is_same_v<Element, cutlass::half_t> && ArchTag::kMinComputeCapability >= 75 ||
+ std::is_same_v<Element, cutlass::bfloat16_t> && ArchTag::kMinComputeCapability >= 80 ||
+ std::is_same_v<Element, cutlass::float_e4m3_t> && ArchTag::kMinComputeCapability >= 89);
+
+ static constexpr uint32_t MaxThreadsPerBlock = 256;
+ static constexpr uint32_t MinBlocksPerMultiprocessor = 2;
+ static constexpr int SharedStorageSize = 0;
+
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(Element);
+ static_assert(get<1>(TileShape_MK{}) % kGmemElemsPerLoad == 0, "Headdim must be a multiple of kGmemElemsPerLoad");
+ static constexpr int kBlockM = get<0>(TileShape_MK{});
+ static constexpr int kHeadDim = get<1>(TileShape_MK{});
+ // We want kBlockKGmem to be a power of 2 so that when we do the summing,
+ // it's just between threads in the same warp
+ static constexpr int kBlockKGmem = kHeadDim % 128 == 0 ? 128 : (kHeadDim % 64 == 0 ? 64 : 32);
+ static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad;
+ static_assert(MaxThreadsPerBlock % kGmemThreadsPerRow == 0, "MaxThreadsPerBlock must be a multiple of kGmemThreadsPerRow");
+ using GmemLayoutAtom = Layout<Shape <Int<MaxThreadsPerBlock / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+ using GmemTiledCopy = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per load
+
+ static constexpr int kGmemElemsPerLoadAccum = sizeof(cute::uint128_t) / sizeof(ElementAccum);
+ static_assert((kBlockM * kHeadDim / kGmemElemsPerLoadAccum) % MaxThreadsPerBlock == 0, "MaxThreadsPerBlock must divide kBlockM * kHeadDim / kGmemElemsPerLoadAccum");
+ using GmemLayoutAtomAccum = Layout<Shape<Int<MaxThreadsPerBlock>>>;
+ using GmemTiledCopyAccum = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementAccum>{},
+ GmemLayoutAtomAccum{},
+ Layout<Shape<Int<kGmemElemsPerLoadAccum>>>{})); // Val layout, 4 vals per store
+
+ using ShapeO = cute::Shape<int32_t, int32_t, int32_t, int32_t>; // (seqlen_q, d, head, batch)
+ using StrideO = cute::Stride<int64_t, _1, int64_t, int64_t>;
+ using ShapedPsum = cute::Shape<int32_t, int32_t, int32_t>; // (seqlen_q, head, batch)
+ using StridedPsum = cute::Stride<_1, int64_t, int64_t>;
+ using ShapedQaccum = cute::Shape<int32_t, int32_t, int32_t>; // (seqlen_q * d, head, batch)
+ using StridedQaccum = cute::Stride<_1, int64_t, int64_t>;
+
+ // Device side arguments
+ struct Arguments {
+ Element const* ptr_O;
+ ShapeO const shape_O;
+ StrideO const stride_O;
+ Element const* ptr_dO;
+ StrideO const stride_dO;
+ float* ptr_dPsum;
+ ShapedPsum const shape_dPsum;
+ StridedPsum const stride_dPsum;
+ float const* ptr_LSE;
+ StridedPsum const stride_LSE;
+ float *ptr_LSE_log2;
+ StridedPsum const stride_LSE_log2;
+ ElementAccum* ptr_dQaccum;
+ ShapedQaccum const shape_dQaccum;
+ StridedQaccum const stride_dQaccum;
+ int num_batch; // We need this to know the size of dq_semaphore in case of varlen
+ int* dq_semaphore;
+ int const* cu_seqlens = nullptr;
+ int const* seqused = nullptr;
+ };
+
+ // Kernel entry point API
+ struct Params {
+ Element const* ptr_O;
+ ShapeO const shape_O;
+ StrideO const stride_O;
+ Element const* ptr_dO;
+ StrideO const stride_dO;
+ float* ptr_dPsum;
+ ShapedPsum const shape_dPsum;
+ StridedPsum const stride_dPsum;
+ float const* ptr_LSE;
+ StridedPsum const stride_LSE;
+ float* ptr_LSE_log2;
+ StridedPsum const stride_LSE_log2;
+ ElementAccum* ptr_dQaccum;
+ ShapedQaccum const shape_dQaccum;
+ StridedQaccum const stride_dQaccum;
+ int num_batch;
+ int* dq_semaphore;
+ int const* cu_seqlens = nullptr;
+ int const* seqused = nullptr;
+ };
+
+ // Convert to underlying arguments. In this case, a simple copy for the aliased type.
+ static
+ Params
+ to_underlying_arguments(Arguments const& args) {
+ return {
+ args.ptr_O,
+ args.shape_O,
+ args.stride_O,
+ args.ptr_dO,
+ args.stride_dO,
+ args.ptr_dPsum,
+ args.shape_dPsum,
+ args.stride_dPsum,
+ args.ptr_LSE,
+ args.stride_LSE,
+ args.ptr_LSE_log2,
+ args.stride_LSE_log2,
+ args.ptr_dQaccum,
+ args.shape_dQaccum,
+ args.stride_dQaccum,
+ args.num_batch,
+ args.dq_semaphore,
+ args.cu_seqlens,
+ args.seqused
+ };
+ }
+
+ CUTLASS_DEVICE
+ void
+ operator()(Params const& params, [[maybe_unused]] char* smem_buf) {
+
+ static constexpr int kBlockM = get<0>(TileShape_MK{});
+
+ int const thread_idx = threadIdx.x;
+ int const m_block = blockIdx.x;
+ int const bidh = blockIdx.y;
+ int const bidb = blockIdx.z;
+
+ flash::SeqlenInfo<Varlen, kBlockM> seqlen_info(bidb, size<0>(params.shape_O), params.cu_seqlens, params.seqused);
+ bool const is_varlen = Varlen && params.cu_seqlens;
+ int const seqlen_o = seqlen_info.seqlen;
+ if (is_varlen && m_block * kBlockM >= seqlen_o) { return; }
+
+ Tensor mO = make_tensor(make_gmem_ptr(params.ptr_O), params.shape_O, params.stride_O)(_, _, bidh, !is_varlen ? bidb : 0);
+ Tensor gO = local_tile(cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mO), TileShape_MK{}, make_coord(m_block, _0{})); // (M, K)
+ Tensor mdO = make_tensor(make_gmem_ptr(params.ptr_dO), params.shape_O, params.stride_dO)(_, _, bidh, !is_varlen ? bidb : 0);
+ Tensor gdO = local_tile(cute::domain_offset(make_coord(seqlen_info.offset, _0{}), mdO), TileShape_MK{}, make_coord(m_block, _0{})); // (M, K)
+
+ auto shape_LSE = select<0, 2, 3>(params.shape_O);
+ Tensor mLSE = make_tensor(make_gmem_ptr(params.ptr_LSE), shape_LSE, params.stride_LSE)(_, bidh, !is_varlen ? bidb : 0);
+ Tensor gLSE = local_tile(cute::domain_offset(make_coord(seqlen_info.offset), mLSE), Shape<Int<kBlockM>>{}, make_coord(m_block));
+ static_assert(kBlockM <= MaxThreadsPerBlock);
+ float lse = thread_idx < seqlen_o - m_block * kBlockM && thread_idx < kBlockM ? gLSE(thread_idx) : INFINITY;
+
+ GmemTiledCopy gmem_tiled_copy_O;
+ auto gmem_thr_copy_O = gmem_tiled_copy_O.get_thread_slice(thread_idx);
+
+ Tensor tOgO = gmem_thr_copy_O.partition_S(gO);
+ Tensor tOgdO = gmem_thr_copy_O.partition_S(gdO);
+ // Construct identity layout for gO
+ Tensor cO = cute::make_identity_tensor(TileShape_MK{}); // (BLK_M,BLK_K) -> (blk_m,blk_k)
+ // Repeat the partitioning with identity layouts
+ Tensor tOcO = gmem_thr_copy_O.partition_D(cO);
+ Tensor tOpO = make_tensor<bool>(make_shape(size<2>(tOgO)));
+ #pragma unroll
+ for (int k = 0; k < size(tOpO); ++k) { tOpO(k) = get<1>(tOcO(_0{}, _0{}, k)) < get<1>(params.shape_O); }
+
+ // (8, kBlockM / 32, kHeadDim / 64) or (8, kBlockM / 16, kHeadDim / 128)
+ Tensor tOrO = make_fragment_like(tOgO);
+ Tensor tOrdO = make_fragment_like(tOgdO);
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/true, /*Clearn_OOB_K=*/true>(
+ gmem_tiled_copy_O, tOgO, tOrO, tOcO, tOpO, seqlen_o - m_block * kBlockM
+ );
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/true, /*Clearn_OOB_K=*/true>(
+ gmem_tiled_copy_O, tOgdO, tOrdO, tOcO, tOpO, seqlen_o - m_block * kBlockM
+ );
+ // if (threadIdx.x == 222) { printf("bidx = %d, bidy = %d, bidz = %d, seqlen_o = %d, m_block = %d, seqlen_o - m_block * kBlockM = %d, tOgO addr = %p\n", blockIdx.x, blockIdx.y, blockIdx.z, seqlen_o, m_block, seqlen_o - m_block * kBlockM, &tOgO(0));}
+
+ // Reshape from e.g. (8, kBlockM / 32, kHeadDim / 64) to (kBlockM / 32, (8, kHeadDim / 64))
+ Layout l = make_layout(get<1>(tOrO.layout()), make_layout(get<0>(tOrO.layout()), get<2>(tOrO.layout())));
+ Tensor tOrO_l = make_tensor(tOrO.data(), l);
+ Tensor o_fp32 = make_tensor_like<float>(tOrO_l);
+ flash::convert_type_out(tOrO_l, o_fp32);
+ Tensor tOrdO_l = make_tensor(tOrdO.data(), l);
+ Tensor do_fp32 = make_tensor_like<float>(tOrdO_l);
+ flash::convert_type_out(tOrdO_l, do_fp32);
+ // Sum across the last dimension
+ Tensor dP_sum = make_tensor<float>(make_shape(size<0>(o_fp32)));
+ #pragma unroll
+ for (int mi = 0; mi < size<0>(o_fp32); ++mi) {
+ float dP_sum_cur = do_fp32(mi, 0) * o_fp32(mi, 0);
+ #pragma unroll
+ for (int ni = 1; ni < size<1>(o_fp32); ni++) {
+ dP_sum_cur += do_fp32(mi, ni) * o_fp32(mi, ni);
+ }
+ flash::SumOp<float> sum_op;
+ dP_sum(mi) = flash::Allreduce<kGmemThreadsPerRow>::run(dP_sum_cur, sum_op);
+ }
+
+ Tensor mdPsum = make_tensor(make_gmem_ptr(params.ptr_dPsum), params.shape_dPsum, params.stride_dPsum)(_, bidh, !is_varlen ? bidb : 0);
+ Tensor gdPsum = local_tile(cute::domain_offset(make_coord(seqlen_info.offset_padded), mdPsum), Shape<Int<kBlockM>>{}, make_coord(m_block));
+ if (get<1>(tOcO(_0{}, _0{}, _0{})) == 0) {
+ #pragma unroll
+ for (int mi = 0; mi < size(dP_sum); ++mi) {
+ int const row = get<0>(tOcO(_0{}, mi, _0{}));
+ gdPsum(row) = row < seqlen_o - m_block * kBlockM ? dP_sum(mi) : 0;
+ }
+ }
+
+ int const seqlen_rounded = cute::round_up(seqlen_o, kBlockM);
+ Tensor mLSElog2 = make_tensor(make_gmem_ptr(params.ptr_LSE_log2), params.shape_dPsum, params.stride_LSE_log2)(_, bidh, !is_varlen ? bidb : 0);
+ Tensor gLSElog2 = local_tile(cute::domain_offset(make_coord(seqlen_info.offset_padded), mLSElog2), Shape<Int<kBlockM>>{}, make_coord(m_block));
+ if (thread_idx < seqlen_rounded - m_block * kBlockM && thread_idx < kBlockM) {
+ gLSElog2(thread_idx) = lse == -INFINITY ? 0.f : lse * float(M_LOG2E);
+ }
+
+ if constexpr (Clear_dQaccum) {
+ Tensor mdQaccum = make_tensor(make_gmem_ptr(params.ptr_dQaccum), params.shape_dQaccum, params.stride_dQaccum)(_, bidh, !is_varlen ? bidb : 0);
+ Tensor gdQaccum = local_tile(cute::domain_offset(make_coord(seqlen_info.offset_padded * kHeadDim), mdQaccum), Shape<Int<kBlockM * kHeadDim>>{}, make_coord(m_block));
+ GmemTiledCopyAccum gmem_tiled_copy_dQaccum;
+ auto gmem_thr_copy_dQaccum = gmem_tiled_copy_dQaccum.get_thread_slice(thread_idx);
+ Tensor tdQgdQaccum = gmem_thr_copy_dQaccum.partition_D(gdQaccum);
+ Tensor zero = make_fragment_like(tdQgdQaccum);
+ clear(zero);
+ cute::copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementAccum>{}, zero, tdQgdQaccum);
+ }
+
+ if (params.dq_semaphore != nullptr && thread_idx == 0) {
+ int const num_batch = params.num_batch;
+ int const num_head = get<2>(params.shape_O);
+ params.dq_semaphore[bidh + bidb * num_head + m_block * num_head * num_batch] = 0;
+ }
+
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_combine_kernel.h b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_combine_kernel.h
new file mode 100644
index 0000000..aaec31e
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_combine_kernel.h
@@ -0,0 +1,453 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cute/tensor.hpp"
+
+#include <cutlass/cutlass.h>
+#include <cutlass/arch/memory.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+#include <cutlass/numeric_conversion.h>
+
+#include "seqlen.h"
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <class TileShape_MK_, int kLogMaxSplits_, int kNThreads, int AlignmentLSE_,
+ bool Is_even_K, bool Varlen, class Element, class ElementPartial, class ArchTag_>
+class FlashAttnFwdCombine {
+
+public:
+
+ // Type Aliases
+ using TileShape_MK = TileShape_MK_;
+ using ArchTag = ArchTag_;
+ static constexpr int kMaxSplits = 1 << kLogMaxSplits_;
+ static constexpr int AlignmentLSE = std::min(AlignmentLSE_, int(128 / 8 / sizeof(float)));
+ static_assert(AlignmentLSE >= 1);
+ static constexpr int kStages = 4;
+
+ static_assert(ArchTag::kMinComputeCapability >= 75);
+ static constexpr bool Has_cp_async = ArchTag::kMinComputeCapability >= 80;
+
+ static constexpr uint32_t MaxThreadsPerBlock = kNThreads;
+ static constexpr uint32_t MinBlocksPerMultiprocessor = 2;
+
+ static constexpr int kBlockM = get<0>(TileShape_MK{});
+ static constexpr int kHeadDim = get<1>(TileShape_MK{});
+
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(ElementPartial);
+ static_assert(kHeadDim % kGmemElemsPerLoad == 0, "Headdim must be a multiple of kGmemElemsPerLoad");
+ static constexpr int kBlockKGmem = kHeadDim % 128 == 0 ? 128 : (kHeadDim % 64 == 0 ? 64 : 32);
+ static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad;
+ static_assert(MaxThreadsPerBlock % kGmemThreadsPerRow == 0, "MaxThreadsPerBlock must be a multiple of kGmemThreadsPerRow");
+ using GmemCopyAtom = std::conditional_t<
+ Has_cp_async,
+ cute::Copy_Atom<SM80_CP_ASYNC_CACHEGLOBAL<uint128_t>, ElementPartial>,
+ cute::Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementPartial>
+ >;
+ using GmemLayoutAtom = Layout<Shape <Int<MaxThreadsPerBlock / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+ static_assert(kBlockM % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0);
+ using GmemTiledCopyAccum = decltype(
+ make_tiled_copy(GmemCopyAtom{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 4 vals per load
+ using GmemTiledCopy = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 4 vals per load
+
+ using AlignmentTypeLSE = cute::uint_byte_t<static_cast<int>(sizeof(float)) * AlignmentLSE>;
+ static constexpr int kGmemElemsPerLoadLSE = sizeof(AlignmentTypeLSE) / sizeof(float);
+ static_assert(kBlockM % kGmemElemsPerLoadLSE == 0, "kBlockM must be a multiple of kGmemElemsPerLoadLSE");
+ static_assert(kBlockM % 8 == 0, "kBlockM must be a multiple of 8");
+ static constexpr int kBlockMSmem = kBlockM % 128 == 0 ? 128 : (kBlockM % 64 == 0 ? 64 : (kBlockM % 32 == 0 ? 32 : (kBlockM % 16 == 0 ? 16 : 8)));
+ static constexpr int kGmemThreadsPerRowLSE = kBlockMSmem / kGmemElemsPerLoadLSE;
+ static_assert(MaxThreadsPerBlock % kGmemThreadsPerRowLSE == 0, "MaxThreadsPerBlock must be a multiple of kGmemThreadsPerRowLSE");
+ using GmemLayoutAtomLSE = Layout<Shape <Int<MaxThreadsPerBlock / kGmemThreadsPerRowLSE>, Int<kGmemThreadsPerRowLSE>>,
+ Stride<Int<kGmemThreadsPerRowLSE>, _1>>;
+ static_assert(kMaxSplits % CUTE_STATIC_V(shape<0>(GmemLayoutAtomLSE{})) == 0);
+ using GmemCopyAtomLSE = std::conditional_t<
+ Has_cp_async,
+ cute::Copy_Atom<SM80_CP_ASYNC_CACHEALWAYS<AlignmentTypeLSE>, float>,
+ cute::Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<AlignmentLSE * sizeof(float) * 8>, float>
+ >;
+ using GmemTiledCopyLSE = decltype(
+ make_tiled_copy(GmemCopyAtomLSE{},
+ GmemLayoutAtomLSE{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoadLSE>>>{})); // Val layout, 4 vals per load
+
+ // Otherwise we get IMA when some threads access sLSE, as we're not doing any masking
+ static_assert((kBlockM * kMaxSplits * AlignmentLSE) % kNThreads == 0, "kNThreads must divide kBlockM * kMaxSplits * AlignmentLSE");
+ // This works for kBlockMSmem = 8, 16, 32, 64, 128, no bank conflicts
+ using SmemLSESwizzle = std::conditional_t<
+ kBlockMSmem == 8,
+ Swizzle<5, 0, 5>,
+ std::conditional_t<kBlockMSmem == 16, Swizzle<4, 0, 4>, Swizzle<3, 2, 3>>
+ >;
+ using SmemLayoutAtomLSE =
+ decltype(composition(SmemLSESwizzle{},
+ Layout<Shape<Int<8>, Int<kBlockMSmem>>,
+ Stride<Int<kBlockMSmem>, _1>>{}));
+ using SmemLayoutLSE = decltype(tile_to_shape(SmemLayoutAtomLSE{}, Shape<Int<kMaxSplits>, Int<kBlockM>>{}));
+
+ using SmemLayoutO = Layout<Shape<Int<kBlockM>, Int<kHeadDim>, Int<kStages>>,
+ Stride<Int<kHeadDim>, _1, Int<kBlockM * kHeadDim>>>;
+
+ // We want each column (kMaxSplits) to be processed by threads in the same warp.
+ // To reduce the number of shuffles, we want as few threads on the same column as possible.
+ // E.g., if kBlockM is divisible by 64, and there are 256 threads, we want 4 threads (0, 1, 2, 4) per column
+ // have have 64 such quads.
+ static_assert(MaxThreadsPerBlock % kBlockMSmem == 0, "MaxThreadsPerBlock must be a multiple of kBlockMSmem");
+ static constexpr int kSmemThreadsPerColLSEt = MaxThreadsPerBlock / kBlockMSmem;
+ static_assert(cutlass::NumThreadsPerWarp % kSmemThreadsPerColLSEt == 0, "kSmemThreadsPerColLSEt must divide NumThreadsPerWarp");
+ using S2RLayoutAtomLSE = Layout<Shape<Int<kSmemThreadsPerColLSEt>, Int<MaxThreadsPerBlock / kSmemThreadsPerColLSEt>>>;
+ using S2RTiledCopyLSE = decltype(make_tiled_copy(cute::Copy_Atom<cute::DefaultCopy, float>{}, S2RLayoutAtomLSE{}, Layout<_1>{}));
+
+ using ShapeOPartial = cute::Shape<int32_t, int32_t, int32_t, int32_t, int32_t>; // (seqlen, d, num_splits, head, batch)
+ using StrideOPartial = cute::Stride<int64_t, _1, int64_t, int64_t, int64_t>;
+ using ShapeLSEPartial = cute::Shape<int32_t, int32_t, int32_t, int32_t>; // (seqlen, num_splits, head, batch)
+ using StrideLSEPartial = cute::Stride<_1, int64_t, int64_t, int64_t>; // (seqlen, num_splits, head, batch)
+ using ShapeO = cute::Shape<int32_t, int32_t, int32_t, int32_t>; // (seqlen, d, head, batch)
+ using StrideO = cute::Stride<int64_t, _1, int64_t, int64_t>;
+ using ShapeLSE = cute::Shape<int32_t, int32_t, int32_t>; // (seqlen, head, batch)
+ using StrideLSE = cute::Stride<_1, int64_t, int64_t>; // (seqlen, head, batch)
+
+ struct SharedStorage : cute::aligned_struct<128> {
+ cute::array_aligned<float, cute::cosize_v<SmemLayoutLSE>> smem_lse_partial;
+ cute::array_aligned<int, kBlockM> smem_max_valid_split;
+ cute::array_aligned<ElementPartial, cute::cosize_v<SmemLayoutO>> smem_o_partial;
+ };
+
+ static constexpr int SharedStorageSize = sizeof(SharedStorage);
+
+
+ // Device side arguments
+ struct Arguments {
+ ElementPartial const* ptr_O_partial;
+ ShapeOPartial const shape_O_partial;
+ StrideOPartial const stride_O_partial;
+ float const* ptr_LSE_partial;
+ ShapeLSEPartial const shape_LSE_partial;
+ StrideLSEPartial const stride_LSE_partial;
+ Element* ptr_O;
+ StrideO const stride_O;
+ float* ptr_LSE;
+ StrideLSE const stride_LSE;
+ int const* cu_seqlens = nullptr;
+ int const* seqused = nullptr;
+ };
+
+ // Kernel entry point API
+ struct Params {
+ ElementPartial const* ptr_O_partial;
+ ShapeOPartial const shape_O_partial;
+ StrideOPartial const stride_O_partial;
+ float const* ptr_LSE_partial;
+ ShapeLSEPartial const shape_LSE_partial;
+ StrideLSEPartial const stride_LSE_partial;
+ Element* ptr_O;
+ StrideO const stride_O;
+ float* ptr_LSE;
+ StrideLSE const stride_LSE;
+ cutlass::FastDivmod seqlen_divmod, head_divmod;
+ int const* cu_seqlens = nullptr;
+ int const* seqused = nullptr;
+ };
+
+ // Convert to underlying arguments. In this case, a simple copy for the aliased type.
+ static
+ Params
+ to_underlying_arguments(Arguments const& args) {
+ assert(get<1>(args.shape_LSE_partial) <= kMaxSplits);
+ return {
+ args.ptr_O_partial,
+ args.shape_O_partial,
+ args.stride_O_partial,
+ args.ptr_LSE_partial,
+ args.shape_LSE_partial,
+ args.stride_LSE_partial,
+ args.ptr_O,
+ args.stride_O,
+ args.ptr_LSE,
+ args.stride_LSE,
+ cutlass::FastDivmod(get<0>(args.shape_LSE_partial)), cutlass::FastDivmod(get<2>(args.shape_LSE_partial)),
+ args.cu_seqlens,
+ args.seqused
+ };
+ }
+
+ CUTLASS_DEVICE
+ void
+ operator()(Params const& params, char* smem_buf) {
+
+ SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(smem_buf);
+ Tensor sLSE = make_tensor(make_smem_ptr(shared_storage.smem_lse_partial.data()), SmemLayoutLSE{});
+ Tensor sMaxValidSplit = make_tensor(make_smem_ptr(shared_storage.smem_max_valid_split.data()), Shape<Int<kBlockM>>{});
+ Tensor sO = make_tensor(make_smem_ptr(shared_storage.smem_o_partial.data()), SmemLayoutO{});
+
+ int const thread_idx = threadIdx.x;
+ int const m_block = blockIdx.x;
+ int const batch = !Varlen ? 0 : blockIdx.y;
+ int const num_splits = get<1>(params.shape_LSE_partial);
+ flash::SeqlenInfo<Varlen, kBlockM> seqlen_info{batch, size<0>(params.shape_LSE_partial), params.cu_seqlens, params.seqused};
+ int const offset = seqlen_info.offset;
+ int const seqlen = seqlen_info.seqlen;
+ int max_idx = seqlen * get<2>(params.shape_LSE_partial) * get<3>(params.shape_LSE_partial);
+
+ cutlass::FastDivmod seqlen_divmod_dynamic(seqlen);
+
+ // Step 1: load LSE_partial from gmem -> smem
+ Tensor mLSEpartial = make_tensor(make_gmem_ptr(params.ptr_LSE_partial + offset * get<0>(params.stride_LSE_partial)), select<1, 0, 2, 3>(params.shape_LSE_partial), select<1, 0, 2, 3>(params.stride_LSE_partial)); // (num_splits, seqlen, head, batch)
+ Tensor mLSEpartial_copy = cute::tiled_divide(mLSEpartial, Shape<_1, Int<kGmemElemsPerLoadLSE>>{});
+ GmemTiledCopyLSE gmem_tiled_copy_LSE;
+ auto gmem_thr_copy_LSE = gmem_tiled_copy_LSE.get_thread_slice(thread_idx);
+ Tensor tLSEsLSE = gmem_thr_copy_LSE.partition_D(sLSE);
+
+ // Construct identity layout for sLSE
+ Tensor cLSE = make_identity_tensor(make_shape(size<0>(sLSE), size<1>(sLSE))); // (NUM_SPLITS, BLK_M) -> (num_splits, blk_m)
+ // Repeat the partitioning with identity layouts
+ Tensor tLSEcLSE = gmem_thr_copy_LSE.partition_S(cLSE);
+
+ #pragma unroll
+ for (int m = 0; m < size<2>(tLSEcLSE); ++m) {
+ int mi = int(get<1>(tLSEcLSE(_0{}, _0{}, m)));
+ int idx = m_block * kBlockM + mi;
+ if (idx < max_idx) {
+ int m_idx, bidh, bidb;
+ if constexpr (!Varlen) {
+ bidb = params.head_divmod.divmod(bidh, params.seqlen_divmod.divmod(m_idx, idx));
+ } else {
+ bidh = seqlen_divmod_dynamic.divmod(m_idx, idx);
+ bidb = 0;
+ }
+ Tensor mLSEpartial_cur_copy = mLSEpartial_copy(_, _, m_idx, bidh, bidb);
+ #pragma unroll
+ for (int s = 0; s < size<1>(tLSEcLSE); ++s) {
+ int si = get<0>(tLSEcLSE(_0{}, s, _0{}));
+ // if (blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0 && thread_idx < 32) { printf("thread_idx = %d, m = %d, s = %d, addr = %p, bank = %d\n", thread_idx, m, s, reinterpret_cast<float *>(&(tLSEsLSE(_0{}, s, m))), reinterpret_cast<int>(&(tLSEsLSE(_0{}, s, m))) / 4 % 32);}
+ if (si < num_splits) {
+ cute::copy(gmem_tiled_copy_LSE, mLSEpartial_cur_copy(_, si), tLSEsLSE(_, s, m));
+ } else {
+ cute::fill(tLSEsLSE(_, s, m), -INFINITY);
+ }
+ }
+ } else {
+ // We don't need to zero out the rest of the LSEs, as we will not write the output to gmem
+ // cute::fill(tLSEsLSE(_, _, m), -INFINITY);
+ }
+ }
+ if constexpr (Has_cp_async) { cute::cp_async_fence(); }
+
+ // Step 2: Load O_partial from gmem -> smem for split = 0, 1, ..., kStages - 2.
+ // We want these async loads to be in flight as we compute the LSE.
+ GmemTiledCopyAccum gmem_tiled_copy_O_partial;
+ auto gmem_thr_copy_O_partial = gmem_tiled_copy_O_partial.get_thread_slice(thread_idx);
+ // Construct identity layout for gO
+ Tensor cO = cute::make_identity_tensor(TileShape_MK{}); // (BLK_M,BLK_K) -> (blk_m,blk_k)
+ // Repeat the partitioning with identity layouts
+ Tensor tOcO = gmem_thr_copy_O_partial.partition_D(cO);
+ Tensor mOpartial = make_tensor(make_gmem_ptr(params.ptr_O_partial + offset * get<0>(params.stride_O_partial)), params.shape_O_partial, params.stride_O_partial); // (seqlen, d, num_splits, head, batch)
+
+ // Precompute these values to avoid recomputing them in the loop
+ Tensor tOmidx = make_tensor<int>(make_shape(size<1>(tOcO)));
+ Tensor tObidh = make_tensor<int>(make_shape(size<1>(tOcO)));
+ Tensor tObidb = make_tensor<int>(make_shape(size<1>(tOcO)));
+ Tensor tOrOptr = make_tensor<ElementPartial const*>(make_shape(size<1>(tOcO)));
+ #pragma unroll
+ for (int m = 0; m < size<1>(tOcO); ++m) {
+ int mi = get<0>(tOcO(_0{}, m, _0{}));
+ int idx = m_block * kBlockM + mi;
+ if constexpr (!Varlen) {
+ tObidb[m] = params.head_divmod.divmod(tObidh(m), params.seqlen_divmod.divmod(tOmidx(m), idx));
+ } else {
+ tObidh[m] = seqlen_divmod_dynamic.divmod(tOmidx(m), idx);
+ tObidb[m] = 0;
+ }
+ tOrOptr[m] = &mOpartial(tOmidx(m), _0{}, _0{}, tObidh(m), tObidb(m));
+ if (idx >= max_idx) {
+ tObidb[m] = -1;
+ }
+ }
+
+ Tensor tOpO = make_tensor<bool>(make_shape(size<2>(tOcO)));
+ if constexpr (!(Is_even_K)) {
+ #pragma unroll
+ for (int k = 0; k < size(tOpO); ++k) { tOpO(k) = get<1>(tOcO(_0{}, _0{}, k)) < get<1>(params.shape_O_partial); }
+ }
+
+ Tensor tOsOpartial = gmem_thr_copy_O_partial.partition_D(sO);
+
+ auto load_O_partial = [&] (int split, int stage) {
+ Tensor tOsOpartial_cur = tOsOpartial(_, _, _, stage);
+ #pragma unroll
+ for (int m = 0; m < size<1>(tOcO); ++m) {
+ if (tObidb(m) >= 0) {
+ Tensor mOpartial_cur = make_tensor(make_gmem_ptr(tOrOptr[m]), mOpartial(_0{}, _, _, _0{}, _0{}).layout());
+ Tensor mOpartial_cur_copy = cute::tiled_divide(mOpartial_cur, Shape<Int<kGmemElemsPerLoad>>{});
+ #pragma unroll
+ for (int k = 0; k < size<2>(tOcO); ++k) {
+ int k_idx = get<1>(tOcO(_0{}, _0{}, k)) / kGmemElemsPerLoad;
+ if (Is_even_K || tOpO(k)) {
+ cute::copy(gmem_tiled_copy_O_partial, mOpartial_cur_copy(_, k_idx, split), tOsOpartial_cur(_, m, k));
+ }
+ }
+ }
+ }
+ };
+
+ for (int s = 0; s < kStages - 1; ++s) {
+ if (s < num_splits) { load_O_partial(s, s); }
+ if constexpr (Has_cp_async) { cute::cp_async_fence(); }
+ }
+
+ // Step 3: load and transpose LSE_partial from smem -> rmem
+ if constexpr (Has_cp_async) { cutlass::arch::cp_async_wait<kStages - 1>(); }
+ __syncthreads();
+
+ S2RTiledCopyLSE s2r_tiled_copy_LSE;
+ auto s2r_thr_copy_LSE = s2r_tiled_copy_LSE.get_thread_slice(thread_idx);
+ Tensor ts2rsLSE = s2r_thr_copy_LSE.partition_S(sLSE);
+ Tensor ts2rrLSE = make_fragment_like(ts2rsLSE);
+ cute::copy(s2r_tiled_copy_LSE, ts2rsLSE, ts2rrLSE);
+
+ // Step 4: compute the final LSE along the split dimension
+ Tensor lse_sum = make_tensor<float>(make_shape(size<2>(ts2rrLSE)));
+ Tensor ts2rcLSE = s2r_thr_copy_LSE.partition_D(cLSE);
+ // We compute the max valid split for each row to short-circuit the computation later
+ Tensor max_valid_split = make_tensor<int>(make_shape(size<2>(ts2rrLSE)));
+ static_assert(CUTE_STATIC_V(size<0>(ts2rrLSE)) == 1);
+ #pragma unroll
+ for (int m = 0; m < size<2>(ts2rrLSE); ++m) {
+ float lse_max = ts2rrLSE(_0{}, _0{}, m);
+ #pragma unroll
+ for (int s = 1; s < size<1>(ts2rrLSE); ++s) { lse_max = max(lse_max, ts2rrLSE(_0{}, s, m)); }
+ MaxOp<float> max_op;
+ lse_max = Allreduce<kSmemThreadsPerColLSEt>::run(lse_max, max_op);
+ int max_valid_idx = -1;
+ #pragma unroll
+ for (int s = 0; s < size<1>(ts2rrLSE); ++s) {
+ if (ts2rrLSE(_0{}, s, m) != -INFINITY) { max_valid_idx = get<0>(ts2rcLSE(_0{}, s, _0{})); }
+ }
+ MaxOp<int> max_int_op;
+ max_valid_split[m] = Allreduce<kSmemThreadsPerColLSEt>::run(max_valid_idx, max_int_op);
+ float lse_max_cur = lse_max == -INFINITY ? 0.0f : lse_max; // In case all local LSEs are -inf
+ float lse_sum_cur = 0.f;
+ #pragma unroll
+ for (int s = 0; s < size<1>(ts2rrLSE); ++s) {
+ float scale = expf(ts2rrLSE(_0{}, s, m) - lse_max_cur);
+ lse_sum_cur += scale;
+ // if (blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0 && thread_idx < 32) { printf("thread_idx = %d, m = %d, s = %d, addr = %p, bank = %d\n", thread_idx, m, s, reinterpret_cast<float *>(&(ts2rsLSE(_0{}, s, m))), reinterpret_cast<int>(&(ts2rsLSE(_0{}, s, m))) / 4 % 32);}
+ // ts2rsLSE(_0{}, m, s) = scale;
+ ts2rrLSE(_0{}, s, m) = scale;
+ }
+ SumOp<float> sum_op;
+ lse_sum_cur = Allreduce<kSmemThreadsPerColLSEt>::run(lse_sum_cur, sum_op);
+ lse_sum(m) = logf(lse_sum_cur) + lse_max;
+ float inv_sum = (lse_sum_cur == 0.f || lse_sum_cur != lse_sum_cur) ? 0.f : 1.f / lse_sum_cur;
+ #pragma unroll
+ for (int s = 0; s < size<1>(ts2rrLSE); ++s) { ts2rrLSE(_0{}, s, m) *= inv_sum; }
+ }
+ // Store the scales exp(lse - lse_logsum) back to smem
+ cute::copy(s2r_tiled_copy_LSE, ts2rrLSE, ts2rsLSE);
+
+ // Step 5: store final LSE back to gmem
+ auto shape_LSE = select<0, 2, 3>(params.shape_LSE_partial);
+ Tensor mLSE = make_tensor(make_gmem_ptr(params.ptr_LSE + offset * get<0>(params.stride_LSE)), shape_LSE, params.stride_LSE);
+ #pragma unroll
+ for (int m = 0; m < size<2>(ts2rrLSE); ++m) {
+ if (get<0>(ts2rcLSE(_0{}, _0{}, m)) == 0) { // Only the thread responsible for s=0 writes to gmem
+ int mi = int(get<1>(ts2rcLSE(_0{}, _0{}, m)));
+ int idx = m_block * kBlockM + mi;
+ if (idx < max_idx) {
+ int m_idx, bidh, bidb;
+ if constexpr (!Varlen) {
+ bidb = params.head_divmod.divmod(bidh, params.seqlen_divmod.divmod(m_idx, idx));
+ } else {
+ bidh = seqlen_divmod_dynamic.divmod(m_idx, idx);
+ bidb = 0;
+ }
+ // printf("thread_idx = %d, m = %d, mi = %d, idx = %d, m_idx = %d, bidh = %d, bidb = %d, lse_sum = %f\n", thread_idx, m, mi, idx, m_idx, bidh, bidb, lse_sum(m));
+ mLSE(m_idx, bidh, bidb) = lse_sum(m);
+ }
+ if (mi < kBlockM) { sMaxValidSplit[mi] = max_valid_split[m]; }
+ }
+ }
+
+ // Step 6: read O_partial from gmem -> smem -> rmem and accumulate the final O
+ __syncthreads();
+ int thr_max_valid_split = sMaxValidSplit[get<0>(tOcO(_0{}, _0{}, _0{}))];
+ #pragma unroll
+ for (int m = 1; m < size<1>(tOcO); ++m) { thr_max_valid_split = max(thr_max_valid_split, sMaxValidSplit[get<0>(tOcO(_0{}, m, _0{}))]); }
+ Layout tOrOpartial_layout = gmem_thr_copy_O_partial.partition_S(make_tensor<ElementPartial>(TileShape_MK{})).layout();
+ Tensor tOrOpartial = make_fragment_like<ElementPartial>(tOrOpartial_layout);
+ Tensor tOrO = make_fragment_like<float>(tOrOpartial);
+ clear(tOrO);
+ int stage_load = kStages - 1, stage_compute = 0;
+ #pragma unroll 4 // Already tuned for speed
+ for (int s = 0; s <= thr_max_valid_split; ++s) {
+ Tensor scale = make_tensor<float>(make_shape(size<1>(tOrOpartial)));
+ #pragma unroll
+ for (int m = 0; m < size<1>(tOrOpartial); ++m) { scale(m) = sLSE(s, get<0>(tOcO(_0{}, m, _0{}))); }
+
+ if (s + kStages - 1 <= thr_max_valid_split) { load_O_partial(s + kStages - 1, stage_load); }
+ if constexpr (Has_cp_async) { cute::cp_async_fence(); }
+ stage_load = stage_load < kStages - 1 ? stage_load + 1 : 0;
+ if constexpr (Has_cp_async) { cutlass::arch::cp_async_wait<kStages - 1>(); }
+ // We don't need __syncthreads() because each thread is just reading its own data from smem
+ cute::copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementPartial>{},
+ tOsOpartial(_, _, _, stage_compute), tOrOpartial);
+ stage_compute = stage_compute < kStages - 1 ? stage_compute + 1 : 0;
+
+ #pragma unroll
+ for (int m = 0; m < size<1>(tOrOpartial); ++m) {
+ if (tObidb(m) >= 0 && scale(m) > 0.f) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(tOrOpartial); ++k) {
+ if (Is_even_K || tOpO(k)) {
+ Tensor rOpartial = make_tensor_like<float>(tOrOpartial(_, m, k));
+ flash::convert_type_out(tOrOpartial(_, m, k), rOpartial);
+ #pragma unroll
+ for (int i = 0; i < size<0>(tOrOpartial); ++i) {
+ tOrO(i, m, k) += scale(m) * rOpartial[i];
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Step 7: Write the final O to gmem
+ Tensor rO = make_tensor_like<Element>(tOrO);
+ flash::convert_type_out(tOrO, rO);
+ auto shape_O = select<0, 1, 3, 4>(params.shape_O_partial);
+ Tensor mO = make_tensor(make_gmem_ptr(params.ptr_O + offset * get<0>(params.stride_O)), shape_O, params.stride_O);
+ Tensor mO_copy = cute::tiled_divide(mO, Shape<_1, Int<kGmemElemsPerLoad>>{});
+ GmemTiledCopy gmem_tiled_copy_O;
+ auto gmem_thr_copy_O = gmem_tiled_copy_O.get_thread_slice(thread_idx);
+
+ #pragma unroll
+ for (int m = 0; m < size<1>(tOcO); ++m) {
+ if (tObidb(m) >= 0) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(tOcO); ++k) {
+ int k_idx = get<1>(tOcO(_0{}, _0{}, k)) / kGmemElemsPerLoad;
+ if (Is_even_K || tOpO(k)) {
+ cute::copy(gmem_tiled_copy_O, rO(_, m, k), mO_copy(_, tOmidx(m), k_idx, tObidh(m), tObidb(m)));
+ }
+ }
+ }
+ }
+
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_combine_launch_template.h b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_combine_launch_template.h
new file mode 100644
index 0000000..33e66c2
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_combine_launch_template.h
@@ -0,0 +1,74 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cute/tensor.hpp"
+
+#include "cutlass/cutlass.h"
+#include "cutlass/arch/arch.h" // For cutlass::arch::Sm80
+#include "cutlass/device_kernel.h" // For device_kernel
+
+#include "static_switch.h"
+#include "flash.h"
+#include "flash_fwd_combine_kernel.h"
+
+using namespace cute;
+
+template <int kHeadDim, int kBlockM, int kLogMaxSplits, bool IsEvenK, bool Varlen, typename Element, typename ElementPartial>
+void run_flash_fwd_combine(Flash_fwd_params &params, cudaStream_t stream) {
+ using TileShape_MK = cute::Shape<Int<kBlockM>, Int<kHeadDim>>;
+ using CombineKernel = flash::FlashAttnFwdCombine<TileShape_MK, kLogMaxSplits, 256 /*kNThreads*/, 1 /*AlignmentLSE*/,
+ IsEvenK, Varlen, Element, ElementPartial, cutlass::arch::Sm80>;
+
+ typename CombineKernel::Arguments args {
+ static_cast<ElementPartial const*>(params.oaccum_ptr),
+ {!Varlen ? params.seqlen_q : params.total_q, params.d, params.num_splits, params.h, !Varlen ? params.b : 1}, // shape_O_partial
+ {params.oaccum_row_stride, _1{}, params.oaccum_split_stride, params.oaccum_head_stride, !Varlen ? params.oaccum_batch_stride : 0}, // stride_O_partial
+ static_cast<float*>(params.softmax_lseaccum_ptr),
+ {!Varlen ? params.seqlen_q : params.total_q, params.num_splits, params.h, !Varlen ? params.b : 1}, // shape_LSE_partial
+ {_1{}, params.lseaccum_split_stride, params.lseaccum_head_stride, !Varlen ? params.lseaccum_batch_stride : 0}, // stride_LSE_partial
+ static_cast<Element*>(params.o_ptr),
+ {params.o_row_stride, _1{}, params.o_head_stride, !Varlen ? params.o_batch_stride : 0}, // stride_O
+ static_cast<float*>(params.softmax_lse_ptr),
+ {_1{}, !Varlen ? params.seqlen_q : params.total_q, !Varlen ? params.h * params.seqlen_q : 0}, // stride_LSE
+ params.cu_seqlens_q, params.seqused_q
+ };
+
+ typename CombineKernel::Params kernel_params = CombineKernel::to_underlying_arguments(args);
+ int num_blocks_m = cute::ceil_div(params.seqlen_q * params.h * (!Varlen ? params.b : 1), kBlockM);
+ dim3 grid_m(num_blocks_m, !Varlen ? 1 : params.b);
+ auto kernel = cutlass::device_kernel<CombineKernel>;
+ int smem_size = CombineKernel::SharedStorageSize;
+ if (smem_size >= 48 * 1024) {
+ CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
+ }
+ kernel<<<grid_m, CombineKernel::MaxThreadsPerBlock, smem_size, stream>>>(kernel_params);
+ CHECK_CUDA_KERNEL_LAUNCH();
+}
+
+template<typename T, typename Tpartial, int kHeadDim>
+void run_mha_fwd_combine_(Flash_fwd_params &params, cudaStream_t stream) {
+ // We want kBlockM to be as small as possible to maximize parallelism.
+ // E.g., if hdim is 64, we want kBlockM to be 16 so that we can use 256 threads, each reading 4 elements (floats).
+ static_assert(kHeadDim % 32 == 0, "kHeadDim must be a multiple of 32");
+ static constexpr int kBlockM = kHeadDim % 128 == 0 ? 8 : (kHeadDim % 64 == 0 ? 16 : 32);
+ BOOL_SWITCH(params.seqused_q != nullptr, Varlen, [&] {
+ if constexpr (kBlockM >= 16) { // If kBlockM == 8 then the minimum number of splits is 32.
+ if (params.num_splits <= 16) {
+ run_flash_fwd_combine<kHeadDim, kBlockM, 4, false /*IsEvenK*/, Varlen, T, Tpartial>(params, stream);
+ return;
+ }
+ }
+ if (params.num_splits <= 32) {
+ run_flash_fwd_combine<kHeadDim, kBlockM, 5, false /*IsEvenK*/, Varlen, T, Tpartial>(params, stream);
+ } else if (params.num_splits <= 64) {
+ run_flash_fwd_combine<kHeadDim, kBlockM, 6, false /*IsEvenK*/, Varlen, T, Tpartial>(params, stream);
+ } else if (params.num_splits <= 128) {
+ run_flash_fwd_combine<kHeadDim, kBlockM, 7, false /*IsEvenK*/, Varlen, T, Tpartial>(params, stream);
+ } else {
+ run_flash_fwd_combine<kHeadDim, kBlockM, 8, false /*IsEvenK*/, Varlen, T, Tpartial>(params, stream);
+ }
+ });
+}
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_hdim128_bf16_sm90.cu b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_hdim128_bf16_sm90.cu
new file mode 100644
index 0000000..502bc1d
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_hdim128_bf16_sm90.cu
@@ -0,0 +1,9 @@
+// Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+// Splitting the different template instantiations to different files to speed up compilation.
+// This file is auto-generated. See "generate_kernels.py"
+
+#include "flash_fwd_launch_template.h"
+
+#ifndef FLASHATTENTION_DISABLE_HDIM128
+template void run_mha_fwd_<90, cutlass::bfloat16_t, 128, false, false, false, false>(Flash_fwd_params &params, cudaStream_t stream);
+#endif
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_kernel_sm80.h b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_kernel_sm80.h
new file mode 100644
index 0000000..a2f5504
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_kernel_sm80.h
@@ -0,0 +1,216 @@
+/******************************************************************************
+ * Copyright (c) 2024, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cute/tensor.hpp"
+
+#include <cutlass/cutlass.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+#include <cutlass/kernel_hardware_info.h>
+
+#include "seqlen.h"
+#include "utils.h"
+#include "softmax.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <class CollectiveMainloop_, class CollectiveEpilogue_, class TileScheduler_>
+class FlashAttnFwdSm80 {
+
+public:
+
+ // Type Aliases
+ using CollectiveMainloop = CollectiveMainloop_;
+ using CollectiveEpilogue = CollectiveEpilogue_;
+ static constexpr bool Is_causal = CollectiveMainloop::Is_causal;
+ static constexpr bool Is_local = CollectiveMainloop::Is_local;
+ static_assert(CollectiveMainloop::Varlen == CollectiveEpilogue::Varlen);
+ static constexpr bool Has_softcap = CollectiveMainloop::Has_softcap;
+ static constexpr bool Varlen = CollectiveMainloop::Varlen;
+ static constexpr bool PagedKV = CollectiveMainloop::PagedKV;
+ static constexpr bool Split = CollectiveMainloop::Split;
+ static constexpr bool Is_FP8 = CollectiveMainloop::Is_FP8;
+ static constexpr bool Transpose_V = CollectiveMainloop::Transpose_V;
+ static constexpr bool AppendKV = CollectiveMainloop::AppendKV;
+ static constexpr bool PackGQA = CollectiveMainloop::PackGQA;
+ static constexpr int NumProducerThreads = CollectiveMainloop::NumProducerThreads;
+ using SeqlenInfo_t = typename CollectiveMainloop::SeqlenInfo_t;
+
+ // Mainloop derived types
+ using TileShape_MNK = typename CollectiveMainloop::TileShape_MNK;
+ using TiledMma = typename CollectiveMainloop::TiledMma;
+ using ArchTag = typename CollectiveMainloop::ArchTag;
+ using MainloopArguments = typename CollectiveMainloop::Arguments;
+ using MainloopParams = typename CollectiveMainloop::Params;
+
+ // Epilogue derived types
+ using EpilogueArguments = typename CollectiveEpilogue::Arguments;
+ using EpilogueParams = typename CollectiveEpilogue::Params;
+
+ static_assert(ArchTag::kMinComputeCapability >= 80);
+
+ using TileScheduler = TileScheduler_;
+ using TileSchedulerArguments = typename flash::TileSchedulerArguments;
+ using TileSchedulerParams = typename TileScheduler::Params;
+
+ static constexpr uint32_t NumThreads = CUTE_STATIC_V(size(TiledMma{}));
+ static constexpr uint32_t MaxThreadsPerBlock = CUTE_STATIC_V(size(TiledMma{}));
+ static constexpr uint32_t MinBlocksPerMultiprocessor = NumThreads == 128 ? 2 : 1;
+
+ // Kernel level shared memory storage
+ // We overlap the shared memory for the mainloop and epilogue. However, we only want smem_o to overlap with smem_v + smem_k and not smem_q
+ // and nothing else, so we'll pad in case sizeof(smem_o) > sizeof(smem_v) + sizeof(smem_k).
+ static constexpr int mainloop_smem_padding_ = int(sizeof(typename CollectiveEpilogue::TensorStorage))
+ - int(sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_v)))
+ - int(sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_k)));
+ static constexpr int mainloop_smem_padding = mainloop_smem_padding_ < 0 ? 0 : mainloop_smem_padding_;
+ struct SharedStorage {
+ struct TensorStorage : cute::aligned_struct<128> {
+ union {
+ struct {
+ cute::array<uint32_t, mainloop_smem_padding / sizeof(uint32_t)> padding_;
+ typename CollectiveMainloop::TensorStorage mainloop;
+ };
+ // We want smem_o to line up with the start of smem_v
+ typename CollectiveEpilogue::TensorStorage epilogue;
+ };
+ } tensors;
+
+ alignas(16) typename TileScheduler::SharedStorage smem_scheduler;
+
+ };
+
+ static constexpr int SharedStorageSize = sizeof(SharedStorage);
+
+ // Device side arguments
+ struct Arguments {
+ MainloopArguments mainloop{};
+ EpilogueArguments epilogue{};
+ cutlass::KernelHardwareInfo hw_info{};
+ TileSchedulerArguments scheduler{};
+ };
+
+ // Kernel entry point API
+ struct Params {
+ MainloopParams mainloop{};
+ EpilogueParams epilogue{};
+ cutlass::KernelHardwareInfo hw_info{};
+ TileSchedulerParams scheduler{};
+ };
+
+ //
+ // Methods
+ //
+
+ // Convert to underlying arguments. In this case, a simple copy for the aliased type.
+ static
+ Params
+ to_underlying_arguments(Arguments const& args) {
+ CUTLASS_TRACE_HOST("to_underlying_arguments():");
+
+ // Get SM count if needed, otherwise use user supplied SM count
+ int sm_count = args.hw_info.sm_count;
+ if (sm_count <= 0) {
+ CUTLASS_TRACE_HOST(" WARNING: Arguments do not include a valid SM count.\n"
+ " For optimal performance, populate the arguments KernelHardwareInfo struct with the SM count.");
+ sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(args.hw_info.device_id);
+ }
+
+ CUTLASS_TRACE_HOST("to_underlying_arguments(): Setting persistent grid SM count to " << sm_count);
+
+ cutlass::KernelHardwareInfo hw_info{args.hw_info.device_id, sm_count};
+ return {
+ CollectiveMainloop::to_underlying_arguments(args.mainloop),
+ CollectiveEpilogue::to_underlying_arguments(args.epilogue),
+ hw_info,
+ TileScheduler::to_underlying_arguments(args.scheduler)
+ };
+ }
+
+ // Computes the kernel launch grid shape based on runtime parameters
+ static dim3
+ get_grid_shape(Params const& params) {
+ return TileScheduler::get_grid_shape(params.scheduler, params.hw_info.sm_count * MinBlocksPerMultiprocessor);
+ }
+
+ static dim3
+ get_block_shape() {
+ return dim3(MaxThreadsPerBlock, 1, 1);
+ }
+
+ CUTLASS_DEVICE
+ void
+ operator()(Params const& params, char* smem_buf) {
+
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+
+ SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(smem_buf);
+
+ CollectiveMainloop collective_mainloop;
+ CollectiveEpilogue collective_epilogue;
+
+ TileScheduler scheduler(reinterpret_cast<typename TileScheduler::SharedStorage*>(&shared_storage.smem_scheduler));
+ // Initialize matmul objects.
+ TiledMma tiled_mma;
+
+ scheduler.init_consumer();
+
+ int warp_idx = cutlass::canonical_warp_idx_sync();
+ CUTLASS_PRAGMA_NO_UNROLL
+ for (auto work_tile_info = warp_idx == 0 ? scheduler.template get_initial_work</*IsProducerWarp=*/true>(params.scheduler) : scheduler.template get_initial_work</*IsProducerWarp=*/false>(params.scheduler);
+ work_tile_info.is_valid(params.scheduler);
+ work_tile_info = warp_idx == 0 ? scheduler.template get_next_work</*IsProducerWarp=*/true>(params.scheduler, work_tile_info) : scheduler.template get_next_work</*IsProducerWarp=*/false>(params.scheduler, work_tile_info)) {
+ // Attention output (GEMM-II) accumulator.
+ Tensor tOrO = partition_fragment_C(tiled_mma, select<0, 2>(TileShape_MNK{}));
+ float softmax_scale_log2 = params.mainloop.softmax_scale_log2;
+ // If there's tanh softcap, the scaling will be done before tanh.
+ auto block_coord = work_tile_info.get_block_coord(params.scheduler);
+ int const bidb = get<2>(block_coord);
+ if constexpr (Is_FP8 && !Has_softcap) {
+ int const bidh = get<1>(block_coord);
+ int const bidh_kv = !PackGQA ? params.mainloop.qhead_per_khead_divmod.divide(bidh) : bidh;
+ float const q_descale = params.mainloop.ptr_q_descale == nullptr ? 1.0f : params.mainloop.ptr_q_descale[bidb * get<0>(params.mainloop.stride_q_descale) + bidh_kv * get<1>(params.mainloop.stride_q_descale)];
+ float const k_descale = params.mainloop.ptr_k_descale == nullptr ? 1.0f : params.mainloop.ptr_k_descale[bidb * get<0>(params.mainloop.stride_k_descale) + bidh_kv * get<1>(params.mainloop.stride_k_descale)];
+ softmax_scale_log2 *= q_descale * k_descale;
+ }
+ flash::Softmax<2 * (2 * kBlockM / NumThreads), /*Max_offset=*/!Is_FP8 ? 0 : 8> softmax(softmax_scale_log2);
+
+ SeqlenInfo_t seqlen_info{
+ bidb,
+ get<0>(params.mainloop.shape_Q),
+ !PagedKV ? size<0>(params.mainloop.shape_K) : size<0>(params.mainloop.shape_K) * size<1>(params.mainloop.shape_pagetable),
+ get<0>(params.mainloop.shape_K_new),
+ params.mainloop.cu_seqlens_q, params.mainloop.cu_seqlens_k, params.mainloop.cu_seqlens_k_new,
+ params.mainloop.seqused_q, params.mainloop.seqused_k, params.mainloop.leftpad_k,
+ };
+ if constexpr (AppendKV) {
+ bool tile_new_valid = collective_mainloop.store_kv_new(
+ params.mainloop, threadIdx.x, shared_storage, seqlen_info, block_coord);
+ if (tile_new_valid) { __syncthreads(); }
+ }
+ bool tile_valid = collective_mainloop.mma(
+ params.mainloop, tOrO, softmax, threadIdx.x, seqlen_info, block_coord,
+ shared_storage);
+ scheduler.prefetch_next_work(params.scheduler, work_tile_info);
+ if (tile_valid) {
+ // if (threadIdx.x == 128) { printf("Before epilogue, bid.x = %d, bid.y = %d, bid.z = %d, m_block = %d, bidb = %d, split_idx = %d\n", blockIdx.x, blockIdx.y, blockIdx.z, m_block, bidb, split_idx); }
+ collective_epilogue.store(params.epilogue, tOrO, softmax.row_sum, shared_storage, tiled_mma,
+ threadIdx.x, block_coord);
+ } else {
+ // Write 0 to gO and -inf to gLSE.
+ // If Split, we don't have to write 0 to O if the mha_combine kernel is used, since it will
+ // not use the value of O if LSE is -inf.
+ collective_epilogue.template store_zero<!Split /*Clear_O*/>(params.epilogue, threadIdx.x, block_coord);
+ }
+ }
+
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_kernel_sm90.h b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_kernel_sm90.h
new file mode 100644
index 0000000..e541104
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_kernel_sm90.h
@@ -0,0 +1,420 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cute/tensor.hpp"
+
+#include <cutlass/cutlass.h>
+#include <cutlass/arch/reg_reconfig.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+#include <cutlass/numeric_conversion.h>
+#include <cutlass/kernel_hardware_info.h>
+#include "cutlass/pipeline/pipeline.hpp"
+
+#include "seqlen.h"
+#include "utils.h"
+#include "softmax.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <class CollectiveMainloop_, class CollectiveEpilogue_, class TileScheduler_>
+class FlashAttnFwdSm90 {
+
+public:
+
+ // Type Aliases
+ using CollectiveMainloop = CollectiveMainloop_;
+ using CollectiveEpilogue = CollectiveEpilogue_;
+ static constexpr bool Is_causal = CollectiveMainloop::Is_causal;
+ static constexpr bool Is_local = CollectiveMainloop::Is_local;
+ static_assert(CollectiveMainloop::Varlen == CollectiveEpilogue::Varlen);
+ static constexpr bool Has_softcap = CollectiveMainloop::Has_softcap;
+ static constexpr bool Varlen = CollectiveMainloop::Varlen;
+ static constexpr bool PagedKV = CollectiveMainloop::PagedKV;
+ static constexpr bool Split = CollectiveMainloop::Split;
+ static constexpr bool Is_FP8 = CollectiveMainloop::Is_FP8;
+ static constexpr bool Transpose_V = CollectiveMainloop::Transpose_V;
+ static constexpr bool AppendKV = CollectiveMainloop::AppendKV;
+ static constexpr bool Use_TMA_Q = CollectiveMainloop::Use_TMA_Q;
+ static constexpr bool Use_TMA_KV = CollectiveMainloop::Use_TMA_KV;
+ static constexpr bool Use_TMA_O = CollectiveEpilogue::Use_TMA_O;
+ static constexpr bool PackGQA = CollectiveMainloop::PackGQA;
+ static constexpr int NumProducerThreads = CollectiveMainloop::NumProducerThreads;
+ using SeqlenInfo_t = typename CollectiveMainloop::SeqlenInfo_t;
+
+ // Mainloop derived types
+ using TileShape_MNK = typename CollectiveMainloop::TileShape_MNK;
+ using TiledMma0 = typename CollectiveMainloop::TiledMma0;
+ using TiledMma1 = typename CollectiveMainloop::TiledMma1;
+ using ArchTag = typename CollectiveMainloop::ArchTag;
+ using ClusterShape = typename CollectiveMainloop::ClusterShape;
+ using MainloopArguments = typename CollectiveMainloop::Arguments;
+ using MainloopParams = typename CollectiveMainloop::Params;
+ using BarrierQ = std::conditional_t<Use_TMA_Q, cutlass::arch::ClusterTransactionBarrier, cutlass::arch::ClusterBarrier>;
+
+ // Epilogue derived types
+ using EpilogueArguments = typename CollectiveEpilogue::Arguments;
+ using EpilogueParams = typename CollectiveEpilogue::Params;
+
+ static_assert(ArchTag::kMinComputeCapability >= 90);
+
+ using TileScheduler = TileScheduler_;
+ using TileSchedulerArguments = typename flash::TileSchedulerArguments;
+ using TileSchedulerParams = typename TileScheduler::Params;
+
+ static constexpr uint32_t NumLoadWarpGroups = 1;
+ static constexpr uint32_t NumMmaWarpGroups = CUTE_STATIC_V(size(TiledMma0{})) / cutlass::NumThreadsPerWarpGroup;
+ static constexpr uint32_t MaxThreadsPerBlock = CUTE_STATIC_V(size(TiledMma0{})) + (NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup);
+ static constexpr uint32_t MinBlocksPerMultiprocessor = 1;
+ static_assert(NumMmaWarpGroups == 1 || NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3);
+
+ /// Register requirement for Load and Math WGs
+ // If we use cp.async to load K and V, we need more registers for the producer WG.
+ static constexpr uint32_t LoadRegisterRequirement = NumMmaWarpGroups == 1 ? 56 : (NumMmaWarpGroups == 2 ? (Use_TMA_KV ? 24 : 40) : 32);
+ static constexpr uint32_t MmaRegisterRequirement = NumMmaWarpGroups == 1 ? 256 : (NumMmaWarpGroups == 2 ? (Use_TMA_KV ? 240 : 232) : 160);
+ // If you want to print from the producer warp, you'd need to increase the number of registers
+ // Otherwise you'll get CUDA error.
+ // static constexpr uint32_t LoadRegisterRequirement = 40;
+ // static constexpr uint32_t MmaRegisterRequirement = NumMmaWarpGroups == 2 ? 232 : 152;
+
+ // Kernel level shared memory storage
+ // We overlap the shared memory for the mainloop and epilogue. However, we only want smem_o to overlap with smem_v
+ // and nothing else, so we'll pad in case sizeof(smem_o) > sizeof(smem_v).
+ static constexpr int mainloop_smem_padding_ = int(sizeof(typename CollectiveEpilogue::TensorStorage)) - int(sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_v)));
+ static constexpr int mainloop_smem_padding = mainloop_smem_padding_ < 0 ? 0 : mainloop_smem_padding_;
+ struct SharedStorage {
+ struct TensorStorage : cute::aligned_struct<128> {
+ union {
+ struct {
+ cute::array<uint32_t, mainloop_smem_padding / sizeof(uint32_t)> padding_;
+ typename CollectiveMainloop::TensorStorage mainloop;
+ };
+ // We want smem_o to line up with the start of smem_v
+ typename CollectiveEpilogue::TensorStorage epilogue;
+ };
+ } tensors;
+
+ struct PipelineStorage : cute::aligned_struct<16> {
+ alignas(16) BarrierQ barrier_Q;
+ alignas(16) cutlass::arch::ClusterBarrier barrier_O;
+ alignas(16) typename CollectiveMainloop::MainloopPipelineK::SharedStorage pipeline_k;
+ alignas(16) typename CollectiveMainloop::MainloopPipelineV::SharedStorage pipeline_v;
+ alignas(16) typename CollectiveMainloop::MainloopPipelineVt::SharedStorage pipeline_vt;
+ alignas(16) typename CollectiveMainloop::MainloopPipelineKVNew::SharedStorage pipeline_k_new;
+ alignas(16) typename CollectiveMainloop::MainloopPipelineKVNew::SharedStorage pipeline_v_new;
+ alignas(16) typename TileScheduler::SharedStorage smem_scheduler;
+ } pipelines;
+
+ };
+
+ static constexpr int SharedStorageSize = sizeof(SharedStorage);
+
+ // Device side arguments
+ struct Arguments {
+ MainloopArguments mainloop{};
+ EpilogueArguments epilogue{};
+ cutlass::KernelHardwareInfo hw_info{};
+ TileSchedulerArguments scheduler{};
+ };
+
+ // Kernel entry point API
+ struct Params {
+ MainloopParams mainloop{};
+ EpilogueParams epilogue{};
+ cutlass::KernelHardwareInfo hw_info{};
+ TileSchedulerParams scheduler{};
+ };
+
+ //
+ // Methods
+ //
+
+ // Convert to underlying arguments. In this case, a simple copy for the aliased type.
+ static
+ Params
+ to_underlying_arguments(Arguments const& args) {
+ CUTLASS_TRACE_HOST("to_underlying_arguments():");
+
+ // Get SM count if needed, otherwise use user supplied SM count
+ int sm_count = args.hw_info.sm_count;
+ if (sm_count <= 0) {
+ CUTLASS_TRACE_HOST(" WARNING: Arguments do not include a valid SM count.\n"
+ " For optimal performance, populate the arguments KernelHardwareInfo struct with the SM count.");
+ sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(args.hw_info.device_id);
+ }
+
+ CUTLASS_TRACE_HOST("to_underlying_arguments(): Setting persistent grid SM count to " << sm_count);
+
+ cutlass::KernelHardwareInfo hw_info{args.hw_info.device_id, sm_count};
+ return {
+ CollectiveMainloop::to_underlying_arguments(args.mainloop),
+ CollectiveEpilogue::to_underlying_arguments(args.epilogue),
+ hw_info,
+ TileScheduler::to_underlying_arguments(args.scheduler)
+ };
+ }
+
+ // Computes the kernel launch grid shape based on runtime parameters
+ static dim3
+ get_grid_shape(Params const& params) {
+ return TileScheduler::get_grid_shape(params.scheduler, params.hw_info.sm_count);
+ }
+
+ static dim3
+ get_block_shape() {
+ return dim3(MaxThreadsPerBlock, 1, 1);
+ }
+
+ CUTLASS_DEVICE
+ void
+ operator()(Params const& params, char* smem_buf) {
+
+ static constexpr int NumMmaThreads = NumMmaWarpGroups * cutlass::NumThreadsPerWarpGroup;
+ static constexpr int MmaThreadOffset = NumLoadWarpGroups * cutlass::NumThreadsPerWarpGroup;
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+
+ using MainloopPipelineK = typename CollectiveMainloop::MainloopPipelineK;
+ using MainloopPipelineV = typename CollectiveMainloop::MainloopPipelineV;
+ using MainloopPipelineVt = typename CollectiveMainloop::MainloopPipelineVt;
+ using MainloopPipelineKVNew = typename CollectiveMainloop::MainloopPipelineKVNew;
+ using PipelineState = typename CollectiveMainloop::PipelineState;
+ using PipelineParamsK = typename MainloopPipelineK::Params;
+ using PipelineParamsV = typename MainloopPipelineV::Params;
+ using PipelineParamsVt = typename MainloopPipelineVt::Params;
+ using PipelineParamsKVNew = typename MainloopPipelineKVNew::Params;
+
+ SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(smem_buf);
+
+ int const lane_predicate = cute::elect_one_sync();
+ int const warp_idx = cutlass::canonical_warp_idx_sync();
+
+ // Issue Tma Descriptor Prefetch from a single thread
+ if (warp_idx == 0 && lane_predicate) {
+ CollectiveMainloop::prefetch_tma_descriptors(params.mainloop);
+ CollectiveEpilogue::prefetch_tma_descriptors(params.epilogue);
+ }
+
+ // Obtain warp index
+ int const warp_group_thread_idx = threadIdx.x % cutlass::NumThreadsPerWarpGroup;
+ int warp_group_idx = cutlass::canonical_warp_group_idx();
+
+ if (warp_idx == 0 && lane_predicate) {
+ shared_storage.pipelines.barrier_Q.init(Use_TMA_Q ? 1 : NumProducerThreads /*numThreads*/);
+ shared_storage.pipelines.barrier_O.init(size(ClusterShape{}) * (Use_TMA_O ? 1 : NumMmaThreads) /*numThreads*/);
+ }
+
+ // We're counting on pipeline_k to call cutlass::arch::fence_barrier_init();
+ PipelineParamsK pipeline_params_k;
+ pipeline_params_k.role = warp_group_idx == 0
+ ? MainloopPipelineK::ThreadCategory::Producer
+ : MainloopPipelineK::ThreadCategory::Consumer;
+ if constexpr (Use_TMA_KV) {
+ pipeline_params_k.transaction_bytes = CollectiveMainloop::TmaTransactionBytesK;
+ pipeline_params_k.is_leader = warp_group_thread_idx == 0;
+ pipeline_params_k.num_consumers = NumMmaThreads;
+ } else {
+ pipeline_params_k.consumer_arv_count = NumMmaThreads;
+ pipeline_params_k.producer_arv_count = NumProducerThreads;
+ }
+
+ MainloopPipelineK pipeline_k = [&] {
+ if constexpr (Use_TMA_KV) {
+ return MainloopPipelineK(shared_storage.pipelines.pipeline_k, pipeline_params_k, ClusterShape{});
+ } else {
+ return MainloopPipelineK(shared_storage.pipelines.pipeline_k, pipeline_params_k);
+ }
+ }();
+ // MainloopPipelineV pipeline_v(shared_storage.pipelines.pipeline_v, pipeline_params_v, ClusterShape{});
+ MainloopPipelineV pipeline_v = [&] {
+ if constexpr (!Transpose_V) {
+ static_assert(is_same_v<PipelineParamsK, PipelineParamsV>);
+ if constexpr (Use_TMA_KV) {
+ return MainloopPipelineV(shared_storage.pipelines.pipeline_v, pipeline_params_k, ClusterShape{});
+ } else {
+ return MainloopPipelineV(shared_storage.pipelines.pipeline_v, pipeline_params_k);
+ }
+ } else {
+ PipelineParamsV pipeline_params_v;
+ pipeline_params_v.role = warp_group_idx == 0
+ ? MainloopPipelineV::ThreadCategory::Producer
+ : MainloopPipelineV::ThreadCategory::Consumer;
+ pipeline_params_v.producer_arv_count = NumProducerThreads;
+ pipeline_params_v.consumer_arv_count = NumMmaThreads;
+ return MainloopPipelineV(shared_storage.pipelines.pipeline_v, pipeline_params_v);
+ }
+ }();
+ static_assert(is_same_v<PipelineParamsK, PipelineParamsVt>);
+ // If we need to transpose V (e.g. FP8 and V is row-major), we use pipeline_vt for the TMA, then
+ // the producer WG will read from pipeline_vt and write to pipeline_v.
+ // If we don't need to transpose V, we use pipeline_v for the TMA, and pipeline_vt won't be used.
+ // Technically for pipeline_params_vt, warp0 of WG0 is the producer and all of WG0 are consumers.
+ // However, the thread role isn't used in the pipeline implementation.
+ MainloopPipelineVt pipeline_vt = [&] {
+ if constexpr (Use_TMA_KV) {
+ pipeline_params_k.num_consumers = NumProducerThreads; // TMA_V is only consumed by the producer WG
+ return MainloopPipelineVt(shared_storage.pipelines.pipeline_vt, pipeline_params_k, ClusterShape{});
+ } else {
+ pipeline_params_k.consumer_arv_count = NumProducerThreads; // TMA_V is only consumed by the producer WG
+ return MainloopPipelineVt(shared_storage.pipelines.pipeline_vt, pipeline_params_k);
+ }
+ }();
+
+ PipelineParamsKVNew pipeline_params_kv_new;
+ pipeline_params_kv_new.role = warp_group_idx == 0
+ ? MainloopPipelineKVNew::ThreadCategory::Producer
+ : MainloopPipelineKVNew::ThreadCategory::Consumer;
+ pipeline_params_kv_new.transaction_bytes = CollectiveMainloop::TmaTransactionBytesK;
+ pipeline_params_kv_new.is_leader = warp_group_thread_idx == 0;
+ pipeline_params_kv_new.num_consumers = NumMmaThreads;
+ auto pipeline_k_new = cute::conditional_return<AppendKV>(MainloopPipelineKVNew(shared_storage.pipelines.pipeline_k_new, pipeline_params_kv_new, ClusterShape{}), nullptr);
+ auto pipeline_v_new = cute::conditional_return<AppendKV>(MainloopPipelineKVNew(shared_storage.pipelines.pipeline_v_new, pipeline_params_kv_new, ClusterShape{}), nullptr);
+
+ CollectiveMainloop collective_mainloop;
+ CollectiveEpilogue collective_epilogue;
+
+ // We need this to guarantee that the Pipeline init is visible to all producers and consumer blocks in the Cluster
+ if constexpr (size(ClusterShape{}) > 1) {
+ cute::cluster_arrive_relaxed();
+ cute::cluster_wait();
+ } else {
+ __syncthreads();
+ }
+
+ if (warp_group_idx == 0) { // Producer
+ cutlass::arch::warpgroup_reg_dealloc<LoadRegisterRequirement>();
+
+ // The pipelines for AppendKV and main attention are different, since e.g. main attention
+ // might use cp.async to load KV (if PagedKV) while AppendKV always uses TMA to load
+ // KV_new. Since the pipeline states are different, we have to manually sync to make
+ // sure the two pipelines don't race when accessing smem_k and smem_v.
+ PipelineState smem_pipe_write = cutlass::make_producer_start_state<MainloopPipelineK>();
+ PipelineState smem_pipe_write_new = cutlass::make_producer_start_state<MainloopPipelineKVNew>();
+ int work_idx = 0;
+
+ TileScheduler scheduler(reinterpret_cast<typename TileScheduler::SharedStorage*>(&shared_storage.pipelines.smem_scheduler));
+ int warp_idx_in_warpgroup = __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0);
+ static constexpr bool SingleProducerWarp = NumProducerThreads == cutlass::NumThreadsPerWarp;
+ if constexpr (SingleProducerWarp) {
+ if (warp_idx_in_warpgroup != 0) { return; }
+ }
+ if (!SingleProducerWarp && warp_idx_in_warpgroup != 0) { scheduler.init_consumer(); }
+
+ // Load Q, K, V
+ for (auto work_tile_info = SingleProducerWarp || warp_idx_in_warpgroup == 0 ? scheduler.template get_initial_work</*IsProducerWarp=*/true>(params.scheduler) : scheduler.template get_initial_work</*IsProducerWarp=*/false>(params.scheduler);
+ work_tile_info.is_valid(params.scheduler);
+ work_tile_info = SingleProducerWarp || warp_idx_in_warpgroup == 0 ? scheduler.template get_next_work</*IsProducerWarp=*/true>(params.scheduler, work_tile_info) : scheduler.template get_next_work</*IsProducerWarp=*/false>(params.scheduler, work_tile_info)) {
+
+ auto block_coord = work_tile_info.get_block_coord(params.scheduler);
+ SeqlenInfo_t seqlen_info{
+ get<2>(block_coord) /*bidb*/,
+ get<0>(params.mainloop.shape_Q),
+ !PagedKV ? size<0>(params.mainloop.shape_K) : size<0>(params.mainloop.shape_K) * size<1>(params.mainloop.shape_pagetable),
+ get<0>(params.mainloop.shape_K_new),
+ params.mainloop.cu_seqlens_q, params.mainloop.cu_seqlens_k, params.mainloop.cu_seqlens_k_new,
+ params.mainloop.seqused_q, params.mainloop.seqused_k, params.mainloop.leftpad_k,
+ };
+ if constexpr (AppendKV) {
+ bool tile_new_valid = collective_mainloop.load_kv_new(
+ params.mainloop, pipeline_k_new, pipeline_v_new,
+ smem_pipe_write_new, shared_storage, seqlen_info, block_coord, work_idx);
+ if (tile_new_valid) {
+ // if (threadIdx.x == 0) { printf("Producer: Before sync\n"); }
+ cutlass::arch::NamedBarrier::sync(NumMmaThreads + NumProducerThreads, static_cast<uint32_t>(FwdNamedBarriers::AppendKV) /*id*/);
+ // if (threadIdx.x == 0) { printf("Producer: After sync\n"); }
+ }
+ }
+ auto scheduler_prefetch = [&scheduler, &params, &work_tile_info]() {
+ scheduler.prefetch_next_work(params.scheduler, work_tile_info);
+ };
+ // pipeline_vt won't be used if we don't need to transpose V.
+ collective_mainloop.load(params.mainloop, pipeline_k, pipeline_v, pipeline_vt, smem_pipe_write,
+ shared_storage, scheduler_prefetch, seqlen_info, block_coord, work_idx);
+ }
+ collective_mainloop.load_tail(pipeline_k, pipeline_v, pipeline_vt, smem_pipe_write, shared_storage, work_idx);
+ } else { // Consumer
+ cutlass::arch::warpgroup_reg_alloc<MmaRegisterRequirement>();
+
+ TileScheduler scheduler(reinterpret_cast<typename TileScheduler::SharedStorage*>(&shared_storage.pipelines.smem_scheduler));
+ // Initialize matmul objects.
+ TiledMma1 tiled_mma1;
+
+ PipelineState smem_pipe_read;
+ PipelineState smem_pipe_read_new;
+ // We don't need separate variables smem_pipe_release_k and smem_pipe_release_v
+ // (like in Cutlass's gemm) because the read and release pipeline states are always the same.
+
+ scheduler.init_consumer();
+ collective_mainloop.mma_init();
+
+ int work_idx = 0;
+ CUTLASS_PRAGMA_NO_UNROLL
+ for (auto work_tile_info = scheduler.template get_initial_work</*IsProducerWarp=*/false>(params.scheduler);
+ work_tile_info.is_valid(params.scheduler);
+ work_tile_info = scheduler.template get_next_work</*IsProducerWarp=*/false>(params.scheduler, work_tile_info)) {
+ // Attention output (GEMM-II) accumulator.
+ Tensor tOrO = partition_fragment_C(tiled_mma1, select<0, 2>(TileShape_MNK{}));
+ float softmax_scale_log2 = params.mainloop.softmax_scale_log2;
+ // If there's tanh softcap, the scaling will be done before tanh.
+ auto block_coord = work_tile_info.get_block_coord(params.scheduler);
+ int const bidb = get<2>(block_coord);
+ if constexpr (Is_FP8 && !Has_softcap) {
+ int const bidh = get<1>(block_coord);
+ int const bidh_kv = !PackGQA ? params.mainloop.qhead_per_khead_divmod.divide(bidh) : bidh;
+ float const q_descale = params.mainloop.ptr_q_descale == nullptr ? 1.0f : params.mainloop.ptr_q_descale[bidb * get<0>(params.mainloop.stride_q_descale) + bidh_kv * get<1>(params.mainloop.stride_q_descale)];
+ float const k_descale = params.mainloop.ptr_k_descale == nullptr ? 1.0f : params.mainloop.ptr_k_descale[bidb * get<0>(params.mainloop.stride_k_descale) + bidh_kv * get<1>(params.mainloop.stride_k_descale)];
+ softmax_scale_log2 *= q_descale * k_descale;
+ }
+ flash::Softmax<2 * (2 * kBlockM / NumMmaThreads), /*Max_offset=*/!Is_FP8 ? 0 : 8> softmax(softmax_scale_log2);
+
+ SeqlenInfo_t seqlen_info{
+ bidb,
+ get<0>(params.mainloop.shape_Q),
+ !PagedKV ? size<0>(params.mainloop.shape_K) : size<0>(params.mainloop.shape_K) * size<1>(params.mainloop.shape_pagetable),
+ get<0>(params.mainloop.shape_K_new),
+ params.mainloop.cu_seqlens_q, params.mainloop.cu_seqlens_k, params.mainloop.cu_seqlens_k_new,
+ params.mainloop.seqused_q, params.mainloop.seqused_k, params.mainloop.leftpad_k,
+ };
+ if constexpr (AppendKV) {
+ bool tile_new_valid = collective_mainloop.store_kv_new(
+ params.mainloop, pipeline_k_new, pipeline_v_new, smem_pipe_read_new,
+ threadIdx.x - MmaThreadOffset, shared_storage, seqlen_info, block_coord);
+ if (tile_new_valid) {
+ // if (threadIdx.x == 128) { printf("Consumer: Before sync\n"); }
+ // We need this sync so that the gmem write from the consumers is visible to the producer
+ // that might do TMA read after that.
+ asm volatile ("fence.proxy.async.global;");
+ cutlass::arch::NamedBarrier::arrive(NumMmaThreads + NumProducerThreads, static_cast<uint32_t>(FwdNamedBarriers::AppendKV) /*id*/);
+ // arrive is enough, we don't need sync. The producer will sync, which means
+ // after that sync we're guaranteed that the AppendKV pipeline have finished
+ // loading and consumer smem_k and smem_v.
+ // if (threadIdx.x == 128) { printf("Consumer: After sync\n"); }
+ }
+ }
+ bool tile_valid = collective_mainloop.mma(
+ params.mainloop, pipeline_k, pipeline_v, smem_pipe_read,
+ tOrO, softmax, threadIdx.x - MmaThreadOffset, work_idx, seqlen_info, block_coord, shared_storage);
+ if (tile_valid) {
+ // if (threadIdx.x == 128) { printf("Before epilogue, bid.x = %d, bid.y = %d, bid.z = %d, m_block = %d, bidb = %d, split_idx = %d\n", blockIdx.x, blockIdx.y, blockIdx.z, m_block, bidb, split_idx); }
+ collective_epilogue.store(params.epilogue, tOrO, softmax.row_sum, shared_storage, tiled_mma1,
+ threadIdx.x - MmaThreadOffset, block_coord);
+ } else {
+ // Write 0 to gO and -inf to gLSE.
+ // If Split, we don't have to write 0 to O if the mha_combine kernel is used, since it will
+ // not use the value of O if LSE is -inf.
+ collective_epilogue.template store_zero<!Split /*Clear_O*/>(params.epilogue, threadIdx.x - MmaThreadOffset, block_coord);
+ // collective_epilogue.store_zero(params.epilogue, threadIdx.x - MmaThreadOffset, block_coord);
+ }
+ }
+ collective_epilogue.store_tail();
+ }
+
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_launch_template.h b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_launch_template.h
new file mode 100644
index 0000000..16701f1
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/flash_fwd_launch_template.h
@@ -0,0 +1,205 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cute/tensor.hpp"
+
+#include "cutlass/cutlass.h"
+#include "cutlass/device_kernel.h" // For device_kernel
+#include <cutlass/kernel_hardware_info.h>
+#include "cutlass/cluster_launch.hpp"
+
+#include "static_switch.h"
+#include "flash.h"
+#include "tile_size.h"
+#include "tile_scheduler.hpp"
+#include "flash_fwd_kernel_sm90.h"
+#include "flash_fwd_kernel_sm80.h"
+#include "mainloop_fwd_sm90_tma_gmma_ws.hpp"
+#include "mainloop_fwd_sm80.hpp"
+#include "epilogue_fwd.hpp"
+
+using namespace cute;
+
+template <int Arch, int kHeadDim, int ClusterM, typename Element, typename ElementOut,
+ bool Is_causal, bool Is_local, bool Has_softcap, bool Varlen, bool PagedKV, bool AppendKV,
+ bool PackGQA, bool Split, bool V_colmajor>
+void run_flash_fwd(Flash_fwd_params &params, cudaStream_t stream) {
+ static_assert(!(Is_causal && Is_local), "Causal and Local cannot be enabled at the same time");
+ static_assert(!(AppendKV && V_colmajor), "AppendKV and V_colmajor cannot be enabled at the same time");
+ static_assert(!(AppendKV && !Varlen), "AppendKV requires Varlen");
+ static constexpr bool Is_FP8 = cute::is_same_v<Element, cutlass::float_e4m3_t> || cute::is_same_v<Element, cutlass::float_e5m2_t>;
+ static constexpr bool FP8_TransposeV = Is_FP8 && !V_colmajor;
+ using ArchTag = std::conditional_t<Arch >= 90, cutlass::arch::Sm90, cutlass::arch::Sm80>;
+
+ // Can't use structured binding since it's not compatible with constexpr
+ static constexpr std::tuple<int, int, bool, bool> kBlockMN_RS_IntraWGOverlap = tile_size_fwd_sm90(kHeadDim, Is_causal, Is_local, sizeof(Element) /*element_size*/, V_colmajor, PagedKV, Has_softcap);
+ static constexpr std::tuple<int, int, int, int, bool> kBlockMN_kNWarps_Stages_RS = tile_size_fwd_sm8x(Arch == 86 || Arch == 89, kHeadDim, Is_causal, Is_local, sizeof(Element) /*element_size*/, PagedKV, Varlen && Split, Has_softcap, AppendKV);
+ static constexpr int kBlockM = Arch >= 90 ? std::get<0>(kBlockMN_RS_IntraWGOverlap) : std::get<0>(kBlockMN_kNWarps_Stages_RS);
+ static constexpr int kBlockN = Arch >= 90 ? std::get<1>(kBlockMN_RS_IntraWGOverlap) : std::get<1>(kBlockMN_kNWarps_Stages_RS);
+ static constexpr bool Mma1_is_RS = std::get<2>(kBlockMN_RS_IntraWGOverlap);
+ static constexpr bool IntraWGOverlap = std::get<3>(kBlockMN_RS_IntraWGOverlap);
+ static constexpr int kNWarps = std::get<2>(kBlockMN_kNWarps_Stages_RS);
+ static constexpr int kStages = Arch >= 90 ? 2 : std::get<3>(kBlockMN_kNWarps_Stages_RS);
+ static constexpr bool Q_in_regs = Arch >= 90 ? false : std::get<4>(kBlockMN_kNWarps_Stages_RS);
+
+ using TileShape_MNK = cute::Shape<Int<kBlockM>, Int<kBlockN>, Int<kHeadDim>>;
+ using ClusterShape = cute::Shape<Int<ClusterM>, _1, _1>;
+ using CollectiveMainloop = std::conditional_t<
+ Arch >= 90,
+ flash::CollectiveMainloopFwdSm90<kStages, ClusterShape, TileShape_MNK, Element, float, cutlass::arch::Sm90, Is_causal, Is_local, Has_softcap, Varlen, PagedKV, AppendKV, Mma1_is_RS, IntraWGOverlap, PackGQA, Split, V_colmajor>,
+ flash::CollectiveMainloopFwdSm80<kNWarps, kStages, Q_in_regs, TileShape_MNK, Element, float, cutlass::arch::Sm80, Is_causal, Is_local, Has_softcap, Varlen, PagedKV, AppendKV, PackGQA, Split>
+ >;
+ using CollectiveEpilogue = flash::CollectiveEpilogueFwd<TileShape_MNK, ClusterShape, ElementOut, ArchTag, CollectiveMainloop::NumMmaThreads, Varlen, PackGQA, FP8_TransposeV>;
+
+ static constexpr int NumProducerThreads = Arch >= 90 ? CollectiveMainloop::NumProducerThreads : CollectiveMainloop::NumMmaThreads;
+ using SchedulerPersistent = std::conditional_t<Varlen,
+ flash::VarlenDynamicPersistentTileScheduler<kBlockM, CollectiveMainloop::NumMmaThreads, NumProducerThreads, Split, PackGQA, Arch >= 90 /*WarpSpecialized*/>,
+ std::conditional_t<!Is_causal && !Is_local,
+ flash::StaticPersistentTileScheduler<Split>,
+ flash::DynamicPersistentTileScheduler<CollectiveMainloop::NumMmaThreads, NumProducerThreads, Split, PackGQA, Arch >= 90 /*WarpSpecialized*/>
+ >
+ >;
+ using SchedulerSingleTile = flash::SingleTileScheduler<Varlen, Split, PackGQA, kBlockM>;
+ // If Split then we probably don't have enough work for PersistentScheduler to be useful.
+ // However, if Varlen (e.g., during decode where we have max_seqlens), using PersistentScheduler is better
+ // since we'll avoid launching a bunch of thread blocks that immediately exit.
+ // On Sm80, noncausal persistent seems a bit slower.
+ using Scheduler = std::conditional_t<Arch >= 90 ? (Split && !Varlen) : !((Is_causal && !Varlen) || (Varlen && Split)), SchedulerSingleTile, SchedulerPersistent>;
+ using AttnKernel = std::conditional_t<
+ Arch >= 90,
+ flash::enable_sm90_or_later<flash::FlashAttnFwdSm90<CollectiveMainloop, CollectiveEpilogue, Scheduler>>,
+ flash::enable_sm80_to_sm89<flash::FlashAttnFwdSm80<CollectiveMainloop, CollectiveEpilogue, Scheduler>>
+ >;
+
+ bool const is_varlen_q = params.cu_seqlens_q;
+ bool const is_varlen_k = params.cu_seqlens_k;
+ bool const is_varlen_k_new = params.cu_seqlens_knew;
+ int seqlen_q = !is_varlen_q ? params.seqlen_q : params.total_q;
+ int batch_q = !is_varlen_q ? params.b : 1;
+ int batch_k = !is_varlen_k ? (params.kv_batch_idx ? params.b_k : params.b) : 1;
+ typename CollectiveMainloop::StrideV v_strides =
+ cute::conditional_return<!V_colmajor>(
+ make_stride(params.v_row_stride, _1{}, params.v_head_stride, !is_varlen_k ? params.v_batch_stride : 0),
+ make_stride(_1{}, params.v_dim_stride, params.v_head_stride, !is_varlen_k ? params.v_batch_stride : 0));
+ typename CollectiveMainloop::Arguments mainloop_args {
+ static_cast<Element const*>(params.q_ptr),
+ {seqlen_q, params.d, params.h, batch_q}, // shape_Q
+ {params.q_row_stride, _1{}, params.q_head_stride, !is_varlen_q ? params.q_batch_stride : 0}, // stride_Q
+ static_cast<Element*>(params.k_ptr),
+ {!PagedKV ? (!is_varlen_k ? params.seqlen_k : params.total_k) : params.page_size,
+ params.d, params.h_k, !PagedKV ? batch_k : params.num_pages}, // shape_K
+ {params.k_row_stride, _1{}, params.k_head_stride, !is_varlen_k ? params.k_batch_stride : 0}, // stride_K
+ static_cast<Element*>(params.v_ptr),
+ v_strides, // stride_V
+ static_cast<Element const*>(params.knew_ptr),
+ {!is_varlen_k_new ? params.seqlen_knew : params.total_knew, params.d, params.h_k, !is_varlen_k_new ? params.b : 1}, // shape_K_new
+ {params.knew_row_stride, _1{}, params.knew_head_stride, !is_varlen_k_new ? params.knew_batch_stride : 0}, // stride_K_new
+ static_cast<Element const*>(params.vnew_ptr),
+ {params.vnew_row_stride, _1{}, params.vnew_head_stride, !is_varlen_k_new ? params.vnew_batch_stride : 0}, // stride_V_new
+ static_cast<Element const*>(params.rotary_cos_ptr),
+ {params.seqlen_k, params.rotary_dim / 2}, // shape_rotary, the seqlen shape doesn't matter
+ {params.rotary_dim / 2, _1{}}, // stride_rotary_cos
+ static_cast<Element const*>(params.rotary_sin_ptr),
+ {params.rotary_dim / 2, _1{}}, // stride_rotary_sin
+ params.is_rotary_interleaved,
+ params.page_table,
+ // if page_size is not set, avoid dividing by zero
+ {params.kv_batch_idx ? params.b_k : params.b, !PagedKV ? 0 : params.seqlen_k / params.page_size}, // shape_page_table
+ {params.page_table_batch_stride, _1{}}, // stride_page_table
+ params.scale_softmax,
+ params.q_descale_ptr, params.k_descale_ptr, params.v_descale_ptr,
+ {params.q_descale_batch_stride, params.q_descale_head_stride},
+ {params.k_descale_batch_stride, params.k_descale_head_stride},
+ {params.v_descale_batch_stride, params.v_descale_head_stride},
+ params.window_size_left, params.window_size_right, params.sink_token_length,
+ params.softcap,
+ params.num_splits,
+ params.kv_batch_idx,
+ params.cu_seqlens_q, params.cu_seqlens_k, params.cu_seqlens_knew,
+ params.seqused_q, params.seqused_k,
+ params.leftpad_k,
+ };
+ typename CollectiveEpilogue::Arguments epilogue_args {
+ static_cast<ElementOut*>(!Split ? params.o_ptr : params.oaccum_ptr),
+ {seqlen_q, params.d, params.h, batch_q, params.num_splits}, // shape_O
+ {!Split ? params.o_row_stride : params.oaccum_row_stride,
+ _1{},
+ !Split ? params.o_head_stride : params.oaccum_head_stride,
+ !is_varlen_q ? (!Split ? params.o_batch_stride : params.oaccum_batch_stride) : 0,
+ !Split ? 0 : params.oaccum_split_stride}, // stride_O
+ static_cast<float*>(!Split ? params.softmax_lse_ptr : params.softmax_lseaccum_ptr),
+ {_1{}, seqlen_q, !is_varlen_q ? params.h * seqlen_q : 0, !Split ? 0 : params.h * seqlen_q * batch_q}, // stride_LSE
+ params.h_k,
+ params.cu_seqlens_q, params.seqused_q
+ };
+
+ int qhead_per_khead = !PackGQA ? 1 : cutlass::ceil_div(params.h, params.h_k);
+ int num_blocks_m = cutlass::ceil_div(params.seqlen_q * qhead_per_khead, get<0>(TileShape_MNK{}));
+ num_blocks_m = cutlass::round_up(num_blocks_m, size<0>(ClusterShape{}));
+ typename flash::TileSchedulerArguments scheduler_args {
+ num_blocks_m, !PackGQA ? params.h : params.h_k, params.b, params.num_splits,
+ params.h / params.h_k,
+ params.seqlen_q,
+ params.seqlen_k, params.d, sizeof(Element),
+ params.tile_count_semaphore, params.cu_seqlens_q, params.seqused_q
+ };
+
+ int device;
+ CHECK_CUDA(cudaGetDevice(&device));
+ typename AttnKernel::Params kernel_params = AttnKernel::to_underlying_arguments({
+ mainloop_args, epilogue_args, {device, params.num_sm}, scheduler_args
+ });
+
+ dim3 grid_dims = AttnKernel::get_grid_shape(kernel_params);
+ dim3 block_dims = AttnKernel::get_block_shape();
+ int smem_size = AttnKernel::SharedStorageSize;
+ // int smem_size_q = sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_q));
+ // int smem_size_k = sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_k));
+ // int smem_size_v = sizeof(decltype((typename CollectiveMainloop::TensorStorage{}).smem_v));
+ // printf("smem_size = %d, q = %d, k = %d, v = %d\n", smem_size, smem_size_q, smem_size_k, smem_size_v);
+ // Get the ptr to kernel function.
+ if constexpr (size(ClusterShape{}) > 1) {
+ void const* kernel = (void const*) cutlass::device_kernel<AttnKernel>;
+ if (smem_size >= 48 * 1024) {
+ CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
+ }
+ dim3 cluster_dims(size<0>(ClusterShape{}), size<1>(ClusterShape{}), size<2>(ClusterShape{}));
+ cutlass::ClusterLaunchParams launch_params{grid_dims, block_dims, cluster_dims, smem_size, stream};
+ cutlass::launch_kernel_on_cluster(launch_params, kernel, kernel_params);
+ } else {
+ auto kernel = cutlass::device_kernel<AttnKernel>;
+ if (smem_size >= 48 * 1024) {
+ CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size));
+ }
+ kernel<<<grid_dims, block_dims, smem_size, stream>>>(kernel_params);
+ }
+ CHECK_CUDA_KERNEL_LAUNCH();
+}
+
+template<int Arch, typename T, int kHeadDim, bool Split, bool PagedKV, bool Has_softcap, bool PackGQA>
+void run_mha_fwd_(Flash_fwd_params &params, cudaStream_t stream) {
+ static_assert(sizeof(T) == 2 || sizeof(T) == 1, "Only 16bit and 8bit are supported");
+ static constexpr bool Is_FP8 = cute::is_same_v<T, cutlass::float_e4m3_t> || cute::is_same_v<T, cutlass::float_e5m2_t>;
+ using T_out = std::conditional_t<!Split, std::conditional_t<!Is_FP8, T, cutlass::bfloat16_t>, float>;
+ CAUSAL_LOCAL_SWITCH(params.is_causal, params.is_local, Is_causal, Is_local, [&] {
+ VCOLMAJOR_SWITCH(params.v_dim_stride != 1, V_colmajor_, [&] {
+ static constexpr bool V_colmajor = V_colmajor_ && sizeof(T) == 1;
+ VARLEN_SWITCH(params.cu_seqlens_q || params.cu_seqlens_k || params.seqused_q || params.seqused_k || params.leftpad_k, Varlen, [&] {
+ // Only needed here to decide if we should use cluster
+ static constexpr int kBlockM = Arch >= 90 ? std::get<0>(tile_size_fwd_sm90(kHeadDim, Is_causal, Is_local, sizeof(T) /*element_size*/, V_colmajor, PagedKV, Has_softcap)) : 128;
+
+ static constexpr bool Enable_cluster = Arch >= 90 && (sizeof(T) == 2 ? (kHeadDim >= 128) : (kHeadDim == 192)) && !Is_causal && !Is_local && !Split && !PagedKV && !Varlen;
+ APPENDKV_SWITCH(params.knew_ptr, AppendKV, [&] {
+ // Only use Cluster if number of tiles along seqlen_q is even and not varlen
+ CLUSTER_SWITCH(cutlass::ceil_div(params.seqlen_q * (!PackGQA ? 1 : params.h / params.h_k), kBlockM) % 2 == 0, Use_cluster, [&] {
+ static constexpr int ClusterM = Enable_cluster && Use_cluster ? 2 : 1;
+ run_flash_fwd<Arch, kHeadDim, ClusterM, T, T_out, Is_causal, Is_local, Has_softcap, Varlen, PagedKV, AppendKV && Varlen, PackGQA, Split, V_colmajor>(params, stream);
+ });
+ });
+ });
+ });
+ });
+}
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/heuristics.h b/src/solarwm/backends/minimax_h3/hopper_src/heuristics.h
new file mode 100644
index 0000000..8e7b4a3
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/heuristics.h
@@ -0,0 +1,48 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <vector>
+
+inline bool should_pack_gqa(bool varlen_q, int seqlen_q, int qhead_per_khead, int blockM) {
+ // If varlen, we don't actually know seqlen_q but only max_seqlen_q.
+ if (varlen_q) return true;
+ // Heuristic: PackGQA is a bit slower but can help if seqlen_q is small or not near a multiple of kBlockM
+ auto round_up = [](int a, int b) { return (a + b - 1) / b * b; };
+ float nopack_gqa_efficiency = float(seqlen_q) / float(round_up(seqlen_q, blockM));
+ float pack_gqa_efficiency = float(seqlen_q * qhead_per_khead) / float(round_up(seqlen_q * qhead_per_khead, blockM));
+ return nopack_gqa_efficiency < 0.9 * pack_gqa_efficiency;
+};
+
+// Find the number of splits that maximizes the occupancy. For example, if we have
+// batch * n_heads = 48 and we have 108 SMs, having 2 splits (efficiency = 0.89) is
+// better than having 3 splits (efficiency = 0.67). However, we also don't want too many
+// splits as that would incur more HBM reads/writes.
+// So we find the best efficiency, then find the smallest number of splits that gets 85%
+// of the best efficiency.
+inline int num_splits_heuristic(int batch_nheads_mblocks, int num_SMs, int num_n_blocks, int max_splits) {
+ // If we have enough to almost fill the SMs, then just use 1 split
+ if (batch_nheads_mblocks >= 0.8f * num_SMs) { return 1; }
+ // If num_n_blocks is too small, use 1 split. For example, we never split for hdim = 128 and seqlen_k = 512.
+ if (num_n_blocks <= 4) { return 1; }
+ max_splits = std::min({max_splits, num_SMs, num_n_blocks});
+ float max_efficiency = 0.f;
+ std::vector<float> efficiency;
+ efficiency.reserve(max_splits);
+ for (int num_splits = 1; num_splits <= max_splits; num_splits++) {
+ float n_waves = float(batch_nheads_mblocks * num_splits) / num_SMs;
+ float eff = n_waves / ceil(n_waves);
+ // printf("num_splits = %d, eff = %f\n", num_splits, eff);
+ if (eff > max_efficiency) { max_efficiency = eff; }
+ efficiency.push_back(eff);
+ }
+ for (int num_splits = 1; num_splits <= max_splits; num_splits++) {
+ if (efficiency[num_splits - 1] >= 0.85 * max_efficiency) {
+ // printf("num_splits chosen = %d\n", num_splits);
+ return num_splits;
+ }
+ }
+ return 1;
+}
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/kernel_traits.h b/src/solarwm/backends/minimax_h3/hopper_src/kernel_traits.h
new file mode 100644
index 0000000..8c08974
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/kernel_traits.h
@@ -0,0 +1,344 @@
+/******************************************************************************
+ * Copyright (c) 2024, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cute/tensor.hpp"
+
+#include "cutlass/cutlass.h"
+#include "cutlass/layout/layout.h"
+#include <cutlass/numeric_types.h>
+
+using namespace cute;
+
+template<int kHeadDim_, int kBlockM_, int kBlockN_, int kNWarps_, typename elem_type=cutlass::half_t>
+struct Flash_kernel_traits {
+
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+ using Element = elem_type;
+ static constexpr bool Has_cp_async = true;
+#else
+ using Element = cutlass::half_t;
+ static constexpr bool Has_cp_async = false;
+#endif
+
+ using ElementAccum = float;
+ using index_t = int64_t;
+
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+ using MMA_Atom_Arch = std::conditional_t<
+ std::is_same_v<elem_type, cutlass::half_t>,
+ MMA_Atom<SM80_16x8x16_F32F16F16F32_TN>,
+ MMA_Atom<SM80_16x8x16_F32BF16BF16F32_TN>
+ >;
+#else
+ using MMA_Atom_Arch = MMA_Atom<SM75_16x8x8_F32F16F16F32_TN>;
+#endif
+
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 750
+ using SmemCopyAtom = Copy_Atom<SM75_U32x4_LDSM_N, elem_type>;
+ using SmemCopyAtomTransposed = Copy_Atom<SM75_U16x8_LDSM_T, elem_type>;
+#else
+ using SmemCopyAtom = Copy_Atom<DefaultCopy, elem_type>;
+ using SmemCopyAtomTransposed = Copy_Atom<DefaultCopy, elem_type>;
+#endif
+};
+
+// If Share_Q_K_smem is true, that forces Is_Q_in_regs to be true
+template<int kHeadDim_, int kBlockM_, int kBlockN_, int kNWarps_, bool Is_Q_in_regs_=false, bool Share_Q_K_smem_=false, typename elem_type=cutlass::half_t,
+ typename Base=Flash_kernel_traits<kHeadDim_, kBlockM_, kBlockN_, kNWarps_, elem_type> >
+struct Flash_fwd_kernel_traits : public Base {
+ using Element = typename Base::Element;
+ using ElementAccum = typename Base::ElementAccum;
+ using index_t = typename Base::index_t;
+ static constexpr bool Has_cp_async = Base::Has_cp_async;
+ using SmemCopyAtom = typename Base::SmemCopyAtom;
+ using SmemCopyAtomTransposed = typename Base::SmemCopyAtomTransposed;
+
+ static constexpr bool Share_Q_K_smem = Share_Q_K_smem_;
+ static constexpr bool Is_Q_in_regs = Is_Q_in_regs_ || Share_Q_K_smem;
+
+ // The number of threads.
+ static constexpr int kNWarps = kNWarps_;
+ static constexpr int kNThreads = kNWarps * 32;
+
+ static constexpr int kBlockM = kBlockM_;
+ static constexpr int kBlockN = kBlockN_;
+ static constexpr int kHeadDim = kHeadDim_;
+ static_assert(kHeadDim % 32 == 0);
+ static constexpr int kBlockKSmem = kHeadDim % 64 == 0 ? 64 : 32;
+ static constexpr int kBlockKGmem = kHeadDim % 128 == 0 ? 128 : (kHeadDim % 64 == 0 ? 64 : 32);
+ static constexpr int kSwizzle = kBlockKSmem == 32 ? 2 : 3;
+
+ using TiledMma = TiledMMA<
+ typename Base::MMA_Atom_Arch,
+ Layout<Shape<Int<kNWarps>,_1,_1>>, // 4x1x1 or 8x1x1 thread group
+ Tile<Int<16 * kNWarps>, _16, _16>>;
+
+ using SmemLayoutAtomQ = decltype(
+ composition(Swizzle<kSwizzle, 3, 3>{},
+ // This has to be kBlockKSmem, using kHeadDim gives wrong results for d=128
+ Layout<Shape<_8, Int<kBlockKSmem>>,
+ Stride<Int<kBlockKSmem>, _1>>{}));
+ using SmemLayoutQ = decltype(tile_to_shape(
+ SmemLayoutAtomQ{},
+ Shape<Int<kBlockM>, Int<kHeadDim>>{}));
+
+ using SmemLayoutKV = decltype(tile_to_shape(
+ SmemLayoutAtomQ{},
+ Shape<Int<kBlockN>, Int<kHeadDim>>{}));
+
+ // https://github.com/ColfaxResearch/cutlass-kernels/blob/a222587e6d59b93ba704853d3946fb686d8b8892/src/fmha/fmha_forward.cu#L434
+ using SmemLayoutVtransposed = decltype(
+ composition(SmemLayoutKV{}, make_layout(Shape<Int<kHeadDim>, Int<kBlockN>>{}, GenRowMajor{})));
+ using SmemLayoutVtransposedNoSwizzle = decltype(get_nonswizzle_portion(SmemLayoutVtransposed{}));
+
+ using SmemLayoutAtomO = decltype(
+ composition(Swizzle<kSwizzle, 3, 3>{},
+ Layout<Shape<Int<8>, Int<kBlockKSmem>>,
+ Stride<Int<kBlockKSmem>, _1>>{}));
+ using SmemLayoutO = decltype(tile_to_shape(
+ SmemLayoutAtomO{},
+ Shape<Int<kBlockM>, Int<kHeadDim>>{}));
+ using SmemCopyAtomO = Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>;
+ using SmemCopyAtomOaccum = Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementAccum>;
+
+ static constexpr int kSmemQSize = size(SmemLayoutQ{}) * sizeof(Element);
+ static constexpr int kSmemKVSize = size(SmemLayoutKV{}) * 2 * sizeof(Element);
+ static constexpr int kSmemSize = Share_Q_K_smem ? std::max(kSmemQSize, kSmemKVSize) : kSmemQSize + kSmemKVSize;
+
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(Element);
+ static_assert(kHeadDim % kGmemElemsPerLoad == 0, "kHeadDim must be a multiple of kGmemElemsPerLoad");
+ // Using kBlockKSmem here is 6-10% faster than kBlockKGmem for d=128 because of bank conflicts.
+ // For example, for d=128, smem is split into 2 "pages", each page takes care of columns
+ // 0-63 and 64-127. If we have 16 threads per row for gmem read, when we write to smem,
+ // thread 0 - 7 will write to the first page and thread 8 - 15 will write to the second page,
+ // to the same banks.
+ static constexpr int kGmemThreadsPerRow = kBlockKSmem / kGmemElemsPerLoad;
+ static_assert(kNThreads % kGmemThreadsPerRow == 0, "kNThreads must be a multiple of kGmemThreadsPerRow");
+ using GmemLayoutAtom = Layout<Shape <Int<kNThreads / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+
+ // We use CACHEGLOBAL instead of CACHEALWAYS for both Q and K/V, since we won't be reading
+ // from the same address by the same threadblock. This is slightly faster.
+ using Gmem_copy_struct = std::conditional_t<
+ Has_cp_async,
+ SM80_CP_ASYNC_CACHEGLOBAL<cute::uint128_t>,
+ AutoVectorizingCopyWithAssumedAlignment<128>
+ >;
+ using GmemTiledCopyQKV = decltype(
+ make_tiled_copy(Copy_Atom<Gmem_copy_struct, Element>{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, _8>>{})); // Val layout, 8 vals per read
+ using GmemTiledCopyO = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, _8>>{})); // Val layout, 8 vals per store
+
+ using GmemLayoutAtomOaccum = std::conditional_t<
+ kBlockKSmem == 32,
+ Layout<Shape <_16, _8>, // Thread layout, 8 threads per row
+ Stride< _8, _1>>,
+ Layout<Shape <_8, _16>, // Thread layout, 16 threads per row
+ Stride< _16, _1>>
+ >;
+ using GmemTiledCopyOaccum = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementAccum>{},
+ GmemLayoutAtomOaccum{},
+ Layout<Shape < _1, _4>>{})); // Val layout, 4 vals per store
+ using GmemLayoutAtomRotcossin = GmemLayoutAtom;
+ using GmemTiledCopyRotcossin = decltype(
+ make_tiled_copy(Copy_Atom<UniversalCopy<uint64_t>, Element>{},
+ GmemLayoutAtomRotcossin{},
+ Layout<Shape < _1, _4>>{})); // Val layout, 4 vals per load
+ using GmemTiledCopyRotcossinCont = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ GmemLayoutAtomRotcossin{},
+ Layout<Shape < _1, _8>>{})); // Val layout, 8 vals per load
+};
+
+// Is_V_in_regs is an option to reduce smem usage, but will increase register pressue.
+// No_double_buffer is another option to reduce smem usage, but will slow things down.
+template<int kHeadDim_, int kBlockM_, int kBlockN_, int kNWarps_,
+ int AtomLayoutMSdP_=1, int AtomLayoutNdKV=2, int AtomLayoutMdQ=2,
+ bool Is_V_in_regs_=false, bool No_double_buffer_=false, typename elem_type=cutlass::half_t,
+ typename Base=Flash_kernel_traits<kHeadDim_, kBlockM_, kBlockN_, kNWarps_, elem_type> >
+struct Flash_bwd_kernel_traits : public Base {
+ using Element = typename Base::Element;
+ using ElementAccum = typename Base::ElementAccum;
+ using index_t = typename Base::index_t;
+ static constexpr bool Has_cp_async = Base::Has_cp_async;
+ using SmemCopyAtom = typename Base::SmemCopyAtom;
+ using SmemCopyAtomTransposed = typename Base::SmemCopyAtomTransposed;
+
+ static constexpr bool Is_V_in_regs = Is_V_in_regs_;
+ static constexpr bool No_double_buffer = No_double_buffer_;
+
+ // The number of threads.
+ static constexpr int kNWarps = kNWarps_;
+ static constexpr int kNThreads = kNWarps * 32;
+
+ static constexpr int kBlockM = kBlockM_;
+ static constexpr int kBlockN = kBlockN_;
+ static constexpr int kHeadDim = kHeadDim_;
+ static_assert(kHeadDim % 32 == 0);
+ static constexpr int kBlockKSmem = kHeadDim % 64 == 0 ? 64 : 32;
+ static constexpr int kBlockKGmem = kHeadDim % 128 == 0 ? 128 : (kHeadDim % 64 == 0 ? 64 : 32);
+ static constexpr int kSwizzle = kBlockKSmem == 32 ? 2 : 3;
+
+ static constexpr int AtomLayoutMSdP = AtomLayoutMSdP_;
+ static_assert(kNWarps % AtomLayoutMSdP == 0);
+ static_assert(kNWarps % AtomLayoutNdKV == 0);
+ static_assert(kNWarps % AtomLayoutMdQ == 0);
+
+ using TiledMmaSdP = TiledMMA<
+ typename Base::MMA_Atom_Arch,
+ Layout<Shape<Int<AtomLayoutMSdP>, Int<kNWarps / AtomLayoutMSdP>, _1>>,
+ Tile<Int<16 * AtomLayoutMSdP>, Int<16 * kNWarps / AtomLayoutMSdP>, _16>>;
+
+ using TiledMmadKV = TiledMMA<
+ typename Base::MMA_Atom_Arch,
+ Layout<Shape<Int<AtomLayoutNdKV>, Int<kNWarps / AtomLayoutNdKV>, _1>>,
+ Tile<Int<16 * AtomLayoutNdKV>, Int<16 * kNWarps / AtomLayoutNdKV>, _16>>;
+
+ using TiledMmadQ = TiledMMA<
+ typename Base::MMA_Atom_Arch,
+ Layout<Shape<Int<AtomLayoutMdQ>, Int<kNWarps / AtomLayoutMdQ>, _1>>, // 2x4x1 or 4x2x1 thread group
+ Tile<Int<16 * AtomLayoutMdQ>, Int<16 * kNWarps / AtomLayoutMdQ>, _16>>;
+
+ using SmemLayoutAtomQdO = decltype(
+ composition(Swizzle<kSwizzle, 3, 3>{},
+ Layout<Shape<_8, Int<kBlockKSmem>>,
+ Stride<Int<kBlockKSmem>, _1>>{}));
+ using SmemLayoutQdO = decltype(tile_to_shape(
+ SmemLayoutAtomQdO{},
+ make_shape(Int<kBlockM>{}, Int<kHeadDim>{})));
+
+ using SmemLayoutAtomKV = decltype(
+ composition(Swizzle<kSwizzle, 3, 3>{},
+ Layout<Shape<Int<kBlockM / kNWarps>, Int<kBlockKSmem>>,
+ Stride<Int<kBlockKSmem>, _1>>{}));
+ using SmemLayoutKV = decltype(tile_to_shape(
+ // SmemLayoutAtomQdO{},
+ SmemLayoutAtomKV{},
+ make_shape(Int<kBlockN>{}, Int<kHeadDim>{})));
+
+ using SmemLayoutKtransposed = decltype(
+ composition(SmemLayoutKV{}, make_layout(Shape<Int<kHeadDim>, Int<kBlockN>>{}, GenRowMajor{})));
+ using SmemLayoutKtransposedNoSwizzle = decltype(get_nonswizzle_portion(SmemLayoutKtransposed{}));
+
+ // TODO: generalize to other values of kBlockN
+ // TODO: what should be the Swizzle here? 3 is faster than 1, and 1 is faster than 2
+ // static constexpr int kPBlockN = kBlockN;
+ // Temporarily disabling this for hdim 256 on sm86 and sm89
+ // static_assert(kBlockN >= 64);
+ static_assert(kBlockN >= 32);
+ // TD [2023-03-19]: Idk why kPBlockN = 16 and kSwizzlePdS=3 is the fastest.
+ static constexpr int kPBlockN = kBlockN >= 64 ? 64 : 32;
+ static_assert(kPBlockN == 16 || kPBlockN == 32 || kPBlockN == 64);
+ // static constexpr int kSwizzlePdS = kPBlockN == 16 ? 1 : (kPBlockN == 32 ? 2 : 3);
+ static constexpr int kSwizzlePdS = 3;
+ using SmemLayoutAtomPdS = decltype(
+ composition(Swizzle<kSwizzlePdS, 3, 3>{},
+ Layout<Shape<Int<kBlockM>, Int<kPBlockN>>,
+ Stride<Int<kPBlockN>, _1>>{}));
+ using SmemLayoutPdS = decltype(tile_to_shape(
+ SmemLayoutAtomPdS{},
+ make_shape(Int<kBlockM>{}, Int<kBlockN>{})));
+ using SmemLayoutPdStransposed = decltype(
+ composition(SmemLayoutPdS{}, make_layout(Shape<Int<kBlockN>, Int<kBlockM>>{}, GenRowMajor{})));
+ using SmemLayoutPdStransposedNoSwizzle = decltype(get_nonswizzle_portion(SmemLayoutPdStransposed{}));
+
+ using SmemCopyAtomPdS = Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, elem_type>;
+
+ using SmemLayoutQdOtransposed = decltype(
+ composition(SmemLayoutQdO{}, make_layout(Shape<Int<kHeadDim>, Int<kBlockM>>{}, GenRowMajor{})));
+ using SmemLayoutQdOtransposedNoSwizzle = decltype(get_nonswizzle_portion(SmemLayoutQdOtransposed{}));
+
+ using SmemLayoutAtomdKV = decltype(
+ composition(Swizzle<kSwizzle, 3, 3>{},
+ Layout<Shape<_8, Int<kBlockKSmem>>,
+ Stride<Int<kBlockKSmem>, _1>>{}));
+ using SmemLayoutdKV = decltype(tile_to_shape(
+ SmemLayoutAtomdKV{},
+ make_shape(Int<kBlockN>{}, Int<kHeadDim>{})));
+ using SmemCopyAtomdKV = Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, elem_type>;
+
+ using SmemLayoutAtomdQ = decltype(
+ composition(Swizzle<kSwizzle, 3, 3>{},
+ Layout<Shape<_8, Int<kBlockKSmem>>,
+ Stride<Int<kBlockKSmem>, _1>>{}));
+ using SmemLayoutdQ = decltype(tile_to_shape(
+ SmemLayoutAtomdQ{},
+ make_shape(Int<kBlockM>{}, Int<kHeadDim>{})));
+ using SmemCopyAtomdQ = Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, elem_type>;
+
+ // Double buffer for sQ
+ static constexpr int kSmemQdOSize = size(SmemLayoutQdO{}) * (No_double_buffer ? 2 : 3) * sizeof(Element);
+ static constexpr int kSmemKVSize = size(SmemLayoutKV{}) * 2 * sizeof(Element);
+ static constexpr int kSmemdSSize = size(SmemLayoutPdS{}) * sizeof(Element);
+ static constexpr int kSmemPSize = size(SmemLayoutPdS{}) * sizeof(Element);
+ static constexpr int kSmemdQSize = size(SmemLayoutdQ{}) * sizeof(Element);
+ static constexpr int kSmemSize = kSmemQdOSize
+ + (!Is_V_in_regs
+ ? kSmemKVSize + kSmemdSSize + std::max(kSmemPSize, kSmemdQSize)
+ : std::max(kSmemKVSize, kSmemKVSize / 2 + kSmemdSSize + std::max(kSmemPSize, kSmemdQSize)));
+ static constexpr int kSmemSize1colblock = kSmemQdOSize
+ + (!Is_V_in_regs
+ ? kSmemKVSize + kSmemdSSize + kSmemPSize
+ : std::max(kSmemKVSize, kSmemKVSize / 2 + kSmemdSSize + kSmemPSize));
+
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(Element);
+ static_assert(kHeadDim % kGmemElemsPerLoad == 0, "kHeadDim must be a multiple of kGmemElemsPerLoad");
+ // Using kBlockKSmem instead of kHeadDim here to avoid bank conflicts, but doesn't seem
+ // to affect speed in practice.
+ static constexpr int kGmemThreadsPerRow = kBlockKSmem / kGmemElemsPerLoad;
+ static_assert(kNThreads % kGmemThreadsPerRow == 0, "kNThreads must be a multiple of kGmemThreadsPerRow");
+ using GmemLayoutAtom = Layout<Shape <Int<kNThreads / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+
+ // We use CACHEGLOBAL instead of CACHEALWAYS for both Q and K/V, since we won't be reading
+ // from the same address by the same threadblock. This is slightly faster.
+ using Gmem_copy_struct = std::conditional_t<
+ Has_cp_async,
+ SM80_CP_ASYNC_CACHEGLOBAL<cute::uint128_t>,
+ AutoVectorizingCopyWithAssumedAlignment<128>
+ >;
+ using GmemTiledCopyQKV = decltype(
+ make_tiled_copy(Copy_Atom<Gmem_copy_struct, elem_type>{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, _8>>{})); // Val layout, 8 vals per read
+ using GmemTiledCopydO = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, elem_type>{},
+ GmemLayoutAtom{},
+ Layout<Shape < _1, _8>>{})); // Val layout, 8 vals per store
+ using GmemTiledCopydKV = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, elem_type>{},
+ GmemLayoutAtom{},
+ Layout<Shape < _1, _8>>{})); // Val layout, 8 vals per store
+ using GmemTiledCopydQ = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, elem_type>{},
+ GmemLayoutAtom{},
+ Layout<Shape < _1, _8>>{})); // Val layout, 8 vals per store
+ using GmemLayoutAtomdQaccum = std::conditional_t<
+ kBlockKSmem == 32,
+ Layout<Shape <_32, _8>, // Thread layout, 8 threads per row
+ Stride< _8, _1>>,
+ Layout<Shape <_16, _16>, // Thread layout, 16 threads per row
+ Stride< _16, _1>>
+ >;
+ using GmemTiledCopydQaccum = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementAccum>{},
+ GmemLayoutAtomdQaccum{},
+ Layout<Shape < _1, _4>>{})); // Val layout, 4 vals per store
+
+ using GmemTiledCopydQaccumAtomicAdd = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementAccum>{},
+ Layout<Shape <_8, _32>, // Thread layout, 8 threads per row
+ Stride<_32, _1>>{},
+ Layout<Shape < _1, _1>>{})); // Val layout, 1 val per store
+
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/mainloop_bwd_sm80.hpp b/src/solarwm/backends/minimax_h3/hopper_src/mainloop_bwd_sm80.hpp
new file mode 100644
index 0000000..e7b3d2d
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/mainloop_bwd_sm80.hpp
@@ -0,0 +1,916 @@
+/******************************************************************************
+ * Copyright (c) 2024, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cutlass/cutlass.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+#include <cutlass/numeric_conversion.h>
+
+#include "cute/tensor.hpp"
+
+#include "seqlen.h"
+#include "mask.h"
+#include "softmax.h"
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <int Stages, int Stages_dO, class TileShape_MNK_, class Element_, class ElementAccum_, class ArchTag_,
+ bool Is_causal_, bool Is_local_, bool Has_softcap_, bool Varlen_, bool Deterministic,
+ bool SdP_swapAB_, bool dKV_swapAB_, bool dQ_swapAB_,
+ int NumMmaWarpGroups=2, int AtomLayoutMSdP=1, int AtomLayoutNdKV=8, int AtomLayoutMdQ=1,
+ bool V_in_regs=false>
+struct CollectiveMainloopBwdSm80 {
+
+ static constexpr int kStages = Stages;
+ static constexpr int kStages_dO = Stages_dO;
+ static_assert(kStages >= kStages_dO);
+ using TileShape_MNK = TileShape_MNK_;
+ using Element = Element_;
+ using ElementAccum = ElementAccum_;
+ using ArchTag = ArchTag_;
+ static constexpr bool Is_causal = Is_causal_;
+ static constexpr bool Is_local = Is_local_;
+ static constexpr bool Has_softcap = Has_softcap_;
+ static constexpr bool Varlen = Varlen_;
+ using SeqlenInfo_t = flash::SeqlenInfoQK<Varlen, CUTE_STATIC_V(get<0>(TileShape_MNK{}))>;
+ static constexpr int NumMmaWarps = NumMmaWarpGroups * cutlass::NumWarpsPerWarpGroup;
+
+ static constexpr bool SdP_swapAB = SdP_swapAB_;
+ static constexpr bool dKV_swapAB = dKV_swapAB_;
+ static constexpr bool dQ_swapAB = dQ_swapAB_;
+
+ static constexpr bool Q_dO_same_stages = kStages == kStages_dO;
+
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ static constexpr int kHeadDim = get<2>(TileShape_MNK{});
+
+ static_assert(ArchTag::kMinComputeCapability >= 80);
+
+ static constexpr bool Has_cp_async = ArchTag::kMinComputeCapability >= 80;
+
+ static constexpr int NumMmaThreads = NumMmaWarps * cutlass::NumThreadsPerWarp;
+ static constexpr int NumProducerThreads = NumMmaThreads; // For compatibility with TileScheduler
+
+ using MMA_Atom_Arch = std::conditional_t<
+ ArchTag::kMinComputeCapability >= 80,
+ std::conditional_t<
+ std::is_same_v<Element, cutlass::half_t>,
+ MMA_Atom<SM80_16x8x16_F32F16F16F32_TN>,
+ MMA_Atom<SM80_16x8x16_F32BF16BF16F32_TN>
+ >,
+ MMA_Atom<SM75_16x8x8_F32F16F16F32_TN>
+ >;
+
+ static_assert(NumMmaWarps % AtomLayoutMSdP == 0);
+ static_assert(NumMmaWarps % AtomLayoutNdKV == 0);
+ static_assert(NumMmaWarps % AtomLayoutMdQ == 0);
+ static constexpr bool Mma_dKV_is_RS = AtomLayoutMSdP == 1 && AtomLayoutNdKV == NumMmaWarps && SdP_swapAB && !dKV_swapAB;
+ static constexpr bool Mma_dQ_is_RS = AtomLayoutMSdP == NumMmaWarps && AtomLayoutMdQ == NumMmaWarps && !SdP_swapAB && !dQ_swapAB; // If dQ_swapAB we can't use RS
+
+ using AtomLayoutSdP = std::conditional_t<
+ !SdP_swapAB,
+ Layout<Shape<Int<AtomLayoutMSdP>, Int<NumMmaWarps / AtomLayoutMSdP>, _1>>,
+ Layout<Shape<Int<NumMmaWarps / AtomLayoutMSdP>, Int<AtomLayoutMSdP>, _1>>
+ >;
+ static constexpr bool MmaSdPEvenN = ((!SdP_swapAB ? kBlockN : kBlockM) / size<1>(AtomLayoutSdP{})) % 16 == 0;
+ using TiledMmaSdP = TiledMMA<
+ MMA_Atom_Arch,
+ AtomLayoutSdP,
+ Tile<Int<16 * CUTE_STATIC_V(size<0>(AtomLayoutSdP{}))>, Int<(MmaSdPEvenN ? 16 : 8) * CUTE_STATIC_V(size<1>(AtomLayoutSdP{}))>, _16>>;
+
+ using AtomLayoutdKV = std::conditional_t<
+ !dKV_swapAB,
+ Layout<Shape<Int<AtomLayoutNdKV>, Int<NumMmaWarps / AtomLayoutNdKV>, _1>>,
+ Layout<Shape<Int<NumMmaWarps / AtomLayoutNdKV>, Int<AtomLayoutNdKV>, _1>>
+ >;
+ static constexpr bool MmadKVEvenN = ((!dKV_swapAB ? kHeadDim : kBlockN) / size<1>(AtomLayoutdKV{})) % 16 == 0;
+ using TiledMmadKV = TiledMMA<
+ MMA_Atom_Arch,
+ AtomLayoutdKV,
+ Tile<Int<16 * CUTE_STATIC_V(size<0>(AtomLayoutdKV{}))>, Int<(MmadKVEvenN ? 16 : 8) * CUTE_STATIC_V(size<1>(AtomLayoutdKV{}))>, _16>>;
+
+ using AtomLayoutdQ = std::conditional_t<
+ !dQ_swapAB,
+ Layout<Shape<Int<AtomLayoutMdQ>, Int<NumMmaWarps / AtomLayoutMdQ>, _1>>,
+ Layout<Shape<Int<NumMmaWarps / AtomLayoutMdQ>, Int<AtomLayoutMdQ>, _1>>
+ >;
+ static constexpr bool MmadQEvenN = ((!dQ_swapAB ? kHeadDim : kBlockM) / size<1>(AtomLayoutdQ{})) % 16 == 0;
+ using TiledMmadQ = TiledMMA<
+ MMA_Atom_Arch,
+ AtomLayoutdQ,
+ Tile<Int<16 * CUTE_STATIC_V(size<0>(AtomLayoutdQ{}))>, Int<(MmadQEvenN ? 16 : 8) * CUTE_STATIC_V(size<1>(AtomLayoutdQ{}))>, _16>>;
+
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(Element);
+ static_assert(kHeadDim % kGmemElemsPerLoad == 0, "Headdim must be a multiple of kGmemElemsPerLoad");
+ // We want each "row" to have 64 elements (128 bytes, i.e. 1 cache line). E.g. if hdim=128, we want each
+ // thread to have 4 loads in the M direction and 2 vectorized load in the K direction.
+ static constexpr int kBytePerRow = kHeadDim * sizeof(Element);
+ static constexpr int kBlockKGmem = (kBytePerRow % 128 == 0 ? 128 : (kBytePerRow % 64 == 0 ? 64 : 32)) / sizeof(Element);
+
+ static constexpr int kSwizzle = kBlockKGmem == 128 ? 4 : (kBlockKGmem == 64 ? 3 : (kBlockKGmem == 32 ? 2 : 1));
+ static constexpr int kSwizzleBase = sizeof(Element) == 4 ? 2 : (sizeof(Element) == 2 ? 3 : 4);
+
+ // We need to accommodate both Q and Q^T (and dO and dO^T) in shared memory.
+ // Q & dO are used in the SdP Mma and Q^T and dO^T are used in the dKV Mma.
+ // Since this is GMMA::Major::K, the M dimension (kBlockM) doesn't matter for the layout, only the K dimension
+ // changes the layout.
+ using SmemLayoutAtomQdO = decltype(
+ composition(Swizzle<kSwizzle, kSwizzleBase, kSwizzleBase>{},
+ Layout<Shape<_8, Int<kBlockKGmem>>,
+ Stride<Int<kBlockKGmem>, _1>>{}));
+ using SmemLayoutQ =
+ decltype(tile_to_shape(SmemLayoutAtomQdO{},
+ make_shape(shape<0>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int<kStages>{})));
+ using SmemLayoutdO =
+ decltype(tile_to_shape(SmemLayoutAtomQdO{},
+ make_shape(shape<0>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int<kStages_dO>{})));
+
+ using SmemLayoutAtomKV = decltype(
+ composition(Swizzle<kSwizzle, kSwizzleBase, kSwizzleBase>{},
+ // TODO: FA2 has a slightly different layout, does it matter?
+ Layout<Shape<_8, Int<kBlockKGmem>>,
+ Stride<Int<kBlockKGmem>, _1>>{}));
+ using SmemLayoutK = decltype(tile_to_shape(SmemLayoutAtomKV{}, select<1, 2>(TileShape_MNK{})));
+
+ using SmemLayoutV = decltype(tile_to_shape(SmemLayoutAtomKV{}, select<1, 2>(TileShape_MNK{})));
+
+ // TD [2023-03-19]: Idk why kPBlockN = 16 and kSwizzlePdS=3 is the fastest.
+ static constexpr int kPBlockN = kBlockN % 64 == 0 ? 64 : (kBlockN % 32 == 0 ? 32 : 16);
+ static_assert(kPBlockN == 16 || kPBlockN == 32 || kPBlockN == 64);
+ // static constexpr int kSwizzlePdS = kPBlockN == 16 ? 1 : (kPBlockN == 32 ? 2 : 3);
+ static constexpr int kSwizzlePdS = 3;
+ using SmemLayoutAtomPdS = decltype(
+ composition(Swizzle<kSwizzlePdS, kSwizzleBase, kSwizzleBase>{},
+ Layout<Shape<Int<kBlockM>, Int<kPBlockN>>,
+ Stride<Int<kPBlockN>, _1>>{}));
+ using SmemLayoutPdS = decltype(tile_to_shape(
+ SmemLayoutAtomPdS{},
+ make_shape(Int<kBlockM>{}, Int<kBlockN>{})));
+
+ // We set stride to be multiple of 64 so that if ShuffleLSE, even if threads read from sLSE but out of bounds,
+ // it's still a valid smem address.
+ using SmemLayoutLSE = cute::Layout<cute::Shape<Int<kBlockM>, Int<kStages>>, cute::Stride<_1, Int<cute::round_up(kBlockM, 64)>>>;
+ using SmemLayoutLSEMma = std::conditional_t<
+ SdP_swapAB,
+ cute::Layout<cute::Shape<Int<kBlockN>, Int<kBlockM>, Int<kStages>>, cute::Stride<_0, _1, Int<cute::round_up(kBlockM, 64)>>>,
+ cute::Layout<cute::Shape<Int<kBlockM>, Int<kBlockN>, Int<kStages>>, cute::Stride<_1, _0, Int<cute::round_up(kBlockM, 64)>>>
+ >;
+
+ // Note this is the transpose in terms of the view, not in terms of memory.
+ using SmemLayoutQt =
+ decltype(cute::composition(SmemLayoutQ{},
+ make_layout(make_shape(get<2>(TileShape_MNK{}), get<0>(TileShape_MNK{}), Int<kStages>{}),
+ make_stride(Int<kBlockM>{}, _1{}, Int<kBlockM * kHeadDim>{}))));
+ using SmemLayoutdOt =
+ decltype(cute::composition(SmemLayoutdO{},
+ make_layout(make_shape(get<2>(TileShape_MNK{}), get<0>(TileShape_MNK{}), Int<kStages_dO>{}),
+ make_stride(Int<kBlockM>{}, _1{}, Int<kBlockM * kHeadDim>{}))));
+ using SmemLayoutKt =
+ decltype(cute::composition(SmemLayoutK{},
+ make_layout(make_shape(get<2>(TileShape_MNK{}), get<1>(TileShape_MNK{})),
+ make_stride(Int<kBlockN>{}, _1{}))));
+ using SmemLayoutPdSt =
+ decltype(cute::composition(SmemLayoutPdS{},
+ make_layout(make_shape(Int<kBlockN>{}, Int<kBlockM>{}),
+ make_stride(Int<kBlockM>{}, _1{}))));
+
+ // Thread layout, 256 or 384 threads per row
+ using R2SLayoutAtomdQaccum = Layout<Shape<Int<NumMmaThreads>>>;
+ using R2STiledCopydQaccum = decltype(make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementAccum>{}, R2SLayoutAtomdQaccum{},
+ Layout<Shape < _1>>{})); // Val layout, 1 vals per store
+
+ using SmemCopyAtom = Copy_Atom<SM75_U32x4_LDSM_N, Element>;
+ using SmemCopyAtomTransposed = Copy_Atom<SM75_U16x8_LDSM_T, Element>;
+ // For the case where the N dimension of MmaSdP is divisible by 8 but not by 16
+ using SmemCopyAtomHalf = Copy_Atom<SM75_U32x2_LDSM_N, Element>;
+ // For the case where the N dimension of MmadQ is divisible by 8 but not by 16
+ using SmemCopyAtomTransposedHalf = Copy_Atom<SM75_U16x4_LDSM_T, Element>;
+ // If !SdP_swapAB, the accum registers hold P / dS, otherwise they hold Pt / dSt.
+ // If PdS_major is MN, then we need to "transpose" the write.
+ // TODO: check this write
+ using R2SCopyAtomPdS = Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>;
+
+ // We use CACHEGLOBAL instead of CACHEALWAYS for both Q and K/V, since we won't be reading
+ // from the same address by the same threadblock. This is slightly faster.
+ using GmemCopyStruct = std::conditional_t<
+ Has_cp_async,
+ SM80_CP_ASYNC_CACHEGLOBAL_ZFILL<cute::uint128_t>,
+ AutoVectorizingCopyWithAssumedAlignment<128>
+ >;
+ using GmemCopyAtom = Copy_Atom<GmemCopyStruct, Element>;
+
+ static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad;
+ static_assert(NumMmaThreads % kGmemThreadsPerRow == 0, "NumMmaThreads must be a multiple of kGmemThreadsPerRow");
+ using GmemLayoutAtom = Layout<Shape <Int<NumMmaThreads / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+ using GmemTiledCopyQKV = decltype(
+ make_tiled_copy(GmemCopyAtom{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per read
+ using GmemCopyAtomLSE = Copy_Atom<GmemCopyStruct, float>;
+ using GmemLayoutAtomLSE = Layout<Shape<Int<NumMmaThreads>>>;
+ using GmemTiledCopyLSE = decltype(make_tiled_copy(GmemCopyAtomLSE{}, GmemLayoutAtomLSE{},
+ Layout<Shape<_4>>{})); // Val layout, 4 vals per store
+ // So that we don't have to check if we overshot kBlockM when we load Q
+ // static_assert(kBlockM % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0);
+
+ using ShapeQKV = cute::Shape<int32_t, int32_t, int32_t, int32_t>; // (seqlen, d, head, batch)
+ using StrideQKV = cute::Stride<int64_t, _1, int64_t, int64_t>;
+ using ShapeLSE = cute::Shape<int32_t, int32_t, int32_t>; // (seqlen, head, batch)
+ using StrideLSE = cute::Stride<_1, int64_t, int64_t>; // (seqlen, head, batch)
+ using ShapedQaccum = cute::Shape<int32_t, int32_t, int32_t>; // (seqlen_q * d, head, batch)
+ using StridedQaccum = cute::Stride<_1, int64_t, int64_t>;
+
+ // These are tuned for speed. They don't affect correctness.
+ // We have separate iterations with causal masking. Not necessary for hdim 128 but for hdim 64
+ // this helps quite a bit to not have to do causal masking for most of the iterations.
+ // For hdim 192, separating masking iterations results in register spills.
+ // static constexpr bool SeparateMaskingIterations = kHeadDim <= 64;
+ static constexpr bool SeparateMaskingIterations = false;
+ // Do we keep the LSE and dPsum in each thread, or split them across 8 threads that share them and then
+ // shuffle to get the value whenever we need? This can reduce register pressure when SdP_swapAB, where each
+ // thread needs to keep statistics for (kBlockM / 4) rows. If !SdP_swapAB, each thread only needs to keep
+ // statistic for 2 rows.
+ // static constexpr bool ShuffleLSE = SdP_swapAB && kHeadDim <= 64;
+ // static constexpr bool ShuffledPsum = SdP_swapAB && kHeadDim <= 64;
+ static constexpr bool ShuffleLSE = SdP_swapAB && false;
+ static constexpr bool ShuffledPsum = SdP_swapAB && false;
+
+ static constexpr bool Share_QV_Smem = V_in_regs;
+ using SmemP_t = std::conditional_t<Mma_dKV_is_RS, cute::array<Element, 0>, cute::array_aligned<Element, cute::cosize_v<SmemLayoutPdS>>>;
+
+ struct TensorStorageSharedQV : cute::aligned_struct<128> {
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutK>> smem_k;
+ union {
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutV>> smem_v;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutQ>> smem_q;
+ };
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutdO>> smem_do;
+ cute::array_aligned<ElementAccum, cute::cosize_v<SmemLayoutLSE>, 128> smem_lse;
+ cute::array_aligned<ElementAccum, cute::cosize_v<SmemLayoutLSE>, 128> smem_dpsum;
+ SmemP_t smem_p;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutPdS>> smem_ds;
+ };
+
+ struct TensorStorageSeparateQV : cute::aligned_struct<128> {
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutK>> smem_k;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutV>> smem_v;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutQ>> smem_q;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutdO>> smem_do;
+ cute::array_aligned<ElementAccum, cute::cosize_v<SmemLayoutLSE>, 128> smem_lse;
+ cute::array_aligned<ElementAccum, cute::cosize_v<SmemLayoutLSE>, 128> smem_dpsum;
+ SmemP_t smem_p;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutPdS>> smem_ds;
+ };
+
+ using TensorStorage = std::conditional_t<Share_QV_Smem, TensorStorageSharedQV, TensorStorageSeparateQV>;
+
+ // Host side kernel arguments
+ struct Arguments {
+ Element const* const ptr_Q;
+ ShapeQKV const shape_Q;
+ StrideQKV const stride_Q;
+ Element const* const ptr_K;
+ ShapeQKV const shape_K;
+ StrideQKV const stride_K;
+ Element const* const ptr_V;
+ StrideQKV const stride_V;
+ Element const* const ptr_dO;
+ StrideQKV const stride_dO;
+ ElementAccum* const ptr_dQaccum;
+ ShapedQaccum const shape_dQaccum;
+ StridedQaccum const stride_dQaccum;
+ float const* const ptr_LSE_log2;
+ ShapeLSE const shape_LSE;
+ StrideLSE const stride_LSE_log2;
+ float const* const ptr_dPsum;
+ StrideLSE const stride_dPsum;
+ float const softmax_scale;
+ int const window_size_left, window_size_right, sink_token_length;
+ float const softcap_val;
+ int const num_batch;
+ int* const dq_semaphore;
+ int const* const cu_seqlens_q = nullptr;
+ int const* const cu_seqlens_k = nullptr;
+ int const* const seqused_q = nullptr;
+ int const* const seqused_k = nullptr;
+ };
+
+ // Device side kernel params
+ struct Params {
+ Element const* const ptr_Q;
+ ShapeQKV const shape_Q;
+ StrideQKV const stride_Q;
+ Element const* const ptr_K;
+ ShapeQKV const shape_K;
+ StrideQKV const stride_K;
+ Element const* const ptr_V;
+ StrideQKV const stride_V;
+ Element const* const ptr_dO;
+ StrideQKV const stride_dO;
+ ElementAccum* const ptr_dQaccum;
+ ShapedQaccum const shape_dQaccum;
+ StridedQaccum stride_dQaccum;
+ cutlass::FastDivmod qhead_per_khead_divmod;
+ float const* const ptr_LSE_log2;
+ ShapeLSE const shape_LSE;
+ StrideLSE const stride_LSE_log2;
+ float const* const ptr_dPsum;
+ StrideLSE const stride_dPsum;
+ float const softmax_scale, softmax_scale_log2;
+ int const window_size_left, window_size_right, sink_token_length;
+ float const softcap_val;
+ int const num_batch;
+ int *const dq_semaphore;
+ int const *const cu_seqlens_q = nullptr;
+ int const *const cu_seqlens_k = nullptr;
+ int const *const seqused_q = nullptr;
+ int const *const seqused_k = nullptr;
+ };
+
+ static Params
+ to_underlying_arguments(Arguments const& args) {
+ if constexpr (Deterministic) { assert(args.dq_semaphore != nullptr); }
+ // If there's tanh softcapping, we do tanh(scores * softmax_scale / softcap_val) * softcap_val.
+ // Right after this, we multiply by log2(e) before applying exp2.
+ // To reduce the number of instructions, we instead pre-multiply softmax_scale / softcap_val
+ // (assigning it to params.softcap_val) and pre-multiply softcap_val * log2(e)
+ // (assigning it to params.softmax_scale_log2).
+ // In the backward, we need to multiply by
+ // (1 - tanh^2) * softmax_scale / softcap_val * softcap_val = (1 - tanh^2) * softmax_scale.
+ // Instead we multiply by (1 - tanh^2) and multiply dK and dV by params.softmax_scale
+ // (the original softmax_scale) at the end.
+ return {args.ptr_Q, args.shape_Q, args.stride_Q,
+ args.ptr_K, args.shape_K, args.stride_K,
+ args.ptr_V, args.stride_V,
+ args.ptr_dO, args.stride_dO,
+ args.ptr_dQaccum, args.shape_dQaccum, args.stride_dQaccum,
+ cutlass::FastDivmod(cute::ceil_div(get<2>(args.shape_Q), get<2>(args.shape_K))),
+ args.ptr_LSE_log2, args.shape_LSE, args.stride_LSE_log2, args.ptr_dPsum, args.stride_dPsum,
+ args.softmax_scale,
+ !Has_softcap ? float(args.softmax_scale * M_LOG2E) : float(args.softcap_val * M_LOG2E),
+ args.window_size_left, args.window_size_right, args.sink_token_length,
+ !Has_softcap ? 0.f : args.softmax_scale / args.softcap_val,
+ args.num_batch, args.dq_semaphore,
+ args.cu_seqlens_q, args.cu_seqlens_k, args.seqused_q, args.seqused_k};
+ }
+
+ CUTLASS_DEVICE
+ cute::tuple<int, int> get_m_block_min_max(Params const& params, SeqlenInfo_t const& seqlen_info,
+ int n_block, int bidb) {
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ int const seqlen_q = seqlen_info.seqlen_q;
+ int const seqlen_k = seqlen_info.seqlen_k;
+ int m_block_max = cute::ceil_div(seqlen_q, kBlockM);
+ if constexpr (Is_local) {
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ if (n_block >= cute::ceil_div(params.sink_token_length, kBlockN)) {
+ m_block_max = std::min(m_block_max, cute::ceil_div((n_block + 1) * kBlockN + seqlen_q - seqlen_k + params.window_size_left, kBlockM));
+ }
+ }
+ int m_block_min = 0;
+ if constexpr (Is_causal || Is_local) {
+ m_block_min = std::max(m_block_min, (n_block * kBlockN + seqlen_q - seqlen_k - params.window_size_right) / kBlockM);
+ }
+ return {m_block_min, m_block_max};
+ }
+
+ template <typename SharedStorage, typename FrgTensordKV>
+ CUTLASS_DEVICE bool
+ mma(Params const& params,
+ FrgTensordKV& tdKrdK,
+ FrgTensordKV& tdVrdV,
+ int thread_idx,
+ cute::tuple<int32_t, int32_t, int32_t> block_coord,
+ SharedStorage& shared_storage
+ ) {
+ static_assert(is_rmem<FrgTensordKV>::value, "dK and dV tensor must be rmem resident.");
+
+ int n_block = get<0>(block_coord);
+ int bidh = get<1>(block_coord);
+ int bidb = get<2>(block_coord);
+ SeqlenInfo_t seqlen_info{
+ bidb, get<0>(params.shape_Q), size<0>(params.shape_K),
+ params.cu_seqlens_q, params.cu_seqlens_k, params.seqused_q, params.seqused_k
+ };
+ auto m_block_min_max = get_m_block_min_max(params, seqlen_info, n_block, bidb);
+ int const m_block_min = get<0>(m_block_min_max);
+ int const m_block_max = get<1>(m_block_min_max);
+ // It's possible to have m_block_max <= m_block_min. Exit early
+ if constexpr (Is_causal || Is_local || Varlen) {
+ if (m_block_max <= m_block_min) { return false; }
+ }
+
+ Tensor sQ = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), SmemLayoutQ{});
+ Tensor sdO = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), SmemLayoutdO{});
+ Tensor sK = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), SmemLayoutK{});
+ Tensor sV = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutV{});
+ Tensor sQt = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), SmemLayoutQt{});
+ Tensor sdOt = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), SmemLayoutdOt{});
+ Tensor sKt = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), SmemLayoutKt{});
+ Tensor sP = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), SmemLayoutPdS{});
+ Tensor sPt = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), SmemLayoutPdSt{});
+ Tensor sdS = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), SmemLayoutPdS{});
+ Tensor sdSt = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), SmemLayoutPdSt{});
+ Tensor sLSE = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_lse.data()), SmemLayoutLSE{});
+ Tensor sdPsum = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_dpsum.data()), SmemLayoutLSE{});
+ Tensor sLSEMma = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_lse.data()), SmemLayoutLSEMma{});
+ Tensor sdPsumMma = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_dpsum.data()), SmemLayoutLSEMma{});
+
+ bool const is_varlen_q = Varlen && params.cu_seqlens_q;
+ bool const is_varlen_k = Varlen && params.cu_seqlens_k;
+ int bidh_kv = params.qhead_per_khead_divmod.divide(bidh);
+ Tensor mQ = make_tensor(make_gmem_ptr(params.ptr_Q), params.shape_Q, params.stride_Q)(_, _, bidh, !is_varlen_q ? bidb : 0);
+ Tensor mdO = make_tensor(make_gmem_ptr(params.ptr_dO), params.shape_Q, params.stride_dO)(_, _, bidh, !is_varlen_q ? bidb : 0);
+ Tensor mK = make_tensor(make_gmem_ptr(params.ptr_K), params.shape_K, params.stride_K)(_, _, bidh_kv, !is_varlen_k ? bidb : 0);
+ Tensor mV = make_tensor(make_gmem_ptr(params.ptr_V), params.shape_K, params.stride_V)(_, _, bidh_kv, !is_varlen_k ? bidb : 0);
+ Tensor mLSE = make_tensor(make_gmem_ptr(params.ptr_LSE_log2), params.shape_LSE, params.stride_LSE_log2)(_, bidh, !is_varlen_q ? bidb : 0);
+ Tensor mdPsum = make_tensor(make_gmem_ptr(params.ptr_dPsum), params.shape_LSE, params.stride_dPsum)(_, bidh, !is_varlen_q ? bidb : 0);
+ Tensor mdQaccum = make_tensor(make_gmem_ptr(reinterpret_cast<ElementAccum*>(params.ptr_dQaccum)),
+ params.shape_dQaccum, params.stride_dQaccum)(_, bidh, !is_varlen_q ? bidb : 0);
+
+ Tensor gQ = local_tile(domain_offset(make_coord(seqlen_info.offset_q, _0{}), mQ), select<0, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (M, K, _)
+ Tensor gdO = local_tile(domain_offset(make_coord(seqlen_info.offset_q, _0{}), mdO), select<0, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (M, K, _)
+ Tensor gK = local_tile(domain_offset(make_coord(seqlen_info.offset_k, _0{}), mK), select<1, 2>(TileShape_MNK{}), make_coord(n_block, _0{})); // (N, K)
+ Tensor gV = local_tile(domain_offset(make_coord(seqlen_info.offset_k, _0{}), mV), select<1, 2>(TileShape_MNK{}), make_coord(n_block, _0{})); // (N, K)
+ Tensor gLSE = local_tile(domain_offset(make_coord(seqlen_info.offset_q_padded), mLSE), select<0>(TileShape_MNK{}), make_coord(_)); // (M, _)
+ Tensor gdPsum = local_tile(domain_offset(make_coord(seqlen_info.offset_q_padded), mdPsum), select<0>(TileShape_MNK{}), make_coord(_)); // (M, _)
+ Tensor gdQaccum = local_tile(domain_offset(make_coord(seqlen_info.offset_q_padded * kHeadDim), mdQaccum), Shape<Int<kBlockM * kHeadDim>>{}, make_coord(_)); // (M * K, _)
+
+ GmemTiledCopyQKV gmem_tiled_copy_QKV;
+ auto gmem_thr_copy_QKV = gmem_tiled_copy_QKV.get_thread_slice(thread_idx);
+ auto gmem_thr0_copy_QKV = gmem_tiled_copy_QKV.get_thread_slice(_0{}); // For index calculation
+ GmemTiledCopyLSE gmem_tiled_copy_lse;
+ auto gmem_thr_copy_lse = gmem_tiled_copy_lse.get_thread_slice(thread_idx);
+ R2STiledCopydQaccum r2s_tiled_copy_dQaccum;
+ auto r2s_thr_copy_dQaccum = r2s_tiled_copy_dQaccum.get_thread_slice(thread_idx);
+
+ Tensor tQgQ = gmem_thr_copy_QKV.partition_S(gQ);
+ Tensor tQsQ = gmem_thr_copy_QKV.partition_D(sQ);
+ Tensor tdOgdO = gmem_thr_copy_QKV.partition_S(gdO);
+ Tensor tdOsdO = gmem_thr_copy_QKV.partition_D(sdO);
+ Tensor tLSEgLSE = gmem_thr_copy_lse.partition_S(gLSE);
+ Tensor tLSEsLSE = gmem_thr_copy_lse.partition_D(sLSE);
+ Tensor tLSEgdPsum = gmem_thr_copy_lse.partition_S(gdPsum);
+ Tensor tLSEsdPsum = gmem_thr_copy_lse.partition_D(sdPsum);
+ // We can reuse r2s_thr_copy_dQaccum for this partitioning
+ Tensor tdQgdQaccum = r2s_thr_copy_dQaccum.partition_D(gdQaccum);
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print(mdQaccum); printf("\n"); print(gdQaccum_); printf("\n"); print(gdQaccum); printf("\n"); print(tdQgdQaccum); printf("\n"); }
+
+ TiledMmaSdP tiled_mma_SdP;
+ TiledMmadKV tiled_mma_dKV;
+ TiledMmadQ tiled_mma_dQ;
+
+ auto thr_mma_SdP = tiled_mma_SdP.get_thread_slice(thread_idx);
+ auto thr_mma_dKV = tiled_mma_dKV.get_thread_slice(thread_idx);
+ auto thr_mma_dQ = tiled_mma_dQ.get_thread_slice(thread_idx);
+
+ // Allocate "fragments/descriptors"
+ // We have to use the templated mma_partition_fragment_AB instead of cute::conditional_return or lambda,
+ // because some partition_fragment_A/B don't compile.
+ // https://stackoverflow.com/questions/50051473/if-constexpr-in-c17-does-not-work-in-a-non-templated-function
+ Tensor tdPrV = mma_partition_fragment_AB</*A=*/SdP_swapAB>(thr_mma_SdP, sV);
+
+ // Copy Atom retiling
+ auto smem_copy_atom_SdP_B = cute::conditional_return<MmaSdPEvenN>(SmemCopyAtom{}, SmemCopyAtomHalf{});
+ auto smem_tiled_copy_QdO = cute::conditional_return<!SdP_swapAB>(make_tiled_copy_A(SmemCopyAtom{}, tiled_mma_SdP), make_tiled_copy_B(smem_copy_atom_SdP_B, tiled_mma_SdP));
+ auto smem_thr_copy_QdO = smem_tiled_copy_QdO.get_thread_slice(thread_idx);
+ Tensor tSsQ = smem_thr_copy_QdO.partition_S(sQ);
+ Tensor tdPsdO = smem_thr_copy_QdO.partition_S(sdO);
+
+ auto smem_tiled_copy_KV = cute::conditional_return<!SdP_swapAB>(make_tiled_copy_B(smem_copy_atom_SdP_B, tiled_mma_SdP), make_tiled_copy_A(SmemCopyAtom{}, tiled_mma_SdP));
+ auto smem_thr_copy_KV = smem_tiled_copy_KV.get_thread_slice(thread_idx);
+ Tensor tSsK = smem_thr_copy_KV.partition_S(sK);
+ Tensor tdPsV = smem_thr_copy_KV.partition_S(sV);
+
+ auto r2s_tiled_copy_PdS = make_tiled_copy_C(R2SCopyAtomPdS{}, tiled_mma_SdP);
+ auto r2s_thr_copy_PdS = r2s_tiled_copy_PdS.get_thread_slice(thread_idx);
+ Tensor tPsP = r2s_thr_copy_PdS.partition_D(cute::conditional_return<!SdP_swapAB>(sP, sPt)); // ((Atom,AtomNum),PIPE_M,PIPE_N)
+ Tensor tdSsdS = r2s_thr_copy_PdS.partition_D(cute::conditional_return<!SdP_swapAB>(sdS, sdSt)); // ((Atom,AtomNum),PIPE_M,PIPE_N)
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print(r2s_thr_copy_PdS); print(sP); printf("\n"); print(sPt); printf("\n"); print(tPsP); printf("\n"); print(tdSsdS); printf("\n"); }
+
+ auto smem_copy_atom_dKV_B = cute::conditional_return<MmadKVEvenN>(SmemCopyAtomTransposed{}, SmemCopyAtomTransposedHalf{});
+ auto smem_tiled_copy_PdSt = cute::conditional_return<!dKV_swapAB>(make_tiled_copy_A(SmemCopyAtomTransposed{}, tiled_mma_dKV), make_tiled_copy_B(smem_copy_atom_dKV_B, tiled_mma_dKV));
+ auto smem_thr_copy_PdSt = smem_tiled_copy_PdSt.get_thread_slice(thread_idx);
+ Tensor tdVsPt = smem_thr_copy_PdSt.partition_S(sPt);
+ Tensor tdKsdSt = smem_thr_copy_PdSt.partition_S(sdSt);
+
+ auto smem_tiled_copy_QdOt = cute::conditional_return<!dKV_swapAB>(make_tiled_copy_B(smem_copy_atom_dKV_B, tiled_mma_dKV), make_tiled_copy_A(SmemCopyAtomTransposed{}, tiled_mma_dKV));
+ auto smem_thr_copy_QdOt = smem_tiled_copy_QdOt.get_thread_slice(thread_idx);
+ Tensor tdVsdOt = smem_thr_copy_QdOt.partition_S(sdOt);
+ Tensor tdKsQt = smem_thr_copy_QdOt.partition_S(sQt);
+
+ auto smem_tiled_copy_dS = cute::conditional_return<!dQ_swapAB>(
+ make_tiled_copy_A(SmemCopyAtom{}, tiled_mma_dQ),
+ make_tiled_copy_B(cute::conditional_return<MmadQEvenN>(SmemCopyAtom{}, SmemCopyAtomHalf{}), tiled_mma_dQ));
+ auto smem_thr_copy_dS = smem_tiled_copy_dS.get_thread_slice(thread_idx);
+ Tensor tdQsdS = smem_thr_copy_dS.partition_S(sdS);
+
+ auto smem_tiled_copy_Kt = cute::conditional_return<!dQ_swapAB>(
+ make_tiled_copy_B(cute::conditional_return<MmadQEvenN>(SmemCopyAtomTransposed{}, SmemCopyAtomTransposedHalf{}), tiled_mma_dQ),
+ make_tiled_copy_A(SmemCopyAtomTransposed{}, tiled_mma_dQ));
+ auto smem_thr_copy_Kt = smem_tiled_copy_Kt.get_thread_slice(thread_idx);
+ Tensor tdQsKt = smem_thr_copy_Kt.partition_S(sKt);
+
+ // thr_mma_SdP.partition_C(sLSEMma) has shape (MMA=4, MMA_M, MMA_N, PIPE), we only take the col indices
+ // or row indices, depending on whether SdP_swapAB.
+ Tensor tSsLSEMma = logical_divide(thr_mma_SdP.partition_C(sLSEMma), Shape<_2>{}); // (2, 2, MMA_M, MMA_N, PIPE)
+ Tensor tSsLSE = group_modes<0, 2>(cute::conditional_return<!SdP_swapAB>(
+ tSsLSEMma(make_coord(_0{}, _), _, _0{}, _), // (2, MMA_M, PIPE)
+ tSsLSEMma(make_coord(_, _0{}), _0{}, _, _))); // (2, MMA_N, PIPE)
+ Tensor tSsdPsumMma = logical_divide(thr_mma_SdP.partition_C(sdPsumMma), Shape<_2>{});
+ Tensor tSsdPsum = group_modes<0, 2>(cute::conditional_return<!SdP_swapAB>(
+ tSsdPsumMma(make_coord(_0{}, _), _, _0{}, _), // (2, MMA_M, PIPE)
+ tSsdPsumMma(make_coord(_, _0{}), _0{}, _, _))); // (2, MMA_N, PIPE)
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print(sLSEMma); printf("\n"); print(tLSEsLSE); printf("\n"); }
+ // If we want to split the stats among the 8 threads that share the same rows.
+ static constexpr int kStatsPerThread = cute::ceil_div(decltype(size(tSsLSE))::value, 8);
+
+ // Predicates
+ Tensor cQ = cute::make_identity_tensor(select<0, 2>(TileShape_MNK{}));
+ Tensor tQcQ = gmem_thr_copy_QKV.partition_S(cQ);
+ Tensor t0QcQ = gmem_thr0_copy_QKV.partition_S(cQ);
+ Tensor tQpQ = make_tensor<bool>(make_shape(size<2>(tQsQ)));
+ #pragma unroll
+ for (int k = 0; k < size(tQpQ); ++k) { tQpQ(k) = get<1>(tQcQ(_0{}, _0{}, k)) < get<1>(params.shape_Q); }
+ Tensor cLSE = cute::make_identity_tensor(select<0>(TileShape_MNK{}));
+ Tensor tLSEcLSE = gmem_thr_copy_lse.partition_S(cLSE);
+
+ int const seqlen_q = seqlen_info.seqlen_q;
+ int const seqlen_k = seqlen_info.seqlen_k;
+
+ flash::Mask<kBlockM, kBlockN, false /*PackGQA*/, TiledMmaSdP, SdP_swapAB> mask(
+ thread_idx, seqlen_q, seqlen_k, params.window_size_left, params.window_size_right, params.sink_token_length,
+ params.qhead_per_khead_divmod
+ );
+
+ {
+ Tensor tKgK = gmem_thr_copy_QKV.partition_S(gK); // (KCPY, KCPY_N, KCPY_K, nblocksN)
+ Tensor tKsK = gmem_thr_copy_QKV.partition_D(sK);
+ Tensor tVgV = gmem_thr_copy_QKV.partition_S(gV); // (VCPY, VCPY_N, VCPY_K, nblocksN)
+ Tensor tVsV = gmem_thr_copy_QKV.partition_D(sV);
+ // Predicates
+ Tensor cKV = cute::make_identity_tensor(select<1, 2>(TileShape_MNK{}));
+ Tensor tKVcKV = gmem_thr_copy_QKV.partition_S(cKV);
+ Tensor t0KVcKV = gmem_thr0_copy_QKV.partition_S(cKV);
+ Tensor tKVpKV = make_tensor<bool>(make_shape(size<2>(tKsK)));
+ #pragma unroll
+ for (int k = 0; k < size(tKVpKV); ++k) { tKVpKV(k) = get<1>(tKVcKV(_0{}, _0{}, k)) < get<1>(params.shape_K); }
+ // Do we need bound check to make sure the row doesn't go above kBlockN
+ static constexpr bool EvenN = kBlockN % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0;
+ // static_assert(EvenN); // It simplifies the loading of K and V
+ // Instead of passing in tKVcKV, we pass in t0KVcKV and subtract the offset from the limit
+ // (seqlen_k - n_block * kBlockN). This is because the entries of t0KVcKV are known at compile time.
+ // int const seqlenk_row_limit = -int(get<0>(tKVcKV(_0{}, _0{}, _0{}))) + (EvenN
+ // ? seqlen_info.seqlen_k - n_block * kBlockN
+ // : std::min(seqlen_info.seqlen_k - n_block * kBlockN, kBlockN));
+ // // Need Clear_OOB_MN to be true here since the gemm will sum over the kBlockN dimension
+ // flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/true, /*Clear_OOB_K=*/true>(
+ // gmem_tiled_copy_QKV, tVgV, tVsV, t0KVcKV, tKVpKV, seqlenk_row_limit);
+ int const seqlenk_row_limit = seqlen_k - n_block * kBlockN - get<0>(tKVcKV(_0{}, _0{}, _0{}));
+ #pragma unroll
+ for (int m = 0; m < size<1>(tVsV); ++m) {
+ // If kBlockN doesn't evenly divide the tiled copy, only the last `m` needs to be checked
+ if (EvenN || m < size<1>(tVsV) - 1 || get<0>(tKVcKV(_0{}, m, _0{})) < kBlockN) {
+ bool const predicate_n = get<0>(t0KVcKV(_0{}, m, _0{})) < seqlenk_row_limit;
+ #pragma unroll
+ for (int k = 0; k < size<2>(tVsV); ++k) {
+ cute::copy(gmem_tiled_copy_QKV.with(tKVpKV(k) && predicate_n), tVgV(_, m, k), tVsV(_, m, k));
+ }
+ }
+ }
+ if constexpr (V_in_regs) { flash::cp_async_fence(); }
+ // flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/true, /*Clear_OOB_K=*/true>(
+ // gmem_tiled_copy_QKV, tKgK, tKsK, t0KVcKV, tKVpKV, seqlenk_row_limit);
+ #pragma unroll
+ for (int m = 0; m < size<1>(tKsK); ++m) {
+ if (EvenN || m < size<1>(tKsK) - 1 || get<0>(tKVcKV(_0{}, m, _0{})) < kBlockN) {
+ bool const predicate_n = get<0>(t0KVcKV(_0{}, m, _0{})) < seqlenk_row_limit;
+ #pragma unroll
+ for (int k = 0; k < size<2>(tKsK); ++k) {
+ cute::copy(gmem_tiled_copy_QKV.with(tKVpKV(k) && predicate_n), tKgK(_, m, k), tKsK(_, m, k));
+ }
+ }
+ }
+ flash::cp_async_fence();
+ }
+
+ if constexpr (V_in_regs) {
+ flash::cp_async_wait<1>();
+ __syncthreads();
+ Tensor tdPrV_copy_view = smem_thr_copy_KV.retile_D(tdPrV);
+ Tensor tdPsV_copy_view = smem_thr_copy_KV.partition_S(sV);
+ cute::copy(smem_tiled_copy_KV, tdPsV_copy_view, tdPrV_copy_view);
+ __syncthreads(); // Sync to avoid loading Q to smem_q, which overlaps with smem_v
+ }
+
+ // Do we need bound check to make sure the row doesn't go above kBlockM
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr bool EvenM = kBlockM % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0;
+
+ auto load_Q_LSE = [&] (int const m_block, int const smem_pipe_write) {
+ // if (cute::thread0()) { printf("Inside load_Q_LSE, m_block = %d, smem_pipe_write = %d\n", m_block, smem_pipe_write); }
+ Tensor tQsQ_cur = tQsQ(_, _, _, smem_pipe_write);
+ Tensor tQgQ_cur = tQgQ(_, _, _, m_block);
+ // Instead of passing in tQcQ, we pass in t0QcQ and subtract the offset from the limit
+ // (seqlen_q - m_block * kBlockM). This is because the entries of t0QcQ are known at compile time.
+ // int const seqlenq_row_limit = -int(get<0>(tQcQ(_0{}, _0{}, _0{}))) + (EvenM
+ // ? seqlen_info.seqlen_q - m_block * kBlockM
+ // : std::min(seqlen_info.seqlen_q - m_block * kBlockM, kBlockM));
+ // Need Clear_OOB_MN to be true here since the gemm will sum over the kBlockM dimension
+ // flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/true, /*Clear_OOB_K=*/true>(
+ // gmem_tiled_copy_QKV, tQgQ(_, _, _, m_block), tQsQ_cur, t0QcQ, tQpQ, seqlenq_row_limit);
+ int const seqlenq_row_limit = seqlen_info.seqlen_q - m_block * kBlockM - get<0>(tQcQ(_0{}, _0{}, _0{}));
+ #pragma unroll
+ for (int m = 0; m < size<1>(tQsQ); ++m) {
+ // If kBlockM doesn't evenly divide the tiled copy, only the last `m` needs to be checked
+ if (EvenM || m < size<1>(tQsQ) - 1 || get<0>(tQcQ(_0{}, m, _0{})) < kBlockM) {
+ bool const predicate_m = get<0>(t0QcQ(_0{}, m, _0{})) < seqlenq_row_limit;
+ #pragma unroll
+ for (int k = 0; k < size<2>(tQsQ); ++k) {
+ cute::copy(gmem_tiled_copy_QKV.with(tQpQ(k) && predicate_m), tQgQ_cur(_, m, k), tQsQ_cur(_, m, k));
+ }
+ }
+ }
+ Tensor tLSEgLSE_cur = tLSEgLSE(_, _, m_block);
+ Tensor tLSEsLSE_cur = tLSEsLSE(_, _, smem_pipe_write);
+ // We made sure LSE length is padded so we read `kBlockM` elements so that all
+ // elements in sLSE are filled. Without this we might have uninitialized sLSE values.
+ #pragma unroll
+ for (int m = 0; m < size<1>(tLSEsLSE); ++m) {
+ if (get<0>(tLSEcLSE(_0{}, m)) < kBlockM) {
+ cute::copy(gmem_tiled_copy_lse, tLSEgLSE_cur(_, m), tLSEsLSE_cur(_, m));
+ }
+ }
+ };
+
+ auto load_dO_dPsum = [&] (int const m_block, int const smem_pipe_write) {
+ // if (cute::thread0()) { printf("Inside load_dO_dPsum, m_block = %d, smem_pipe_write = %d\n", m_block, smem_pipe_write); }
+ Tensor tdOsdO_cur = tdOsdO(_, _, _, smem_pipe_write);
+ Tensor tdOgdO_cur = tdOgdO(_, _, _, m_block);
+ // int const seqlenq_row_limit = -int(get<0>(tQcQ(_0{}, _0{}, _0{}))) + (EvenM
+ // ? seqlen_info.seqlen_q - m_block * kBlockM
+ // : std::min(seqlen_info.seqlen_q - m_block * kBlockM, kBlockM));
+ // flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/true, /*Clear_OOB_K=*/true>(
+ // gmem_tiled_copy_QKV, tdOgdO(_, _, _, m_block), tdOsdO_cur, t0QcQ, tQpQ, seqlenq_row_limit);
+ int const seqlenq_row_limit = seqlen_info.seqlen_q - m_block * kBlockM - get<0>(tQcQ(_0{}, _0{}, _0{}));
+ #pragma unroll
+ for (int m = 0; m < size<1>(tdOsdO); ++m) {
+ // If kBlockM doesn't evenly divide the tiled copy, only the last `m` needs to be checked
+ if (EvenM || m < size<1>(tdOsdO) - 1 || get<0>(tQcQ(_0{}, m, _0{})) < kBlockM) {
+ bool const predicate_m = get<0>(t0QcQ(_0{}, m, _0{})) < seqlenq_row_limit;
+ #pragma unroll
+ for (int k = 0; k < size<2>(tdOsdO); ++k) {
+ cute::copy(gmem_tiled_copy_QKV.with(tQpQ(k) && predicate_m), tdOgdO_cur(_, m, k), tdOsdO_cur(_, m, k));
+ }
+ }
+ }
+ Tensor tLSEgdPsum_cur = tLSEgdPsum(_, _, m_block);
+ Tensor tLSEsdPsum_cur = tLSEsdPsum(_, _, smem_pipe_write);
+ #pragma unroll
+ for (int m = 0; m < size<1>(tLSEsdPsum); ++m) {
+ if (get<0>(tLSEcLSE(_0{}, m)) < kBlockM) {
+ cute::copy(gmem_tiled_copy_lse, tLSEgdPsum_cur(_, m), tLSEsdPsum_cur(_, m));
+ }
+ }
+ };
+
+ int m_block = m_block_min;
+
+ // Note, using the for_each() function here to ensure `stage` is of type Int<x>.
+ for_each(make_int_sequence<kStages>{}, [&] (auto stage) {
+ static constexpr bool Is_first_stage = CUTE_STATIC_V(stage) == 0;
+ static constexpr bool Is_last_stage = CUTE_STATIC_V(stage) == kStages - 1;
+ if constexpr (!Is_last_stage || kStages == 1) {
+ if (Is_first_stage || m_block + stage < m_block_max) {
+ load_Q_LSE(m_block + stage, stage);
+ }
+ }
+ // We want the fence outside the if statement to have a fixed number of cp.async commits.
+ // so that we can wait with the correct number of outstanding commits.
+ cute::cp_async_fence();
+ if constexpr (stage < kStages_dO) {
+ if (Is_first_stage || m_block + stage < m_block_max) {
+ load_dO_dPsum(m_block + stage, stage);
+ }
+ cute::cp_async_fence();
+ }
+ });
+
+ int smem_pipe_read = 0, smem_pipe_read_do = 0, smem_pipe_write = kStages - 1, smem_pipe_write_do = 0;
+
+ auto load_Q_next = [&] {
+ // if (cute::thread0()) { printf("m_block = %d, m_block_max = %d, smem_pipe_write = %d\n", m_block, m_block_max, smem_pipe_write); }
+ if (m_block + (kStages > 1 ? kStages - 1 : 1) < m_block_max) {
+ load_Q_LSE(m_block + (kStages > 1 ? kStages - 1 : 1), kStages > 1 ? smem_pipe_write : 0);
+ }
+ cute::cp_async_fence();
+ };
+
+ auto load_dO_next = [&] {
+ // int smem_pipe_write_do_cur = Q_dO_same_stages ? smem_pipe_write : smem_pipe_write_do;
+ if (m_block + kStages_dO < m_block_max) {
+ // load_dO_dPsum(m_block + kStages_dO, kStages_dO > 1 ? smem_pipe_write_do_cur : 0);
+ load_dO_dPsum(m_block + kStages_dO, kStages_dO > 1 ? smem_pipe_write_do : 0);
+ }
+ cute::cp_async_fence();
+ };
+
+ clear(tdKrdK);
+ clear(tdVrdV);
+
+ auto bwd_step = [&](int m_block, auto mask_fn) {
+ Tensor tSrS = partition_fragment_C(tiled_mma_SdP, select<!SdP_swapAB ? 0 : 1, !SdP_swapAB ? 1 : 0>(TileShape_MNK{}));
+ clear(tSrS);
+ flash::cp_async_wait<(kStages > 1) ? 1 : 0>();
+ __syncthreads();
+ Tensor tSrQ = mma_partition_fragment_AB</*A=*/!SdP_swapAB>(thr_mma_SdP, sQ(_, _, _0{}));
+ Tensor tSrK = mma_partition_fragment_AB</*A=*/SdP_swapAB>(thr_mma_SdP, sK);
+ // if (cute::thread0()) { print(tiled_mma_SdP); print(tSrS); printf("\n"); print(tSrQ); printf("\n"); print(tSrK); printf("\n"); print(tSsQ); printf("\n"); print(tSsK); printf("\n"); }
+ flash::gemm_sm80<false /*A_in_regs*/, false /*B_in_regs*/, SdP_swapAB>(
+ tSrS, tSrQ, tSrK, tSsQ(_, _, _, kStages > 1 ? smem_pipe_read : 0), tSsK,
+ tiled_mma_SdP, smem_tiled_copy_QdO, smem_tiled_copy_KV, smem_thr_copy_QdO, smem_thr_copy_KV, nullptr /*hook*/);
+ Tensor tLSErLSE = cute::conditional_return<!ShuffleLSE>(make_fragment_like(tSsLSE(_, _0{})), make_tensor<ElementAccum>(Int<kStatsPerThread>{}));
+ if constexpr (!ShuffleLSE) {
+ cute::copy(tSsLSE(_, kStages > 1 ? smem_pipe_read : 0), tLSErLSE);
+ } else {
+ #pragma unroll
+ for (int i = 0; i < kStatsPerThread; ++i) {
+ // It's ok to read OOB, since we made sure sLSE is large enough and we won't use the OOB values
+ tLSErLSE(i) = tSsLSE((thread_idx % 32) / 4 + i * 8, kStages > 1 ? smem_pipe_read : 0);
+ }
+ }
+ if constexpr (Has_softcap) { flash::apply_softcap(tSrS, params.softcap_val); }
+
+ // Reshape tSrS from (4, MMA_N, MMA_M) to (nrow=(2, MMA_M), ncol=(2, MMA_N))
+ Tensor scores = make_tensor(tSrS.data(), flash::convert_layout_acc_rowcol</*Transposed=*/SdP_swapAB>(tSrS.layout()));
+ // dtanh needs to happen before masking, otherwise we get 1 - (-inf)^2 = NaN in the dtanh
+ // if (cute::thread0()) { print_tensor(scores); }
+ auto dtanh = [&] { if constexpr (Has_softcap) return flash::calculate_dtanh(scores); else return nullptr; }();
+ mask_fn(tSrS, m_block);
+ #pragma unroll
+ for (int mi = 0; mi < size<0>(scores); ++mi) {
+ float const lse_scaled = [&] {
+ if constexpr (!ShuffleLSE) return tLSErLSE(mi);
+ else return __shfl_sync(0xffffffff, tLSErLSE(mi / 8), (mi % 8) * 4 + (thread_idx % 4));
+ }();
+ #pragma unroll
+ for (int ni = 0; ni < size<1>(scores); ++ni) {
+ scores(mi, ni) = exp2f(scores(mi, ni) * params.softmax_scale_log2 - lse_scaled);
+ }
+ }
+
+ Tensor tdPrdP = partition_fragment_C(tiled_mma_SdP, select<!SdP_swapAB ? 0 : 1, !SdP_swapAB ? 1 : 0>(TileShape_MNK{}));
+ clear(tdPrdP);
+ int smem_pipe_read_do_cur = Q_dO_same_stages ? smem_pipe_read : smem_pipe_read_do;
+ flash::cp_async_wait<(kStages_dO > 1) ? 1 : 0>();
+ __syncthreads();
+ auto hook = cute::conditional_return<(kStages > 1)>(load_Q_next, nullptr);
+ Tensor tdPrdO = mma_partition_fragment_AB</*A=*/!SdP_swapAB>(thr_mma_SdP, sdO(_, _, _0{}));
+ Tensor tdPrV_cur = cute::conditional_return<V_in_regs>(tdPrV, mma_partition_fragment_AB</*A=*/SdP_swapAB>(thr_mma_SdP, sV));
+ flash::gemm_sm80<false /*A_in_regs*/, V_in_regs, SdP_swapAB>(
+ tdPrdP, tdPrdO, tdPrV_cur, tdPsdO(_, _, _, kStages_dO > 1 ? smem_pipe_read_do_cur : 0), tdPsV,
+ tiled_mma_SdP, smem_tiled_copy_QdO, smem_tiled_copy_KV, smem_thr_copy_QdO, smem_thr_copy_KV, hook);
+ Tensor tLSErdPsum = cute::conditional_return<!ShuffledPsum>(make_fragment_like(tSsdPsum(_, _0{})), make_tensor<ElementAccum>(Int<kStatsPerThread>{}));
+ if constexpr (!ShuffledPsum) {
+ cute::copy(tSsdPsum(_, kStages_dO > 1 ? smem_pipe_read_do_cur : 0), tLSErdPsum);
+ } else {
+ #pragma unroll
+ for (int i = 0; i < kStatsPerThread; ++i) {
+ tLSErdPsum(i) = tSsdPsum((thread_idx % 32) / 4 + i * 8, kStages_dO > 1 ? smem_pipe_read_do_cur : 0);
+ }
+ }
+
+ // Reshape tdPrdP from (4, MMA_N, MMA_M) to (nrow=(2, MMA_M), ncol=(2, MMA_N))
+ Tensor dS = make_tensor(tdPrdP.data(), scores.layout());
+ #pragma unroll
+ for (int mi = 0; mi < size<0>(dS); ++mi) {
+ float const dP_sum_cur = [&] {
+ if constexpr (!ShuffledPsum) return tLSErdPsum(mi);
+ else return __shfl_sync(0xffffffff, tLSErdPsum(mi / 8), (mi % 8) * 4 + (thread_idx % 4));
+ }();
+ #pragma unroll
+ for (int ni = 0; ni < size<1>(dS); ++ni) {
+ dS(mi, ni) = scores(mi, ni) * (dS(mi, ni) - dP_sum_cur);
+ if constexpr (Has_softcap) { dS(mi, ni) *= dtanh(mi, ni); }
+ }
+ }
+ // if (cute::thread0()) { print_tensor(dS); }
+
+ // Convert scores from fp32 to fp16/bf16
+ Tensor rP = make_tensor_like<Element>(tSrS);
+ flash::convert_type_out(tSrS, rP);
+ if constexpr (!Mma_dKV_is_RS) {
+ Tensor tPaP = r2s_thr_copy_PdS.retile_S(rP); // ((Atom,AtomNum), MMA_N, MMA_N)
+ cute::copy(r2s_tiled_copy_PdS, tPaP, tPsP);
+ }
+ Tensor rdS = make_tensor_like<Element>(tdPrdP);
+ flash::convert_type_out(tdPrdP, rdS);
+ if constexpr (!Mma_dKV_is_RS) { __syncthreads(); } // Make sure P is written
+ // For hdim 64, It's faster to write to smem_dS first before the dV gemm
+ Tensor tdSadS = r2s_thr_copy_PdS.retile_S(rdS); // ((Atom,AtomNum), MMA_N, MMA_N)
+ cute::copy(r2s_tiled_copy_PdS, tdSadS, tdSsdS);
+
+ Tensor tdVrdO = mma_partition_fragment_AB</*A=*/dKV_swapAB>(thr_mma_dKV, sdOt(_, _, _0{}));
+ Tensor tdVsdO_cur = tdVsdOt(_, _, _, kStages_dO > 1 ? smem_pipe_read_do_cur : 0);
+ if constexpr (Mma_dKV_is_RS) {
+ Tensor tdVrP = make_tensor(rP.data(), convert_layout_acc_Aregs<TiledMmadKV>(tSrS.layout()));
+ flash::gemm_rs_sm80(tdVrdV, tdVrP, tdVrdO, tdVsdO_cur, tiled_mma_dKV, smem_tiled_copy_QdOt, smem_thr_copy_QdOt);
+ } else {
+ Tensor tdVrP = mma_partition_fragment_AB</*A=*/!dKV_swapAB>(thr_mma_dKV, sPt);
+ flash::gemm_sm80<false /*A_in_regs*/, false /*B_in_regs*/, /*SwapAB=*/dKV_swapAB>(
+ tdVrdV, tdVrP, tdVrdO, tdVsPt, tdVsdO_cur,
+ tiled_mma_dKV, smem_tiled_copy_PdSt, smem_tiled_copy_QdOt, smem_thr_copy_PdSt, smem_thr_copy_QdOt, nullptr);
+ }
+ // if (cute::thread0()) { print_tensor(tdVrdV); }
+ __syncthreads(); // make sure sdS is written
+ auto do_mma_dQ = [&] (auto hook) {
+ Tensor tdQrdQ = partition_fragment_C(tiled_mma_dQ, select<!dQ_swapAB ? 0 : 2, !dQ_swapAB ? 2 : 0>(TileShape_MNK{}));
+ clear(tdQrdQ);
+ Tensor tdQrdS = mma_partition_fragment_AB</*A=*/!dQ_swapAB>(thr_mma_dQ, sdS);
+ Tensor tdQrK = mma_partition_fragment_AB</*A=*/dQ_swapAB>(thr_mma_dQ, sKt);
+ flash::gemm_sm80<false /*A_in_regs*/, false /*B_in_regs*/, /*SwapAB=*/dQ_swapAB>(
+ tdQrdQ, tdQrdS, tdQrK, tdQsdS, tdQsKt, tiled_mma_dQ,
+ // smem_tiled_copy_dS, smem_tiled_copy_Kt, smem_thr_copy_dS, smem_thr_copy_Kt, load_dO_next);
+ smem_tiled_copy_dS, smem_tiled_copy_Kt, smem_thr_copy_dS, smem_thr_copy_Kt, hook);
+ // if (cute::thread0()) { print_tensor(tdQrdQ); }
+ // We can reuse r2s_thr_copy_dQaccum for this partitioning
+ Tensor tdQrdQ_atomic = r2s_thr_copy_dQaccum.retile_S(tdQrdQ);
+ Tensor tdQgdQaccum_atomic = tdQgdQaccum(_, _, m_block);
+ static_assert(CUTE_STATIC_V(size(tdQrdQ_atomic)) == CUTE_STATIC_V(size(tdQgdQaccum_atomic)));
+ #pragma unroll
+ for (int i = 0; i < size(tdQrdQ_atomic); ++i) { atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); }
+ };
+ // If kStages == 1, we want to do Mma_dK first so we can start loading Q for the next iteration
+ if constexpr (kStages > 1) { do_mma_dQ(load_dO_next); }
+ Tensor tdKrQ = mma_partition_fragment_AB</*A=*/dKV_swapAB>(thr_mma_dKV, sQt(_, _, _0{}));
+ Tensor tdKsQ_cur = tdKsQt(_, _, _, kStages > 1 ? smem_pipe_read : 0);
+ if constexpr (Mma_dKV_is_RS) {
+ Tensor tdKrdS = make_tensor(rdS.data(), convert_layout_acc_Aregs<TiledMmadKV>(tdPrdP.layout()));
+ flash::gemm_rs_sm80(tdKrdK, tdKrdS, tdKrQ, tdKsQ_cur, tiled_mma_dKV, smem_tiled_copy_QdOt, smem_thr_copy_QdOt);
+ } else {
+ Tensor tdKrdS = mma_partition_fragment_AB</*A=*/!dKV_swapAB>(thr_mma_dKV, sdSt);
+ flash::gemm_sm80<false /*A_in_regs*/, false /*B_in_regs*/, /*SwapAB=*/dKV_swapAB>(
+ tdKrdK, tdKrdS, tdKrQ, tdKsdSt, tdKsQ_cur,
+ tiled_mma_dKV, smem_tiled_copy_PdSt, smem_tiled_copy_QdOt, smem_thr_copy_PdSt, smem_thr_copy_QdOt, cute::conditional_return<(kStages > 1)>(nullptr, load_dO_next));
+ }
+ if constexpr (kStages == 1) {
+ __syncthreads();
+ do_mma_dQ(load_Q_next);
+ }
+ // if (cute::thread0()) { print_tensor(tdKrdK); }
+
+ smem_pipe_read = smem_pipe_read < kStages - 1 ? smem_pipe_read + 1 : 0;
+ smem_pipe_read_do = smem_pipe_read_do < kStages_dO - 1 ? smem_pipe_read_do + 1 : 0;
+ smem_pipe_write = smem_pipe_write < kStages - 1 ? smem_pipe_write + 1 : 0;
+ smem_pipe_write_do = smem_pipe_write_do < kStages_dO - 1 ? smem_pipe_write_do + 1 : 0;
+
+ };
+
+ // We have separate iterations with causal masking. Not necessary for hdim 128 but for hdim 64
+ // this helps quite a bit to not have to do causal masking for most of the iterations.
+ if constexpr ((Is_causal || Is_local) && SeparateMaskingIterations) {
+ auto mask_fn = [&](auto& tSrS, int m_block) { mask.template apply<true /*Seqlenk_mask*/, Is_causal, Is_local>(tSrS, m_block, n_block); };
+ int const m_block_masking_max = ((n_block + 1) * kBlockN - 1 + seqlen_q - seqlen_k - params.window_size_right) / kBlockM + 1;
+ CUTLASS_PRAGMA_NO_UNROLL
+ for (; m_block < std::min(m_block_max, m_block_masking_max); ++m_block) {
+ bwd_step(m_block, mask_fn);
+ }
+ }
+
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ int const m_block_max_before_local_mask = !Is_local || !SeparateMaskingIterations
+ ? m_block_max
+ : std::min(m_block_max, (n_block * kBlockN + seqlen_q - seqlen_k + params.window_size_left) / kBlockM);
+
+ auto mask_fn = [&](auto& tSrS, int m_block) { mask.template apply<true /*Seqlenk_mask*/, Is_causal && !SeparateMaskingIterations, Is_local && !SeparateMaskingIterations>(tSrS, m_block, n_block); };
+ CUTLASS_PRAGMA_NO_UNROLL
+ for (; m_block < m_block_max_before_local_mask; ++m_block) {
+ bwd_step(m_block, mask_fn);
+ }
+
+ if constexpr (Is_local && SeparateMaskingIterations) {
+ auto mask_fn = [&](auto& tSrS, int m_block) { mask.template apply<true /*Seqlenk_mask*/, false /*Causal_mask*/, Is_local>(tSrS, m_block, n_block); };
+ CUTLASS_PRAGMA_NO_UNROLL
+ for (; m_block < m_block_max; ++m_block) {
+ bwd_step(m_block, mask_fn);
+ }
+ }
+
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(tdVrdV); }
+ #pragma unroll
+ for (int i = 0; i < size(tdKrdK); ++i) { tdKrdK(i) *= params.softmax_scale; }
+
+ return true;
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/mainloop_bwd_sm90_tma_gmma_ws.hpp b/src/solarwm/backends/minimax_h3/hopper_src/mainloop_bwd_sm90_tma_gmma_ws.hpp
new file mode 100644
index 0000000..393a6e5
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/mainloop_bwd_sm90_tma_gmma_ws.hpp
@@ -0,0 +1,1040 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cutlass/cutlass.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+#include <cutlass/numeric_conversion.h>
+#include <cutlass/barrier.h>
+#include "cutlass/pipeline/pipeline.hpp"
+
+#include "cute/tensor.hpp"
+
+#include "cutlass/gemm/collective/builders/sm90_common.inl"
+
+#include "named_barrier.hpp"
+#include "seqlen.h"
+#include "mask.h"
+#include "softmax.h"
+#include "utils.h"
+#include "copy_sm90_bulk_reduce.hpp"
+
+namespace flash {
+
+using namespace cute;
+
+template <int Stages, int Stages_dO, int Stages_dS, class ClusterShape_, class TileShape_MNK_, class Element_, class ElementAccum_, class ArchTag_,
+ bool Is_causal_, bool Is_local_, bool Has_softcap_, bool Varlen_, bool Deterministic,
+ bool SdP_swapAB_, bool dKV_swapAB_, bool dQ_swapAB_,
+ int NumMmaWarpGroups=2, int AtomLayoutMSdP=1, int AtomLayoutNdKV=2, int AtomLayoutMdQ=1,
+ bool Mma_dP_is_RS=false>
+struct CollectiveMainloopBwdSm90 {
+
+ static constexpr int kStages = Stages;
+ static constexpr int kStages_dO = Stages_dO;
+ static constexpr int kStages_dS = Stages_dS;
+ static_assert(kStages >= kStages_dO);
+ static_assert(Stages_dS == 1 || Stages_dS == kStages);
+ static_assert(!Mma_dP_is_RS || SdP_swapAB_); // If Mma_dP_is_RS, we need SdP_SwapAB
+ using ClusterShape = ClusterShape_;
+ using TileShape_MNK = TileShape_MNK_;
+ using Element = Element_;
+ using ElementAccum = ElementAccum_;
+ using ArchTag = ArchTag_;
+ static constexpr bool Is_causal = Is_causal_;
+ static constexpr bool Is_local = Is_local_;
+ static constexpr bool Has_softcap = Has_softcap_;
+ static constexpr bool Varlen = Varlen_;
+ using SeqlenInfo_t = flash::SeqlenInfoQK<Varlen, CUTE_STATIC_V(get<0>(TileShape_MNK{}))>;
+
+ static constexpr bool SdP_swapAB = SdP_swapAB_;
+ static constexpr bool dKV_swapAB = dKV_swapAB_;
+ static constexpr bool dQ_swapAB = dQ_swapAB_;
+
+ static constexpr bool Q_dO_same_stages = kStages == kStages_dO;
+
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ static constexpr int kHeadDim = get<2>(TileShape_MNK{});
+
+ static_assert(ArchTag::kMinComputeCapability >= 90);
+ static_assert(get<0>(ClusterShape{}) == 1 && get<2>(ClusterShape{}) == 1);
+
+ static constexpr int NumMmaThreads = NumMmaWarpGroups * cutlass::NumThreadsPerWarpGroup;
+ static constexpr int NumProducerThreads = cutlass::NumThreadsPerWarp * 2;
+
+ static_assert(NumMmaWarpGroups % AtomLayoutMSdP == 0);
+ static_assert(NumMmaWarpGroups % AtomLayoutNdKV == 0);
+ static_assert(NumMmaWarpGroups % AtomLayoutMdQ == 0);
+ static constexpr bool Mma_dKV_is_RS = AtomLayoutMSdP == 1 && AtomLayoutNdKV == NumMmaWarpGroups && SdP_swapAB && !dKV_swapAB;
+ static constexpr bool Mma_dQ_is_RS = AtomLayoutMSdP == NumMmaWarpGroups && AtomLayoutMdQ == NumMmaWarpGroups && !SdP_swapAB && !dQ_swapAB; // If dQ_swapAB we can't use RS
+
+ static constexpr GMMA::Major PdS_Major = GMMA::Major::K;
+ // static constexpr GMMA::Major PdS_Major = GMMA::Major::MN;
+ static constexpr GMMA::Major PdSt_Major = PdS_Major == GMMA::Major::K ? GMMA::Major::MN : GMMA::Major::K;
+
+ using TileShapeAtomSdP = std::conditional_t<
+ !SdP_swapAB,
+ Shape<Int<kBlockM>, Int<kBlockN / (NumMmaWarpGroups / AtomLayoutMSdP)>, Int<kHeadDim>>,
+ Shape<Int<kBlockN>, Int<kBlockM / AtomLayoutMSdP>, Int<kHeadDim>>
+ >;
+ using AtomLayoutSdP = std::conditional_t<
+ !SdP_swapAB,
+ Layout<Shape<Int<AtomLayoutMSdP>, Int<NumMmaWarpGroups / AtomLayoutMSdP>, _1>>,
+ Layout<Shape<Int<NumMmaWarpGroups / AtomLayoutMSdP>, Int<AtomLayoutMSdP>, _1>>
+ >;
+ using TiledMmaSdP = decltype(cute::make_tiled_mma(
+ cute::GMMA::ss_op_selector<Element, Element, ElementAccum, TileShapeAtomSdP>(),
+ AtomLayoutSdP{}));
+
+ using TiledMmadPRS = decltype(cute::make_tiled_mma(
+ cute::GMMA::rs_op_selector<Element, Element, ElementAccum, TileShapeAtomSdP>(),
+ AtomLayoutSdP{}));
+
+ using TileShapeAtomdKV = std::conditional_t<
+ !dKV_swapAB,
+ Shape<Int<kBlockN>, Int<kHeadDim / (NumMmaWarpGroups / AtomLayoutNdKV)>, Int<kBlockM>>,
+ Shape<Int<kHeadDim>, Int<kBlockN / AtomLayoutNdKV>, Int<kBlockM>>
+ >;
+ using AtomLayoutdKV = std::conditional_t<
+ !dKV_swapAB,
+ Layout<Shape<Int<AtomLayoutNdKV>, Int<NumMmaWarpGroups / AtomLayoutNdKV>, _1>>,
+ Layout<Shape<Int<NumMmaWarpGroups / AtomLayoutNdKV>, Int<AtomLayoutNdKV>, _1>>
+ >;
+ using TiledMmadKV = decltype(cute::make_tiled_mma(
+ std::conditional_t<
+ Mma_dKV_is_RS,
+ decltype(cute::GMMA::rs_op_selector<Element, Element, ElementAccum, TileShapeAtomdKV, GMMA::Major::K, GMMA::Major::MN>()),
+ decltype(cute::GMMA::ss_op_selector<Element, Element, ElementAccum, TileShapeAtomdKV, !dKV_swapAB ? PdSt_Major : GMMA::Major::MN, !dKV_swapAB ? GMMA::Major::MN : PdSt_Major>())
+ >{},
+ AtomLayoutdKV{}));
+
+ using TileShapeAtomdQ = std::conditional_t<
+ !dQ_swapAB,
+ Shape<Int<kBlockM>, Int<kHeadDim / (NumMmaWarpGroups / AtomLayoutMdQ)>, Int<kBlockN>>,
+ Shape<Int<kHeadDim>, Int<kBlockM / AtomLayoutMdQ>, Int<kBlockN>>
+ >;
+ using AtomLayoutdQ = std::conditional_t<
+ !dQ_swapAB,
+ Layout<Shape<Int<AtomLayoutMdQ>, Int<NumMmaWarpGroups / AtomLayoutMdQ>, _1>>,
+ Layout<Shape<Int<NumMmaWarpGroups / AtomLayoutMdQ>, Int<AtomLayoutMdQ>, _1>>
+ >;
+ using TiledMmadQ = decltype(cute::make_tiled_mma(
+ std::conditional_t<
+ Mma_dQ_is_RS,
+ decltype(cute::GMMA::rs_op_selector<Element, Element, ElementAccum, TileShapeAtomdQ, GMMA::Major::K, GMMA::Major::MN>()),
+ decltype(cute::GMMA::ss_op_selector<Element, Element, ElementAccum, TileShapeAtomdQ, !dQ_swapAB ? PdS_Major : GMMA::Major::MN, !dQ_swapAB ? GMMA::Major::MN : PdS_Major>())
+ >{},
+ AtomLayoutdQ{}));
+
+ // We need to accommodate both Q and Q^T (and dO and dO^T) in shared memory.
+ // Q & dO are used in the SdP Mma and Q^T and dO^T are used in the dKV Mma.
+ // Since this is GMMA::Major::K, the M dimension (kBlockM) doesn't matter for the layout, only the K dimension
+ // changes the layout.
+ using SmemLayoutAtomQdO = decltype(cutlass::gemm::collective::detail::ss_smem_selector<GMMA::Major::K, Element,
+ Int<kBlockM>, Int<kHeadDim / (NumMmaWarpGroups / AtomLayoutNdKV)>>()); // for dKV_Mma
+ using SmemLayoutQ =
+ decltype(tile_to_shape(SmemLayoutAtomQdO{},
+ make_shape(shape<0>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int<kStages>{})));
+ using SmemLayoutdO =
+ decltype(tile_to_shape(SmemLayoutAtomQdO{},
+ make_shape(shape<0>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int<kStages_dO>{})));
+
+ using SmemLayoutAtomK = decltype(cutlass::gemm::collective::detail::ss_smem_selector<GMMA::Major::K, Element,
+ Int<kBlockN>, Int<kHeadDim / (NumMmaWarpGroups / AtomLayoutMdQ)>>());
+ using SmemLayoutK = decltype(tile_to_shape(SmemLayoutAtomK{}, select<1, 2>(TileShape_MNK{})));
+
+ using SmemLayoutAtomV = decltype(cutlass::gemm::collective::detail::ss_smem_selector<GMMA::Major::K, Element,
+ decltype(cute::get<1>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{}))>());
+ using SmemLayoutV = decltype(tile_to_shape(SmemLayoutAtomV{}, select<1, 2>(TileShape_MNK{})));
+
+ using SmemLayoutAtomPdS = decltype(cutlass::gemm::collective::detail::ss_smem_selector<PdS_Major, Element,
+ Int<kBlockM / AtomLayoutMSdP>,
+ Int<kBlockN / (NumMmaWarpGroups / AtomLayoutMSdP)>>());
+ using SmemLayoutPdS = decltype(tile_to_shape(
+ SmemLayoutAtomPdS{},
+ make_shape(Int<kBlockM>{}, Int<kBlockN>{}, Int<kStages_dS>{}),
+ std::conditional_t<PdS_Major == GMMA::Major::K, cute::Step<_1, _2, _3>, cute::Step<_2, _1, _3>>{}));
+
+ // Need stride to be multiple of 32, otherwise we get error (misaligned address) when doing TMA if e.g. kBlockM=80
+ // We set stride to be multiple of 64 so that if ShuffleLSE, even if threads read from sLSE but out of bounds,
+ // it's still a valid smem address.
+ using SmemLayoutLSE = cute::Layout<cute::Shape<Int<kBlockM>, Int<kStages>>, cute::Stride<_1, Int<cute::round_up(kBlockM, 64)>>>;
+ using SmemLayoutLSEMma = std::conditional_t<
+ SdP_swapAB,
+ cute::Layout<cute::Shape<Int<kBlockN>, Int<kBlockM>, Int<kStages>>, cute::Stride<_0, _1, Int<cute::round_up(kBlockM, 64)>>>,
+ cute::Layout<cute::Shape<Int<kBlockM>, Int<kBlockN>, Int<kStages>>, cute::Stride<_1, _0, Int<cute::round_up(kBlockM, 64)>>>
+ >;
+
+ // Note this is the transpose in terms of the view, not in terms of memory.
+ using SmemLayoutQt =
+ decltype(cute::composition(SmemLayoutQ{},
+ make_layout(make_shape(get<2>(TileShape_MNK{}), get<0>(TileShape_MNK{}), Int<kStages>{}),
+ make_stride(Int<kBlockM>{}, _1{}, Int<kBlockM * kHeadDim>{}))));
+ using SmemLayoutdOt =
+ decltype(cute::composition(SmemLayoutdO{},
+ make_layout(make_shape(get<2>(TileShape_MNK{}), get<0>(TileShape_MNK{}), Int<kStages_dO>{}),
+ make_stride(Int<kBlockM>{}, _1{}, Int<kBlockM * kHeadDim>{}))));
+ using SmemLayoutKt =
+ decltype(cute::composition(SmemLayoutK{},
+ make_layout(make_shape(get<2>(TileShape_MNK{}), get<1>(TileShape_MNK{})),
+ make_stride(Int<kBlockN>{}, _1{}))));
+ using SmemLayoutPdSt =
+ decltype(cute::composition(SmemLayoutPdS{},
+ make_layout(make_shape(Int<kBlockN>{}, Int<kBlockM>{}, Int<kStages_dS>{}),
+ make_stride(Int<kBlockM>{}, _1{}, Int<kBlockM * kBlockN>{}))));
+
+ // Thread layout, 256 or 384 threads per row
+ // We split into NumMmaWarpGroups so that we can do Bulk reduce add for each WG separately.
+ using R2SLayoutAtomdQaccum = Layout<Shape<Int<cutlass::NumThreadsPerWarpGroup>, Int<NumMmaWarpGroups>>>;
+ using R2STiledCopydQaccum = decltype(make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, ElementAccum>{}, R2SLayoutAtomdQaccum{},
+ Layout<Shape < _4>>{})); // Val layout, 4 vals per store
+ using SmemLayoutdQaccum = Layout<Shape<Int<kBlockM * kHeadDim / NumMmaWarpGroups>, Int<NumMmaWarpGroups>>>;
+
+ static constexpr int kNumPdSStore = kBlockM * kBlockN / NumMmaThreads;
+ // If !SdP_swapAB, the accum registers hold P / dS, otherwise they hold Pt / dSt.
+ // If PdS_major is MN, then we need to "transpose" the write.
+ using SmemCopyAtomPdS = Copy_Atom<
+ std::conditional_t<(!SdP_swapAB) ^ (PdS_Major == GMMA::Major::MN),
+ std::conditional_t<kNumPdSStore % 8 == 0, cute::SM90_U32x4_STSM_N, cute::SM90_U32x2_STSM_N>,
+ std::conditional_t<kNumPdSStore % 8 == 0, cute::SM90_U16x8_STSM_T, cute::SM90_U16x4_STSM_T>
+ >,
+ Element
+ >;
+
+ using GmemTiledCopyQdO = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape{})));
+ using GmemTiledCopyKV = cute::SM90_TMA_LOAD;
+
+ using ShapeQKV = cute::Shape<int32_t, int32_t, int32_t, int32_t>; // (seqlen, d, head, batch)
+ using StrideQKV = cute::Stride<int64_t, _1, int64_t, int64_t>;
+ using ShapeLSE = cute::Shape<int32_t, int32_t, int32_t>; // (seqlen, head, batch)
+ using StrideLSE = cute::Stride<_1, int64_t, int64_t>; // (seqlen, head, batch)
+ using ShapedQaccum = cute::Shape<int32_t, int32_t, int32_t>; // (seqlen_q * d, head, batch)
+ using StridedQaccum = cute::Stride<_1, int64_t, int64_t>;
+
+ using TMA_QdO = decltype(make_tma_copy_A_sm90(
+ GmemTiledCopyQdO{},
+ make_tensor(make_gmem_ptr(static_cast<Element const*>(nullptr)), ShapeQKV{}, StrideQKV{}),
+ take<0, 2>(SmemLayoutQ{}),
+ TileShape_MNK{},
+ ClusterShape{})); // mcast along N mode for this M load, if any
+
+ using TMA_K = decltype(make_tma_copy_B_sm90(
+ GmemTiledCopyKV{},
+ make_tensor(make_gmem_ptr(static_cast<Element const*>(nullptr)), ShapeQKV{}, StrideQKV{}),
+ SmemLayoutK{},
+ TileShape_MNK{},
+ ClusterShape{})); // no mcast for KV
+
+ using TMA_V = decltype(make_tma_copy_B_sm90(
+ GmemTiledCopyKV{},
+ make_tensor(make_gmem_ptr(static_cast<Element const*>(nullptr)), ShapeQKV{}, StrideQKV{}),
+ SmemLayoutV{},
+ TileShape_MNK{},
+ ClusterShape{})); // no mcast for KV
+
+ using MainloopPipeline = typename cutlass::PipelineTmaAsync<kStages>;
+ using PipelineState = typename MainloopPipeline::PipelineState;
+ using MainloopPipeline_dO = typename cutlass::PipelineTmaAsync<kStages_dO>;
+ using PipelineState_dO = typename MainloopPipeline_dO::PipelineState;
+
+ // Set the bytes transferred in this TMA transaction (may involve multiple issues)
+ static constexpr uint32_t TmaTransactionBytesQ = static_cast<uint32_t>(size(take<0, 2>(SmemLayoutQ{})) * cutlass::sizeof_bits_v<Element> / 8);
+ static constexpr uint32_t TmaTransactionBytesK = static_cast<uint32_t>(size(SmemLayoutK{}) * cutlass::sizeof_bits_v<Element> / 8);
+ static constexpr uint32_t TmaTransactionBytesV = static_cast<uint32_t>(size(SmemLayoutV{}) * cutlass::sizeof_bits_v<Element> / 8);
+ static constexpr uint32_t TmaTransactionBytesLSE = static_cast<uint32_t>(size(select<0>(SmemLayoutLSE{})) * cutlass::sizeof_bits_v<ElementAccum> / 8);
+
+ // These are tuned for speed. They don't affect correctness.
+ // We have separate iterations with causal masking. Not necessary for hdim 128 but for hdim 64
+ // this helps quite a bit to not have to do causal masking for most of the iterations.
+ // For hdim 192, separating masking iterations results in register spills.
+ static constexpr bool SeparateMaskingIterations = kHeadDim <= 64;
+ // Do we keep the LSE and dPsum in each thread, or split them across 8 threads that share them and then
+ // shuffle to get the value whenever we need? This can reduce register pressure when SdP_swapAB, where each
+ // thread needs to keep statistics for (kBlockM / 4) rows. If !SdP_swapAB, each thread only needs to keep
+ // statistic for 2 rows.
+ static constexpr bool ShuffleLSE = SdP_swapAB && kHeadDim <= 64;
+ static constexpr bool ShuffledPsum = SdP_swapAB && kHeadDim <= 64;
+ static constexpr bool dQacc_use_TMA = kHeadDim < 256;
+ // For hdim256, we want to slice the dQ MMA (64 x 256 on 2 WGs) into two (64 x 128 on 2 WGs) so that we can
+ // do atomic add on one half before doing the other half of the MMA, to reduce register pressure.
+ static constexpr bool Slice_dQKV_Mma = kHeadDim == 256 && !dQacc_use_TMA && dQ_swapAB && AtomLayoutMdQ == 1 && NumMmaWarpGroups == 2;
+ static_assert(!(Deterministic && Slice_dQKV_Mma), "Deterministic mode not supported with Slice_dQKV_Mma");
+
+ static constexpr size_t SmemAlignmentP = cutlass::detail::alignment_for_swizzle(SmemLayoutPdS{});
+ static constexpr size_t SmemAlignmentdS = cutlass::detail::alignment_for_swizzle(SmemLayoutPdS{});
+ // Without this SmemAlignment, with hdim 256 we get "misaligned address" error in TMA
+ static constexpr size_t SmemAlignmentQKVdO = kHeadDim % 256 == 0 ? 256 : 128;
+ static constexpr size_t SmemAlignmentV = !Mma_dP_is_RS ? SmemAlignmentQKVdO : cutlass::detail::alignment_for_swizzle(SmemLayoutV{});
+ static_assert(SmemAlignmentP >= 128 && SmemAlignmentdS >= 128, "Require at least 128B alignment");
+
+ // TODO: do we have to worry that smem_dk and smem_dv in the epilogue don't line up w smem_k and smem_v due to alignment?
+ using SmemdQacc_t = std::conditional_t<!dQacc_use_TMA, cute::array<ElementAccum, 0>, cute::array_aligned<ElementAccum, cute::cosize_v<SmemLayoutdQaccum>>>;
+ using SmemP_t = std::conditional_t<Mma_dKV_is_RS, cute::array<Element, 0>, cute::array_aligned<Element, cute::cosize_v<SmemLayoutPdS>, SmemAlignmentP>>;
+ struct TensorStorage : cute::aligned_struct<cute::max(SmemAlignmentP, SmemAlignmentdS, SmemAlignmentQKVdO)> {
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutK>, SmemAlignmentQKVdO> smem_k;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutV>, SmemAlignmentV> smem_v;
+ SmemdQacc_t smem_dqacc;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutQ>, SmemAlignmentQKVdO> smem_q;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutdO>, SmemAlignmentQKVdO> smem_do;
+ cute::array_aligned<ElementAccum, cute::cosize_v<SmemLayoutLSE>, 128> smem_lse;
+ cute::array_aligned<ElementAccum, cute::cosize_v<SmemLayoutLSE>, 128> smem_dpsum;
+ SmemP_t smem_p;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutPdS>, SmemAlignmentdS> smem_ds;
+ };
+
+ // Host side kernel arguments
+ struct Arguments {
+ Element const* const ptr_Q;
+ ShapeQKV const shape_Q;
+ StrideQKV const stride_Q;
+ Element const* const ptr_K;
+ ShapeQKV const shape_K;
+ StrideQKV const stride_K;
+ Element const* const ptr_V;
+ StrideQKV const stride_V;
+ Element const* const ptr_dO;
+ StrideQKV const stride_dO;
+ ElementAccum* const ptr_dQaccum;
+ ShapedQaccum const shape_dQaccum;
+ StridedQaccum const stride_dQaccum;
+ float const* const ptr_LSE_log2;
+ ShapeLSE const shape_LSE;
+ StrideLSE const stride_LSE_log2;
+ float const* const ptr_dPsum;
+ StrideLSE const stride_dPsum;
+ float const softmax_scale;
+ int const window_size_left, window_size_right, sink_token_length;
+ float const softcap_val;
+ int const num_batch;
+ int* const dq_semaphore;
+ int const* const cu_seqlens_q = nullptr;
+ int const* const cu_seqlens_k = nullptr;
+ int const* const seqused_q = nullptr;
+ int const* const seqused_k = nullptr;
+ };
+
+ // Device side kernel params
+ struct Params {
+ ShapeQKV const shape_Q;
+ ShapeQKV const shape_K;
+ ElementAccum* const ptr_dQaccum;
+ ShapedQaccum const shape_dQaccum;
+ StridedQaccum stride_dQaccum;
+ cutlass::FastDivmod qhead_per_khead_divmod;
+ TMA_QdO tma_load_Q, tma_load_dO;
+ TMA_K tma_load_K;
+ TMA_V tma_load_V;
+ float const* const ptr_LSE_log2;
+ ShapeLSE const shape_LSE;
+ StrideLSE const stride_LSE_log2;
+ float const* const ptr_dPsum;
+ StrideLSE const stride_dPsum;
+ float const softmax_scale, softmax_scale_log2;
+ int const window_size_left, window_size_right, sink_token_length;
+ float const softcap_val;
+ int const num_batch;
+ int* const dq_semaphore;
+ int const* const cu_seqlens_q = nullptr;
+ int const* const cu_seqlens_k = nullptr;
+ int const* const seqused_q = nullptr;
+ int const* const seqused_k = nullptr;
+ };
+
+ static Params
+ to_underlying_arguments(Arguments const& args) {
+ Tensor mQ = make_tensor(make_gmem_ptr(args.ptr_Q), args.shape_Q, args.stride_Q);
+ TMA_QdO tma_load_Q = make_tma_copy_A_sm90(
+ GmemTiledCopyQdO{},
+ mQ,
+ SmemLayoutQ{}(_, _, _0{}),
+ TileShape_MNK{},
+ ClusterShape{}); // mcast along N mode for this M load, if any
+ Tensor mdO = make_tensor(make_gmem_ptr(args.ptr_dO), args.shape_Q, args.stride_dO);
+ TMA_QdO tma_load_dO = make_tma_copy_A_sm90(
+ GmemTiledCopyQdO{},
+ mdO,
+ SmemLayoutdO{}(_, _, _0{}),
+ TileShape_MNK{},
+ ClusterShape{}); // mcast along N mode for this M load, if any
+ Tensor mK = make_tensor(make_gmem_ptr(args.ptr_K), args.shape_K, args.stride_K);
+ TMA_K tma_load_K = make_tma_copy_B_sm90(
+ GmemTiledCopyKV{},
+ mK,
+ SmemLayoutK{},
+ TileShape_MNK{},
+ ClusterShape{}); // no mcast for KV
+ Tensor mV = make_tensor(make_gmem_ptr(args.ptr_V), args.shape_K, args.stride_V);
+ TMA_V tma_load_V = make_tma_copy_B_sm90(
+ GmemTiledCopyKV{},
+ mV,
+ SmemLayoutV{},
+ TileShape_MNK{},
+ ClusterShape{}); // no mcast for KV
+ if constexpr (Deterministic) { assert(args.dq_semaphore != nullptr); }
+ // If there's tanh softcapping, we do tanh(scores * softmax_scale / softcap_val) * softcap_val.
+ // Right after this, we multiply by log2(e) before applying exp2.
+ // To reduce the number of instructions, we instead pre-multiply softmax_scale / softcap_val
+ // (assigning it to params.softcap_val) and pre-multiply softcap_val * log2(e)
+ // (assigning it to params.softmax_scale_log2).
+ // In the backward, we need to multiply by
+ // (1 - tanh^2) * softmax_scale / softcap_val * softcap_val = (1 - tanh^2) * softmax_scale.
+ // Instead we multiply by (1 - tanh^2) and multiply dK and dV by params.softmax_scale
+ // (the original softmax_scale) at the end.
+ return {args.shape_Q, args.shape_K,
+ args.ptr_dQaccum, args.shape_dQaccum, args.stride_dQaccum,
+ cutlass::FastDivmod(cute::ceil_div(get<2>(args.shape_Q), get<2>(args.shape_K))),
+ tma_load_Q, tma_load_dO, tma_load_K, tma_load_V,
+ args.ptr_LSE_log2, args.shape_LSE, args.stride_LSE_log2, args.ptr_dPsum, args.stride_dPsum,
+ args.softmax_scale,
+ !Has_softcap ? float(args.softmax_scale * M_LOG2E) : float(args.softcap_val * M_LOG2E),
+ args.window_size_left, args.window_size_right, args.sink_token_length,
+ !Has_softcap ? 0.f : args.softmax_scale / args.softcap_val,
+ args.num_batch, args.dq_semaphore,
+ args.cu_seqlens_q, args.cu_seqlens_k, args.seqused_q, args.seqused_k};
+ }
+
+ /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best performance
+ CUTLASS_DEVICE
+ static void prefetch_tma_descriptors(Params const& params) {
+ cute::prefetch_tma_descriptor(params.tma_load_Q.get_tma_descriptor());
+ cute::prefetch_tma_descriptor(params.tma_load_dO.get_tma_descriptor());
+ cute::prefetch_tma_descriptor(params.tma_load_K.get_tma_descriptor());
+ cute::prefetch_tma_descriptor(params.tma_load_V.get_tma_descriptor());
+ }
+
+ CUTLASS_DEVICE
+ cute::tuple<int, int> get_m_block_min_max(Params const& params, SeqlenInfo_t const& seqlen_info,
+ int n_block, int bidb) {
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ int const seqlen_q = seqlen_info.seqlen_q;
+ int const seqlen_k = seqlen_info.seqlen_k;
+ int m_block_max = cute::ceil_div(seqlen_q, kBlockM);
+ if constexpr (Is_local) {
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ if (n_block >= cute::ceil_div(params.sink_token_length, kBlockN)) {
+ m_block_max = std::min(m_block_max, cute::ceil_div((n_block + 1) * kBlockN + seqlen_q - seqlen_k + params.window_size_left, kBlockM));
+ }
+ }
+ int m_block_min = 0;
+ if constexpr (Is_causal || Is_local) {
+ m_block_min = std::max(m_block_min, (n_block * kBlockN + seqlen_q - seqlen_k - params.window_size_right) / kBlockM);
+ }
+ return {m_block_min, m_block_max};
+ }
+
+ template <typename SchedulerPrefetch, typename SharedStorage>
+ CUTLASS_DEVICE void
+ load(Params const& params,
+ MainloopPipeline pipeline_q,
+ MainloopPipeline_dO pipeline_do,
+ PipelineState& smem_pipe_write,
+ PipelineState_dO& smem_pipe_write_do,
+ SharedStorage &shared_storage,
+ SchedulerPrefetch const& scheduler_prefetch,
+ cute::tuple<int32_t, int32_t, int32_t> block_coord
+ ) {
+
+ auto [n_block, bidh, bidb] = block_coord;
+ SeqlenInfo_t seqlen_info{
+ bidb, get<0>(params.shape_Q), size<0>(params.shape_K),
+ params.cu_seqlens_q, params.cu_seqlens_k, params.seqused_q, params.seqused_k
+ };
+ auto [m_block_min, m_block_max] = get_m_block_min_max(params, seqlen_info, n_block, bidb);
+ // It's possible to have m_block_max <= m_block_min. Loading Q, K can cause illegal memory access.
+ if constexpr (Is_causal || Is_local || Varlen) {
+ if (m_block_max <= m_block_min) {
+ scheduler_prefetch();
+ return;
+ }
+ }
+
+ Tensor sQ = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), SmemLayoutQ{});
+ Tensor sdO = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), SmemLayoutdO{});
+ Tensor sK = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), SmemLayoutK{});
+ Tensor sV = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutV{});
+ Tensor sLSE = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_lse.data()), SmemLayoutLSE{});
+ Tensor sdPsum = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_dpsum.data()), SmemLayoutLSE{});
+
+ int bidh_kv = params.qhead_per_khead_divmod.divide(bidh);
+
+ // Prepare the TMA loads
+ uint32_t block_rank_in_cluster = cute::block_rank_in_cluster();
+ constexpr uint32_t cluster_shape_x = get<0>(ClusterShape());
+ uint2 cluster_local_block_id = {block_rank_in_cluster % cluster_shape_x, block_rank_in_cluster / cluster_shape_x};
+ bool const is_varlen_q = Varlen && params.cu_seqlens_q;
+ bool const is_varlen_k = Varlen && params.cu_seqlens_k;
+ Tensor mQ = params.tma_load_Q.get_tma_tensor(params.shape_Q)(_, _, bidh, !is_varlen_q ? bidb : 0);
+ Tensor mdO = params.tma_load_dO.get_tma_tensor(params.shape_Q)(_, _, bidh, !is_varlen_q ? bidb : 0);
+ Tensor mK = params.tma_load_K.get_tma_tensor(params.shape_K)(_, _, bidh_kv, !is_varlen_k ? bidb : 0);
+ Tensor mV = params.tma_load_V.get_tma_tensor(params.shape_K)(_, _, bidh_kv, !is_varlen_k ? bidb : 0);
+ Tensor mLSE = make_tensor(make_gmem_ptr(params.ptr_LSE_log2), params.shape_LSE, params.stride_LSE_log2)(_, bidh, !is_varlen_q ? bidb : 0);
+ Tensor mdPsum = make_tensor(make_gmem_ptr(params.ptr_dPsum), params.shape_LSE, params.stride_dPsum)(_, bidh, !is_varlen_q ? bidb : 0);
+
+ Tensor gQ = local_tile(domain_offset(make_coord(seqlen_info.offset_q, _0{}), mQ), select<0, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (M, K, _)
+ Tensor gdO = local_tile(domain_offset(make_coord(seqlen_info.offset_q, _0{}), mdO), select<0, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (M, K, _)
+ Tensor gK = local_tile(domain_offset(make_coord(seqlen_info.offset_k, _0{}), mK), select<1, 2>(TileShape_MNK{}), make_coord(n_block, _0{})); // (N, K)
+ Tensor gV = local_tile(domain_offset(make_coord(seqlen_info.offset_k, _0{}), mV), select<1, 2>(TileShape_MNK{}), make_coord(n_block, _0{})); // (N, K)
+ Tensor gLSE = local_tile(domain_offset(make_coord(seqlen_info.offset_q_padded), mLSE), select<0>(TileShape_MNK{}), make_coord(_)); // (M, _)
+ Tensor gdPsum = local_tile(domain_offset(make_coord(seqlen_info.offset_q_padded), mdPsum), select<0>(TileShape_MNK{}), make_coord(_)); // (M, _)
+
+ Tensor sK_x = make_tensor(sK.data(), make_layout(sK.layout(), Layout<_1>{}));
+ Tensor gK_x = make_tensor(gK.data(), make_layout(gK.layout(), Layout<_1>{}));
+ Tensor sV_x = make_tensor(sV.data(), make_layout(sV.layout(), Layout<_1>{}));
+ Tensor gV_x = make_tensor(gV.data(), make_layout(gV.layout(), Layout<_1>{}));
+ // auto [tQgQ, tQsQ] = tma_partition(params.tma_load_Q, block_rank_in_cluster, Layout<ClusterShape>{},
+ // group_modes<0, 2>(sQ), group_modes<0, 2>(gQ)); // (TMA, k), (TMA, PIPE)
+ // auto [tdOgdO, tdOsdO] = tma_partition(params.tma_load_dO, block_rank_in_cluster, Layout<ClusterShape>{},
+ // group_modes<0, 2>(sdO), group_modes<0, 2>(gdO)); // (TMA, k), (TMA, PIPE)
+ auto block_tma_Q = params.tma_load_Q.get_slice(cluster_local_block_id.y);
+ auto block_tma_dO = params.tma_load_dO.get_slice(cluster_local_block_id.y);
+ Tensor tQgQ = group_modes<0, 3>(block_tma_Q.partition_S(gQ));
+ Tensor tQsQ = group_modes<0, 3>(block_tma_Q.partition_D(sQ));
+ Tensor tdOgdO = group_modes<0, 3>(block_tma_dO.partition_S(gdO));
+ Tensor tdOsdO = group_modes<0, 3>(block_tma_dO.partition_D(sdO));
+ auto [tKgK, tKsK] = tma_partition(params.tma_load_K, _0{}, Layout<_1>{},
+ group_modes<0, 2>(sK_x), group_modes<0, 2>(gK_x)); // (TMA), (TMA)
+ auto [tVgV, tVsV] = tma_partition(params.tma_load_V, _0{}, Layout<_1>{},
+ group_modes<0, 2>(sV_x), group_modes<0, 2>(gV_x)); // (TMA), (TMA)
+ auto bulk_copy = Copy_Traits<SM90_BULK_COPY_AUTO>{};
+
+ uint16_t mcast_mask_qdo = 0;
+ if constexpr (cute::is_same_v<GmemTiledCopyQdO, SM90_TMA_LOAD_MULTICAST>) {
+ auto block_layout = Layout<ClusterShape>{}; // (m,n) -> block_id
+ for (int n = 0; n < size<1>(block_layout); ++n) {
+ mcast_mask_qdo |= (uint16_t(1) << block_layout(cluster_local_block_id.x, n, _0{}));
+ }
+ }
+
+ int m_block = m_block_min;
+
+ int lane_predicate = cute::elect_one_sync();
+
+ if (lane_predicate) {
+ pipeline_q.producer_acquire(smem_pipe_write);
+ copy(params.tma_load_Q.with(*pipeline_q.producer_get_barrier(smem_pipe_write), mcast_mask_qdo, TMA::CacheHintSm90::EVICT_LAST),
+ tQgQ(_, m_block), tQsQ(_, smem_pipe_write.index()));
+ copy(bulk_copy.with(*pipeline_q.producer_get_barrier(smem_pipe_write)),
+ gLSE(_, m_block), sLSE(_, smem_pipe_write.index()));
+ }
+
+ // // Wait for the MMA warpgroups to say that smem_k and smem_v are ready
+ // cutlass::arch::NamedBarrier::sync(NumMmaThreads + cutlass::NumThreadsPerWarp, static_cast<uint32_t>(BwdNamedBarriers::KVEmpty) /*id*/);
+
+ if (lane_predicate) {
+ // Copy K tile and V tile from GMEM to SMEM.
+ shared_storage.pipelines.barrier_KV.arrive_and_expect_tx(TmaTransactionBytesK + TmaTransactionBytesV);
+ copy(params.tma_load_K.with(reinterpret_cast<cutlass::arch::ClusterTransactionBarrier::ValueType&>(shared_storage.pipelines.barrier_KV), 0 /*mcast_mask*/), tKgK, tKsK);
+ copy(params.tma_load_V.with(reinterpret_cast<cutlass::arch::ClusterTransactionBarrier::ValueType&>(shared_storage.pipelines.barrier_KV), 0 /*mcast_mask*/), tVgV, tVsV);
+
+ #pragma unroll (kHeadDim < 256 ? 2 : 1)
+ for (; m_block < m_block_max - 1; ++m_block) {
+ // If Q and dO have the same number of stages, we can use the same pipeline state variable
+ // to reduce registers
+ PipelineState_dO smem_pipe_write_do_cur = cute::conditional_return<Q_dO_same_stages>(smem_pipe_write, smem_pipe_write_do);
+ pipeline_do.producer_acquire(smem_pipe_write_do_cur);
+ copy(params.tma_load_dO.with(*pipeline_do.producer_get_barrier(smem_pipe_write_do_cur), mcast_mask_qdo, TMA::CacheHintSm90::EVICT_LAST),
+ tdOgdO(_, m_block), tdOsdO(_, smem_pipe_write_do_cur.index()));
+ copy(bulk_copy.with(*pipeline_do.producer_get_barrier(smem_pipe_write_do_cur)),
+ gdPsum(_, m_block), sdPsum(_, smem_pipe_write_do_cur.index()));
+ if constexpr (!Q_dO_same_stages) { ++smem_pipe_write_do; }
+ ++smem_pipe_write;
+ pipeline_q.producer_acquire(smem_pipe_write);
+ copy(params.tma_load_Q.with(*pipeline_q.producer_get_barrier(smem_pipe_write), mcast_mask_qdo, TMA::CacheHintSm90::EVICT_LAST),
+ tQgQ(_, m_block + 1), tQsQ(_, smem_pipe_write.index()));
+ copy(bulk_copy.with(*pipeline_q.producer_get_barrier(smem_pipe_write)),
+ gLSE(_, m_block + 1), sLSE(_, smem_pipe_write.index()));
+ }
+ }
+ scheduler_prefetch();
+ if (lane_predicate) {
+ PipelineState_dO smem_pipe_write_do_cur = cute::conditional_return<Q_dO_same_stages>(smem_pipe_write, smem_pipe_write_do);
+ pipeline_do.producer_acquire(smem_pipe_write_do_cur);
+ copy(params.tma_load_dO.with(*pipeline_do.producer_get_barrier(smem_pipe_write_do_cur), mcast_mask_qdo, TMA::CacheHintSm90::EVICT_LAST),
+ tdOgdO(_, m_block), tdOsdO(_, smem_pipe_write_do_cur.index()));
+ copy(bulk_copy.with(*pipeline_do.producer_get_barrier(smem_pipe_write_do_cur)),
+ gdPsum(_, m_block), sdPsum(_, smem_pipe_write_do_cur.index()));
+ if constexpr (!Q_dO_same_stages) { ++smem_pipe_write_do; }
+ ++smem_pipe_write;
+ }
+ if constexpr (Q_dO_same_stages) { smem_pipe_write_do = smem_pipe_write; }
+ }
+
+ /// Perform a Producer Epilogue to prevent early exit of blocks in a Cluster
+ CUTLASS_DEVICE void
+ load_tail(MainloopPipeline pipeline_q, MainloopPipeline_dO pipeline_do,
+ PipelineState& smem_pipe_write) {
+ static_assert(Q_dO_same_stages, "Q and dO must have the same number of stages");
+ // Need to copy since pipeline_q.producer_tail(smem_pipe_write) will increment smem_pipe_write
+ PipelineState smem_pipe_write_do = smem_pipe_write;
+ // Issue the epilogue waits
+ if (cute::elect_one_sync()) {
+ /* This helps avoid early exit of blocks in Cluster
+ * Waits for all stages to either be released (all Consumer UNLOCKs), or if the stage was never used
+ * then would just be acquired since the phase was still inverted from make_producer_start_state
+ */
+ pipeline_q.producer_tail(smem_pipe_write);
+ pipeline_do.producer_tail(smem_pipe_write_do);
+ }
+ }
+
+ /// Perform a Producer Epilogue to prevent early exit of blocks in a Cluster
+ CUTLASS_DEVICE void
+ load_tail(MainloopPipeline pipeline_q, MainloopPipeline_dO pipeline_do,
+ PipelineState& smem_pipe_write, PipelineState_dO& smem_pipe_write_do) {
+ // Issue the epilogue waits
+ if (cute::elect_one_sync()) {
+ /* This helps avoid early exit of blocks in Cluster
+ * Waits for all stages to either be released (all Consumer UNLOCKs), or if the stage was never used
+ * then would just be acquired since the phase was still inverted from make_producer_start_state
+ */
+ pipeline_q.producer_tail(smem_pipe_write);
+ pipeline_do.producer_tail(smem_pipe_write_do);
+ }
+ }
+
+ template <typename SharedStorage>
+ CUTLASS_DEVICE void
+ store_dq(Params const& params,
+ SharedStorage &shared_storage,
+ cute::tuple<int32_t, int32_t, int32_t> block_coord
+ ) {
+ if constexpr (!dQacc_use_TMA) { return; }
+
+ auto [n_block, bidh, bidb] = block_coord;
+ SeqlenInfo_t seqlen_info{
+ bidb, get<0>(params.shape_Q), size<0>(params.shape_K),
+ params.cu_seqlens_q, params.cu_seqlens_k, params.seqused_q, params.seqused_k
+ };
+ auto [m_block_min, m_block_max] = get_m_block_min_max(params, seqlen_info, n_block, bidb);
+ // It's possible to have m_block_max <= m_block_min. Exit early
+ if constexpr (Is_causal || Is_local || Varlen) {
+ if (m_block_max <= m_block_min) { return; }
+ }
+
+ Tensor sdQ = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_dqacc.data()), SmemLayoutdQaccum{});
+ static constexpr int dQ_TMA_num_bytes = CUTE_STATIC_V(size<0>(sdQ)) * sizeof(ElementAccum);
+
+ bool const is_varlen = Varlen && params.cu_seqlens_q;
+ Tensor mdQaccum = make_tensor(make_gmem_ptr(reinterpret_cast<ElementAccum*>(params.ptr_dQaccum)),
+ params.shape_dQaccum, params.stride_dQaccum)(_, bidh, !is_varlen ? bidb : 0);
+ Tensor gdQaccum_ = local_tile(domain_offset(make_coord(seqlen_info.offset_q_padded * kHeadDim), mdQaccum), Shape<Int<kBlockM * kHeadDim>>{}, make_coord(_)); // (M * K, _)
+ Tensor gdQaccum = cute::flat_divide(gdQaccum_, Int<kBlockM * kHeadDim / NumMmaWarpGroups>{}); // (M * K / WG, WG, _)
+
+ int const num_batch = params.num_batch;
+ int const num_head = get<2>(params.shape_Q);
+ int *lock_ptr = !Deterministic ? nullptr : params.dq_semaphore + bidb * num_head + bidh;
+ using Barrier = cutlass::GenericBarrier<cutlass::detail::SyncwarpSync>;
+ bool const lane_predicate = cute::elect_one_sync();
+ int m_block = m_block_min;
+ #pragma unroll 2
+ for (; m_block < m_block_max; ++m_block) {
+ if constexpr (Deterministic) {
+ Barrier::wait_eq(lock_ptr, threadIdx.x % cutlass::NumThreadsPerWarp, m_block * num_batch * num_head, n_block);
+ }
+ #pragma unroll
+ for (int warpgroup_idx = 0; warpgroup_idx < NumMmaWarpGroups; ++warpgroup_idx) {
+ cutlass::arch::NamedBarrier::sync(cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, static_cast<uint32_t>(BwdNamedBarriers::dQFullWG1) + warpgroup_idx /*id*/); // sdQ full, to be written to gmem
+ if (lane_predicate) {
+ SM90_BULK_REDUCE_ADD::copy(raw_pointer_cast(sdQ(_, warpgroup_idx).data()), raw_pointer_cast(gdQaccum(_, warpgroup_idx, m_block).data()), dQ_TMA_num_bytes, static_cast<uint64_t>(TMA::CacheHintSm90::EVICT_LAST));
+ tma_store_arrive();
+ }
+ }
+ // Note, the for_each() function is required here to ensure `warpgroup_idx` is of type Int<x>.
+ for_each(make_int_sequence<NumMmaWarpGroups>{}, [&] (auto warpgroup_idx) {
+ if (lane_predicate) { tma_store_wait<NumMmaWarpGroups - 1 - CUTE_STATIC_V(warpgroup_idx)>(); }
+ cutlass::arch::NamedBarrier::arrive(cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, static_cast<uint32_t>(BwdNamedBarriers::dQEmptyWG1) + warpgroup_idx /*id*/); // sdQ empty, ready to be written to
+ });
+ if constexpr (Deterministic) {
+ Barrier::arrive_inc(lock_ptr, threadIdx.x % cutlass::NumThreadsPerWarp, m_block * num_batch * num_head);
+ }
+ }
+ if constexpr (Is_local && Deterministic) {
+ constexpr int kBlockM = get<0>(TileShape_MNK{});
+ int const m_block_global_max = cute::ceil_div(seqlen_info.seqlen_q, kBlockM);
+ #pragma unroll 2
+ for (; m_block < m_block_global_max; ++m_block) {
+ Barrier::arrive_inc(lock_ptr, threadIdx.x % cutlass::NumThreadsPerWarp, m_block * num_batch * num_head);
+ }
+ }
+ }
+
+ CUTLASS_DEVICE void
+ mma_init() {
+ // We're not currently using this bc we're not using persistent scheduler
+ // // Tell producer (warp 0) that smem_k and smem_v are ready
+ // cutlass::arch::NamedBarrier::arrive(NumMmaThreads + cutlass::NumThreadsPerWarp, static_cast<uint32_t>(BwdNamedBarriers::KVEmpty) /*id*/);
+ int warp_idx_in_warpgroup = __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0);
+ if constexpr (dQacc_use_TMA) {
+ if (warp_idx_in_warpgroup == 0) {
+ cutlass::arch::NamedBarrier::arrive(cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, static_cast<uint32_t>(BwdNamedBarriers::dQEmptyWG1) - 1 + flash::canonical_warp_group_idx_nosync() /*id*/); // sdQ empty, ready to be written to
+ }
+ }
+ }
+
+ template <typename SharedStorage, typename FrgTensordKV>
+ CUTLASS_DEVICE bool
+ mma(Params const& params,
+ MainloopPipeline pipeline_q,
+ MainloopPipeline_dO pipeline_do,
+ PipelineState& smem_pipe_read,
+ PipelineState_dO& smem_pipe_read_do,
+ FrgTensordKV& tdKrdK,
+ FrgTensordKV& tdVrdV,
+ int thread_idx,
+ int &work_idx,
+ cute::tuple<int32_t, int32_t, int32_t> block_coord,
+ SharedStorage& shared_storage
+ ) {
+ static_assert(is_rmem<FrgTensordKV>::value, "dK and dV tensor must be rmem resident.");
+
+ int n_block = get<0>(block_coord);
+ int bidb = get<2>(block_coord);
+ SeqlenInfo_t seqlen_info{
+ bidb, get<0>(params.shape_Q), size<0>(params.shape_K),
+ params.cu_seqlens_q, params.cu_seqlens_k, params.seqused_q, params.seqused_k
+ };
+ auto [m_block_min, m_block_max] = get_m_block_min_max(params, seqlen_info, n_block, bidb);
+ // It's possible to have m_block_max <= m_block_min. Exit early
+ if constexpr (Is_causal || Is_local || Varlen) {
+ if (m_block_max <= m_block_min) { return false; }
+ }
+
+ Tensor sQ = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), SmemLayoutQ{});
+ Tensor sdO = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), SmemLayoutdO{});
+ Tensor sK = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), SmemLayoutK{});
+ Tensor sV = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutV{});
+ Tensor sQt = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), SmemLayoutQt{});
+ Tensor sdOt = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_do.data()), SmemLayoutdOt{});
+ Tensor sKt = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), SmemLayoutKt{});
+ Tensor sP = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), SmemLayoutPdS{});
+ Tensor sP_pi = cute::as_position_independent_swizzle_tensor(sP);
+ Tensor sPt = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), SmemLayoutPdSt{});
+ Tensor sPt_pi = cute::as_position_independent_swizzle_tensor(sPt);
+ Tensor sdS = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), SmemLayoutPdS{});
+ Tensor sdS_pi = cute::as_position_independent_swizzle_tensor(sdS);
+ Tensor sdSt = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_ds.data()), SmemLayoutPdSt{});
+ Tensor sdSt_pi = cute::as_position_independent_swizzle_tensor(sdSt);
+ Tensor sdQ = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_dqacc.data()), SmemLayoutdQaccum{});
+ Tensor sLSEMma = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_lse.data()), SmemLayoutLSEMma{});
+ Tensor sdPsumMma = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_dpsum.data()), SmemLayoutLSEMma{});
+
+ static_assert(stride<0>(typename TiledMmaSdP::ALayout{}) == 0 and
+ stride<0>(typename TiledMmaSdP::BLayout{}) == 0 and
+ size<0>(typename TiledMmaSdP::ALayout{}) == cutlass::NumThreadsPerWarpGroup and
+ size<0>(typename TiledMmaSdP::BLayout{}) == cutlass::NumThreadsPerWarpGroup,
+ "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup");
+ constexpr int MmaWarpGroups = NumMmaThreads / cutlass::NumThreadsPerWarpGroup;
+ Layout warp_group_thread_layout = make_layout(make_shape(Int<MmaWarpGroups>{}),
+ make_stride(Int<cutlass::NumThreadsPerWarpGroup>{}));
+ Layout warp_group_thread_layout_dq = make_layout(make_shape(Int<NumMmaWarpGroups>{}),
+ make_stride(Int<cutlass::NumThreadsPerWarpGroup>{}));
+
+ int warp_group_idx = __shfl_sync(0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0);
+ TiledMmaSdP tiled_mma_SdP;
+ using TiledMmadP = std::conditional_t<!Mma_dP_is_RS, TiledMmaSdP, TiledMmadPRS>;
+ TiledMmadP tiled_mma_dP;
+ TiledMmadKV tiled_mma_dKV;
+ TiledMmadQ tiled_mma_dQ;
+
+ auto wg_mma_SdP = tiled_mma_SdP.get_slice(warp_group_thread_layout(warp_group_idx));
+ auto wg_mma_dP = tiled_mma_dP.get_slice(warp_group_thread_layout(warp_group_idx));
+ auto thread_mma_SdP = tiled_mma_SdP.get_thread_slice(thread_idx);
+ auto wg_mma_dKV = tiled_mma_dKV.get_slice(warp_group_thread_layout(warp_group_idx));
+ auto wg_mma_dQ = tiled_mma_dQ.get_slice(warp_group_thread_layout_dq(warp_group_idx));
+
+ auto smem_tiled_copy_PdS = make_tiled_copy_C(SmemCopyAtomPdS{}, tiled_mma_SdP);
+ auto smem_thr_copy_PdS = smem_tiled_copy_PdS.get_thread_slice(thread_idx);
+
+ R2STiledCopydQaccum r2s_tiled_copy_dQaccum;
+ auto r2s_thr_copy_dQaccum = r2s_tiled_copy_dQaccum.get_thread_slice(thread_idx);
+ Tensor tdQsdQaccum = r2s_thr_copy_dQaccum.partition_D(sdQ);
+ // if (thread_idx == 0) { print(sdQ); printf("\n"); print(tdQsdQaccum); printf("\n"); }
+
+ // Allocate "fragments/descriptors"
+ // We have to use the templated mma_partition_fragment_AB instead of cute::conditional_return or lambda,
+ // because some partition_fragment_A/B don't compile.
+ // https://stackoverflow.com/questions/50051473/if-constexpr-in-c17-does-not-work-in-a-non-templated-function
+ Tensor tSrQ = mma_partition_fragment_AB</*A=*/!SdP_swapAB>(wg_mma_SdP, sQ);
+ Tensor tSrK = mma_partition_fragment_AB</*A=*/SdP_swapAB>(wg_mma_SdP, sK);
+ Tensor tdPrdO = mma_partition_fragment_AB</*A=*/!SdP_swapAB>(wg_mma_SdP, sdO);
+ Tensor tdPrV = mma_partition_fragment_AB</*A=*/SdP_swapAB>(wg_mma_dP, sV);
+ Tensor tdVrdO = mma_partition_fragment_AB</*A=*/dKV_swapAB>(wg_mma_dKV, sdOt);
+ Tensor tdKrQ = mma_partition_fragment_AB</*A=*/dKV_swapAB>(wg_mma_dKV, sQt);
+ Tensor tdQrdS = mma_partition_fragment_AB</*A=*/!dQ_swapAB>(wg_mma_dQ, sdS);
+ Tensor tdQrK = mma_partition_fragment_AB</*A=*/dQ_swapAB>(wg_mma_dQ, sKt);
+
+ Tensor tPsP = smem_thr_copy_PdS.partition_D(cute::conditional_return<!SdP_swapAB>(sP_pi, sPt_pi)); // ((Atom,AtomNum),PIPE_M,PIPE_N)
+ Tensor tdSsdS = smem_thr_copy_PdS.partition_D(cute::conditional_return<!SdP_swapAB>(sdS_pi, sdSt_pi)); // ((Atom,AtomNum),PIPE_M,PIPE_N)
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print(smem_thr_copy_PdS); print(sP_pi); printf("\n"); print(sPt_pi); printf("\n"); print(tPsP); printf("\n"); print(tdSsdS); printf("\n"); }
+
+ // thread_mma_SdP.partition_C(sLSEMma) has shape ((2, 2, V), MMA_M, MMA_N, PIPE), we only take the col indices
+ // or row indices, depending on whether SdP_swapAB.
+ Tensor tLSEsLSE = cute::conditional_return<!SdP_swapAB>(
+ group_modes<0, 2>(thread_mma_SdP.partition_C(sLSEMma)(make_coord(_0{}, _, _0{}), _, _0{}, _)), // (2, MMA_M, PIPE)
+ group_modes<0, 3>(thread_mma_SdP.partition_C(sLSEMma)(make_coord(_, _0{}, _), _0{}, _, _))); // (2, V, MMA_N, PIPE)
+ Tensor tLSEsdPsum = cute::conditional_return<!SdP_swapAB>(
+ group_modes<0, 2>(thread_mma_SdP.partition_C(sdPsumMma)(make_coord(_0{}, _, _0{}), _, _0{}, _)),
+ group_modes<0, 3>(thread_mma_SdP.partition_C(sdPsumMma)(make_coord(_, _0{}, _), _0{}, _, _)));
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print(sLSEMma); printf("\n"); print(tLSEsLSE); printf("\n"); }
+ // If we want to split the stats among the 8 threads that share the same rows.
+ static constexpr int kStatsPerThread = cute::ceil_div(decltype(size(tLSEsLSE))::value, 8);
+
+ auto consumer_wait = [](auto& pipeline, auto& smem_pipe_read) {
+ auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read);
+ pipeline.consumer_wait(smem_pipe_read, barrier_token);
+ };
+
+ int bidh = get<1>(block_coord);
+ int const seqlen_q = seqlen_info.seqlen_q;
+ int const seqlen_k = seqlen_info.seqlen_k;
+
+ // For the case where we do atomicAdd directly to gdQaccum instead of using TMA
+ bool const is_varlen = Varlen && params.cu_seqlens_q;
+ Tensor mdQaccum = make_tensor(make_gmem_ptr(reinterpret_cast<ElementAccum*>(params.ptr_dQaccum)),
+ params.shape_dQaccum, params.stride_dQaccum)(_, bidh, !is_varlen ? bidb : 0);
+ Tensor gdQaccum_ = local_tile(domain_offset(make_coord(seqlen_info.offset_q_padded * kHeadDim), mdQaccum), Shape<Int<kBlockM * kHeadDim>>{}, make_coord(_)); // (M * K, _)
+ Tensor gdQaccum = cute::flat_divide(gdQaccum_, Int<kBlockM * kHeadDim / NumMmaWarpGroups>{}); // (M * K / WG, WG, _)
+ // We can reuse r2s_thr_copy_dQaccum for this partitioning
+ Tensor tdQgdQaccum = r2s_thr_copy_dQaccum.partition_D(gdQaccum);
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print(mdQaccum); printf("\n"); print(gdQaccum_); printf("\n"); print(gdQaccum); printf("\n"); print(tdQgdQaccum); printf("\n"); }
+
+ flash::Mask<kBlockM, kBlockN, false /*PackGQA*/, TiledMmaSdP, SdP_swapAB> mask(
+ thread_idx, seqlen_q, seqlen_k, params.window_size_left, params.window_size_right, params.sink_token_length,
+ params.qhead_per_khead_divmod
+ );
+
+ int m_block = m_block_min;
+
+ clear(tdKrdK);
+ clear(tdVrdV);
+ // tiled_mma_dKV.accumulate_ = GMMA::ScaleOut::Zero;
+
+ cutlass::ConsumerToken barrier_token = static_cast<cutlass::BarrierStatus>(shared_storage.pipelines.barrier_KV.try_wait(work_idx % 2));
+ if (barrier_token == cutlass::BarrierStatus::WaitAgain) { shared_storage.pipelines.barrier_KV.wait(work_idx % 2); }
+
+ if constexpr (Mma_dP_is_RS) {
+ using SmemCopyAtomV = Copy_Atom<cute::SM75_U32x4_LDSM_N, Element>;
+ auto smem_tiled_copy_V = make_tiled_copy_A(SmemCopyAtomV{}, tiled_mma_dP);
+ auto smem_thr_copy_V = smem_tiled_copy_V.get_thread_slice(thread_idx);
+ Tensor tdPrV_copy_view = smem_thr_copy_V.retile_D(tdPrV);
+ Tensor tdPsV_copy_view = smem_thr_copy_V.partition_S(cute::as_position_independent_swizzle_tensor(sV));
+ cute::copy(smem_tiled_copy_V, tdPsV_copy_view, tdPrV_copy_view);
+ }
+
+ auto bwd_step = [&](int m_block, auto mask_fn) {
+ Tensor tSrS = partition_fragment_C(tiled_mma_SdP, select<!SdP_swapAB ? 0 : 1, !SdP_swapAB ? 1 : 0>(TileShape_MNK{}));
+ consumer_wait(pipeline_q, smem_pipe_read);
+ flash::gemm</*zero_init=*/true, /*wg_wait=*/-1, /*SwapAB=*/SdP_swapAB>(tiled_mma_SdP, tSrQ(_, _, _, smem_pipe_read.index()), tSrK, tSrS);
+ Tensor tLSErLSE = cute::conditional_return<!ShuffleLSE>(make_fragment_like(tLSEsLSE(_, _0{})), make_tensor<ElementAccum>(Int<kStatsPerThread>{}));
+ if constexpr (!ShuffleLSE) {
+ cute::copy(tLSEsLSE(_, smem_pipe_read.index()), tLSErLSE);
+ } else {
+ #pragma unroll
+ for (int i = 0; i < kStatsPerThread; ++i) {
+ // It's ok to read OOB, since we made sure sLSE is large enough and we won't use the OOB values
+ tLSErLSE(i) = tLSEsLSE((thread_idx % 32) / 4 + i * 8, smem_pipe_read.index());
+ }
+ }
+ Tensor tdPrdP = partition_fragment_C(tiled_mma_SdP, select<!SdP_swapAB ? 0 : 1, !SdP_swapAB ? 1 : 0>(TileShape_MNK{}));
+ PipelineState_dO smem_pipe_read_do_cur = cute::conditional_return<Q_dO_same_stages>(smem_pipe_read, smem_pipe_read_do);
+ consumer_wait(pipeline_do, smem_pipe_read_do_cur);
+ flash::gemm</*zero_init=*/true, /*wg_wait=*/-1, /*SwapAB=*/SdP_swapAB>(tiled_mma_dP, tdPrdO(_, _, _, smem_pipe_read_do_cur.index()), tdPrV, tdPrdP);
+ warpgroup_wait<1>();
+ if constexpr (Has_softcap) { flash::apply_softcap(tSrS, params.softcap_val); }
+
+ // Reshape tSrS from ((2, 2, V), MMA_N, MMA_M) to (nrow=(2, V, MMA_M), ncol=(2, MMA_N))
+ Tensor scores = make_tensor(tSrS.data(), flash::convert_layout_acc_rowcol</*Transposed=*/SdP_swapAB>(tSrS.layout()));
+ // dtanh needs to happen before masking, otherwise we get 1 - (-inf)^2 = NaN in the dtanh
+ auto dtanh = [&] { if constexpr (Has_softcap) return flash::calculate_dtanh(scores); else return nullptr; }();
+ mask_fn(tSrS, m_block);
+ #pragma unroll
+ for (int mi = 0; mi < size<0>(scores); ++mi) {
+ float const lse_scaled = [&] {
+ if constexpr (!ShuffleLSE) return tLSErLSE(mi);
+ else return __shfl_sync(0xffffffff, tLSErLSE(mi / 8), (mi % 8) * 4 + (thread_idx % 4));
+ }();
+ #pragma unroll
+ for (int ni = 0; ni < size<1>(scores); ++ni) {
+ scores(mi, ni) = exp2f(scores(mi, ni) * params.softmax_scale_log2 - lse_scaled);
+ }
+ }
+
+ Tensor tLSErdPsum = cute::conditional_return<!ShuffledPsum>(make_fragment_like(tLSEsdPsum(_, _0{})), make_tensor<ElementAccum>(Int<kStatsPerThread>{}));
+ if constexpr (!ShuffledPsum) {
+ cute::copy(tLSEsdPsum(_, smem_pipe_read_do_cur.index()), tLSErdPsum);
+ } else {
+ #pragma unroll
+ for (int i = 0; i < kStatsPerThread; ++i) {
+ tLSErdPsum(i) = tLSEsdPsum((thread_idx % 32) / 4 + i * 8, smem_pipe_read_do_cur.index());
+ }
+ }
+
+ warpgroup_wait<0>();
+ // Reshape tdPrdP from ((2, 2, V), MMA_N, MMA_M) to (nrow=(2, V, MMA_M), ncol=(2, MMA_N))
+ Tensor dS = make_tensor(tdPrdP.data(), scores.layout());
+ #pragma unroll
+ for (int mi = 0; mi < size<0>(dS); ++mi) {
+ float const dP_sum_cur = [&] {
+ if constexpr (!ShuffledPsum) return tLSErdPsum(mi);
+ else return __shfl_sync(0xffffffff, tLSErdPsum(mi / 8), (mi % 8) * 4 + (thread_idx % 4));
+ }();
+ #pragma unroll
+ for (int ni = 0; ni < size<1>(dS); ++ni) {
+ dS(mi, ni) = scores(mi, ni) * (dS(mi, ni) - dP_sum_cur);
+ if constexpr (Has_softcap) { dS(mi, ni) *= dtanh(mi, ni); }
+ }
+ }
+
+ // Convert scores from fp32 to fp16/bf16
+ Tensor rP = make_tensor_like<Element>(tSrS);
+ flash::convert_type_out(tSrS, rP);
+ if constexpr (!Mma_dKV_is_RS) {
+ // Need to sync to make sure P has already been used in the previous iteration before writing new values
+ if constexpr (kStages_dS == 1) {
+ cutlass::arch::NamedBarrier::sync(NumMmaThreads, static_cast<uint32_t>(BwdNamedBarriers::PdS) /*id*/);
+ }
+ Tensor tPaP = smem_thr_copy_PdS.retile_S(rP); // ((Atom,AtomNum), MMA_N, MMA_N)
+ cute::copy(smem_tiled_copy_PdS, tPaP, tPsP(_, _, _, cute::conditional_return<kStages_dS==1>(_0{}, smem_pipe_read.index())));
+ }
+ Tensor rdS = make_tensor_like<Element>(tdPrdP);
+ flash::convert_type_out(tdPrdP, rdS);
+ // If there's double buffering on dS, we don't need to sync here.
+ // Otherwise we might have WG1 writing to dS before WG2 is done reading from it during MmadQ.
+ // But because both WGs have to sync at the end of the loop and double buffering,
+ // this race condition is not possible.
+ // This sync is to ensure (1) P is written in case of !Mma_dKV_is_RS and
+ // (2) dS is already read by the Mma in the previous iteration in case of Mma_dKV_is_RS.
+ if constexpr (!Mma_dKV_is_RS || (kStages_dS == 1 && Mma_dKV_is_RS)) {
+ cutlass::arch::fence_view_async_shared();
+ cutlass::arch::NamedBarrier::sync(NumMmaThreads, static_cast<uint32_t>(BwdNamedBarriers::PdS) /*id*/);
+ }
+ // For hdim 64, It's faster to write to smem_dS first before the dV gemm
+ Tensor tdSadS = smem_thr_copy_PdS.retile_S(rdS); // ((Atom,AtomNum), MMA_N, MMA_N)
+ cute::copy(smem_tiled_copy_PdS, tdSadS, tdSsdS(_, _, _, cute::conditional_return<kStages_dS==1>(_0{}, smem_pipe_read.index())));
+
+ if constexpr (!Slice_dQKV_Mma) {
+ // Most cases take this path, except for hdim256 where we want to slice to reduce register pressure
+ if constexpr (Mma_dKV_is_RS) {
+ Tensor tdVrP = make_tensor(rP.data(), convert_layout_acc_Aregs<TiledMmadKV>(tSrS.layout()));
+ flash::gemm</*zero_init=*/false, /*wg_wait=*/-1>(tiled_mma_dKV, tdVrP, tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), tdVrdV);
+ } else {
+ Tensor tdVrP = mma_partition_fragment_AB</*A=*/!dKV_swapAB>(wg_mma_dKV, sPt);
+ Tensor tdVrP_cur = tdVrP(_, _, _, cute::conditional_return<kStages_dS==1>(_0{}, smem_pipe_read.index()));
+ flash::gemm</*zero_init=*/false, /*wg_wait=*/-1, /*SwapAB=*/dKV_swapAB>(tiled_mma_dKV, tdVrP_cur, tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), tdVrdV);
+ }
+ // SMEM fence to make sure sdS is written before it's read by WGMMA
+ cutlass::arch::fence_view_async_shared();
+ cutlass::arch::NamedBarrier::sync(NumMmaThreads, static_cast<uint32_t>(BwdNamedBarriers::PdS) /*id*/);
+ Tensor tdQrdQ = partition_fragment_C(tiled_mma_dQ, select<!dQ_swapAB ? 0 : 2, !dQ_swapAB ? 2 : 0>(TileShape_MNK{}));
+ Tensor tdQrdS_cur = tdQrdS(_, _, _, cute::conditional_return<kStages_dS==1>(_0{}, smem_pipe_read.index()));
+ flash::gemm</*zero_init=*/true, /*wg_wait=*/1, /*SwapAB=*/dQ_swapAB>(tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ);
+ pipeline_do.consumer_release(smem_pipe_read_do_cur); // release dQ
+
+ if constexpr (Mma_dKV_is_RS) {
+ Tensor tdKrdS = make_tensor(rdS.data(), convert_layout_acc_Aregs<TiledMmadKV>(tdPrdP.layout()));
+ flash::gemm</*zero_init=*/false, /*wg_wait=*/1>(tiled_mma_dKV, tdKrdS, tdKrQ(_, _, _, smem_pipe_read.index()), tdKrdK);
+ } else {
+ Tensor tdKrdS = mma_partition_fragment_AB</*A=*/!dKV_swapAB>(wg_mma_dKV, sdSt);
+ Tensor tdKrdS_cur = tdKrdS(_, _, _, cute::conditional_return<kStages_dS==1>(_0{}, smem_pipe_read.index()));
+ flash::gemm</*zero_init=*/false, /*wg_wait=*/1, /*SwapAB=*/dKV_swapAB>(tiled_mma_dKV, tdKrdS_cur, tdKrQ(_, _, _, smem_pipe_read.index()), tdKrdK);
+ }
+ if constexpr (dQacc_use_TMA) {
+ int const warp_group_idx = flash::canonical_warp_group_idx_nosync() - 1;
+ cutlass::arch::NamedBarrier::sync(cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, static_cast<uint32_t>(BwdNamedBarriers::dQEmptyWG1) + warp_group_idx /*id*/); // sdQ full, to be written to gmem
+ Tensor taccdQrdQ = r2s_thr_copy_dQaccum.retile_S(tdQrdQ);
+ cute::copy(r2s_tiled_copy_dQaccum, taccdQrdQ, tdQsdQaccum);
+ cutlass::arch::fence_view_async_shared();
+ cutlass::arch::NamedBarrier::arrive(cutlass::NumThreadsPerWarpGroup + cutlass::NumThreadsPerWarp, static_cast<uint32_t>(BwdNamedBarriers::dQFullWG1) + warp_group_idx /*id*/); // sdQ full, to be written to gmem
+ } else {
+ // We can reuse r2s_thr_copy_dQaccum for this partitioning
+ Tensor tdQrdQ_atomic = recast<float4>(r2s_thr_copy_dQaccum.retile_S(tdQrdQ));
+ Tensor tdQgdQaccum_atomic = recast<float4>(tdQgdQaccum(_, _, _, m_block));
+ static_assert(CUTE_STATIC_V(size(tdQrdQ_atomic)) == CUTE_STATIC_V(size(tdQgdQaccum_atomic)));
+ #pragma unroll
+ for (int i = 0; i < size(tdQrdQ_atomic); ++i) { atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); }
+ }
+
+ } else { // Slice_dQKV_Mma
+
+ static_assert(!(Slice_dQKV_Mma && Mma_dKV_is_RS));
+ Tensor tdVrP = mma_partition_fragment_AB</*A=*/!dKV_swapAB>(wg_mma_dKV, sPt);
+ Tensor tdVrP_cur = tdVrP(_, _, _, cute::conditional_return<kStages_dS==1>(_0{}, smem_pipe_read.index()));
+ flash::gemm</*zero_init=*/false, /*wg_wait=*/-1, /*SwapAB=*/dKV_swapAB, /*M_slice=*/0>(tiled_mma_dKV, tdVrP_cur, tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), tdVrdV);
+
+ cutlass::arch::fence_view_async_shared();
+ cutlass::arch::NamedBarrier::sync(NumMmaThreads, static_cast<uint32_t>(BwdNamedBarriers::PdS) /*id*/);
+ Tensor tdQrdQ = partition_fragment_C(tiled_mma_dQ, select<!dQ_swapAB ? 0 : 2, !dQ_swapAB ? 2 : 0>(TileShape_MNK{}));
+ Tensor tdQrdS_cur = tdQrdS(_, _, _, cute::conditional_return<kStages_dS==1>(_0{}, smem_pipe_read.index()));
+ flash::gemm</*zero_init=*/true, /*wg_wait=*/-1, /*SwapAB=*/dQ_swapAB, /*M_slice=*/0>(tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ);
+ flash::gemm</*zero_init=*/false, /*wg_wait=*/1, /*SwapAB=*/dKV_swapAB, /*M_slice=*/1>(tiled_mma_dKV, tdVrP_cur, tdVrdO(_, _, _, smem_pipe_read_do_cur.index()), tdVrdV);
+ Tensor tdQrdQ_atomic = recast<float4>(r2s_thr_copy_dQaccum.retile_S(tdQrdQ));
+ Tensor tdQgdQaccum_atomic = recast<float4>(tdQgdQaccum(_, _, _, m_block));
+ #pragma unroll
+ for (int i = 0; i < size(tdQrdQ_atomic) / 2; ++i) { atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); }
+
+ Tensor tdKrdS = mma_partition_fragment_AB</*A=*/!dKV_swapAB>(wg_mma_dKV, sdSt);
+ Tensor tdKrdS_cur = tdKrdS(_, _, _, cute::conditional_return<kStages_dS==1>(_0{}, smem_pipe_read.index()));
+ flash::gemm</*zero_init=*/false, /*wg_wait=*/1, /*SwapAB=*/dKV_swapAB, /*M_slice=*/0>(tiled_mma_dKV, tdKrdS_cur, tdKrQ(_, _, _, smem_pipe_read.index()), tdKrdK);
+ pipeline_do.consumer_release(smem_pipe_read_do_cur); // release dO
+
+ flash::gemm</*zero_init=*/true, /*wg_wait=*/0, /*SwapAB=*/dQ_swapAB, /*M_slice=*/1>(tiled_mma_dQ, tdQrdS_cur, tdQrK, tdQrdQ);
+ #pragma unroll
+ for (int i = size(tdQrdQ_atomic) / 2; i < size(tdQrdQ_atomic); ++i) { atomicAdd(&tdQgdQaccum_atomic(i), tdQrdQ_atomic(i)); }
+
+ flash::gemm</*zero_init=*/false, /*wg_wait=*/-1, /*SwapAB=*/dKV_swapAB, /*M_slice=*/1>(tiled_mma_dKV, tdKrdS_cur, tdKrQ(_, _, _, smem_pipe_read.index()), tdKrdK);
+ }
+
+ warpgroup_wait<0>();
+ pipeline_q.consumer_release(smem_pipe_read); // release Q
+ ++smem_pipe_read;
+ if constexpr (!Q_dO_same_stages) { ++smem_pipe_read_do; }
+ };
+
+ // We have separate iterations with causal masking. Not necessary for hdim 128 but for hdim 64
+ // this helps quite a bit to not have to do causal masking for most of the iterations.
+ if constexpr ((Is_causal || Is_local) && SeparateMaskingIterations) {
+ auto mask_fn = [&](auto& tSrS, int m_block) { mask.template apply<true /*Seqlenk_mask*/, Is_causal, Is_local>(tSrS, m_block, n_block); };
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ int const m_block_masking_max = ((n_block + 1) * kBlockN - 1 + seqlen_q - seqlen_k - params.window_size_right) / kBlockM + 1;
+ CUTLASS_PRAGMA_NO_UNROLL
+ for (; m_block < std::min(m_block_max, m_block_masking_max); ++m_block) {
+ bwd_step(m_block, mask_fn);
+ }
+ }
+
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ int const m_block_max_before_local_mask = !Is_local || !SeparateMaskingIterations
+ ? m_block_max
+ : std::min(m_block_max, (n_block * kBlockN + seqlen_q - seqlen_k + params.window_size_left) / kBlockM);
+
+ auto mask_fn = [&](auto& tSrS, int m_block) { mask.template apply<true /*Seqlenk_mask*/, Is_causal && !SeparateMaskingIterations, Is_local && !SeparateMaskingIterations>(tSrS, m_block, n_block); };
+ CUTLASS_PRAGMA_NO_UNROLL
+ for (; m_block < m_block_max_before_local_mask; ++m_block) {
+ bwd_step(m_block, mask_fn);
+ }
+
+ if constexpr (Is_local && SeparateMaskingIterations) {
+ auto mask_fn = [&](auto& tSrS, int m_block) { mask.template apply<true /*Seqlenk_mask*/, false /*Causal_mask*/, Is_local>(tSrS, m_block, n_block); };
+ CUTLASS_PRAGMA_NO_UNROLL
+ for (; m_block < m_block_max; ++m_block) {
+ bwd_step(m_block, mask_fn);
+ }
+ }
+
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(tdVrdV); }
+ #pragma unroll
+ for (int i = 0; i < size(tdKrdK); ++i) { tdKrdK(i) *= params.softmax_scale; }
+
+ if constexpr (Q_dO_same_stages) { smem_pipe_read_do = smem_pipe_read; }
+ ++work_idx;
+ return true;
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/mainloop_fwd_sm80.hpp b/src/solarwm/backends/minimax_h3/hopper_src/mainloop_fwd_sm80.hpp
new file mode 100644
index 0000000..e439045
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/mainloop_fwd_sm80.hpp
@@ -0,0 +1,880 @@
+/******************************************************************************
+ * Copyright (c) 2024, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cutlass/cutlass.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+#include <cutlass/numeric_conversion.h>
+
+#include "cute/tensor.hpp"
+
+#include "seqlen.h"
+#include "mask.h"
+#include "pack_gqa.h"
+#include "paged_kv.h"
+#include "rotary.h"
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <int kNWarps, int Stages, bool Q_in_regs, class TileShape_MNK_, class Element_, class ElementAccum_, class ArchTag_,
+ bool Is_causal_, bool Is_local_, bool Has_softcap_, bool Varlen_, bool PagedKV_, bool AppendKV_,
+ bool PackGQA_, bool Split_>
+struct CollectiveMainloopFwdSm80 {
+
+ static constexpr int kStages = Stages;
+ static_assert(kStages > 0, "kStages must be greater than 0");
+ using TileShape_MNK = TileShape_MNK_;
+ using Element = Element_;
+ using ElementAccum = ElementAccum_;
+ using ArchTag = ArchTag_;
+ static constexpr bool Is_FP8 = cute::is_same_v<Element, cutlass::float_e4m3_t> || cute::is_same_v<Element, cutlass::float_e5m2_t>;;
+ static constexpr bool Is_causal = Is_causal_;
+ static constexpr bool Is_local = Is_local_;
+ static constexpr bool Has_softcap = Has_softcap_;
+ static constexpr bool Varlen = Varlen_;
+ static constexpr bool PagedKV = PagedKV_;
+ static constexpr bool AppendKV = AppendKV_;
+ static constexpr bool PackGQA = PackGQA_;
+ static constexpr bool Split = Split_;
+ static constexpr bool Transpose_V = Is_FP8;
+ using SeqlenInfo_t = flash::SeqlenInfoQKNewK<Varlen, AppendKV>;
+
+ static_assert(ArchTag::kMinComputeCapability >= 80);
+
+ static constexpr bool Has_cp_async = ArchTag::kMinComputeCapability >= 80;
+
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ static constexpr int kHeadDim = get<2>(TileShape_MNK{});
+
+ using MMA_Atom_Arch = std::conditional_t<
+ ArchTag::kMinComputeCapability >= 80,
+ std::conditional_t<
+ std::is_same_v<Element, cutlass::half_t>,
+ MMA_Atom<SM80_16x8x16_F32F16F16F32_TN>,
+ MMA_Atom<SM80_16x8x16_F32BF16BF16F32_TN>
+ >,
+ MMA_Atom<SM75_16x8x8_F32F16F16F32_TN>
+ >;
+ using TiledMma = TiledMMA<
+ MMA_Atom_Arch,
+ Layout<Shape<Int<kNWarps>,_1,_1>>, // 4x1x1 or 8x1x1 thread group
+ Tile<Int<16 * kNWarps>, _16, _16>>;
+
+ static constexpr int NumMmaThreads = size(TiledMma{});
+ static constexpr int NumProducerThreads = NumMmaThreads; // For compatibility with TileScheduler
+
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(Element);
+ static_assert(kHeadDim % kGmemElemsPerLoad == 0, "Headdim must be a multiple of kGmemElemsPerLoad");
+ // We want each "row" to have 64 elements (128 bytes, i.e. 1 cache line). E.g. if hdim=128, we want each
+ // thread to have 4 loads in the M direction and 2 vectorized load in the K direction.
+ static constexpr int kBytePerRow = kHeadDim * sizeof(Element);
+ static constexpr int kBlockKGmem = (kBytePerRow % 128 == 0 ? 128 : (kBytePerRow % 64 == 0 ? 64 : 32)) / sizeof(Element);
+
+ static constexpr int kSwizzle = kBlockKGmem == 128 ? 4 : (kBlockKGmem == 64 ? 3 : (kBlockKGmem == 32 ? 2 : 1));
+ static constexpr int kSwizzleBase = sizeof(Element) == 4 ? 2 : (sizeof(Element) == 2 ? 3 : 4);
+ using SmemLayoutAtomQKV = decltype(
+ composition(Swizzle<kSwizzle, kSwizzleBase, kSwizzleBase>{},
+ Layout<Shape<_8, Int<kBlockKGmem>>,
+ Stride<Int<kBlockKGmem>, _1>>{}));
+ using SmemLayoutQ = decltype(tile_to_shape(SmemLayoutAtomQKV{}, select<0, 2>(TileShape_MNK{})));
+
+ using SmemLayoutK = decltype(tile_to_shape(
+ SmemLayoutAtomQKV{},
+ make_shape(shape<1>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int<kStages>{})));
+
+ using SmemLayoutV = decltype(tile_to_shape(
+ SmemLayoutAtomQKV{},
+ make_shape(shape<1>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int<kStages>{})));
+ using SmemLayoutVt = decltype(
+ composition(SmemLayoutV{},
+ make_ordered_layout(make_shape(shape<2>(TileShape_MNK{}), shape<1>(TileShape_MNK{}), Int<kStages>{}),
+ Step<_2, _1, _3>{})));
+
+ using SmemCopyAtom = Copy_Atom<SM75_U32x4_LDSM_N, Element>;
+ using SmemCopyAtomTransposed = Copy_Atom<SM75_U16x8_LDSM_T, Element>;
+
+ // We use CACHEGLOBAL instead of CACHEALWAYS for both Q and K/V, since we won't be reading
+ // from the same address by the same threadblock. This is slightly faster.
+ using GmemCopyAtom = Copy_Atom<std::conditional_t<
+ Has_cp_async,
+ SM80_CP_ASYNC_CACHEGLOBAL_ZFILL<cute::uint128_t>,
+ AutoVectorizingCopyWithAssumedAlignment<128>
+ >, Element>;
+
+ static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad;
+ static_assert(NumMmaThreads % kGmemThreadsPerRow == 0, "NumMmaThreads must be a multiple of kGmemThreadsPerRow");
+ using GmemLayoutAtom = Layout<Shape <Int<NumMmaThreads / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+ using GmemTiledCopyQKV = decltype(
+ make_tiled_copy(GmemCopyAtom{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per read
+ // So that we don't have to check if we overshot kBlockM when we load Q
+ static_assert(kBlockM % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0);
+
+ // For AppendKV, We want each thread to have at least 2 loads in the K direction since in the case of
+ // non-interleaved rotary (combining elements at indices 0 and rotary_dim/2, 1 and rotary_dim/2+1, etc),
+ // each thread will load twice from the same row.
+ static constexpr int kBytePerHalfRow = kHeadDim / 2 * sizeof(Element);
+ static constexpr int kBlockKGmemAppend = (kBytePerHalfRow % 128 == 0 ? 128 : (kBytePerHalfRow % 64 == 0 ? 64 : 32)) / sizeof(Element);
+ static constexpr int kGmemThreadsPerRowAppend = kBlockKGmemAppend / kGmemElemsPerLoad;
+ static_assert(NumMmaThreads % kGmemThreadsPerRowAppend == 0, "NumMmaThreads must be a multiple of kGmemThreadsPerRowAppend");
+ // We assume threads loading the same row are in the same warp. This is for an optimization in PagedKV where
+ // these threads share the same page table entry and share the work of computing pointers to paged K and paged V.
+ static_assert(cutlass::NumThreadsPerWarp % kGmemThreadsPerRowAppend == 0, "kGmemThreadsPerRowAppend must divide NumThreadsPerWarp");
+ using GmemLayoutAtomAppend = Layout<Shape <Int<NumMmaThreads / kGmemThreadsPerRowAppend>, Int<kGmemThreadsPerRowAppend>>,
+ Stride<Int<kGmemThreadsPerRowAppend>, _1>>;
+ // If AppendKV, we'll be loading Q for rotary, and we assume divisibility to avoid predication
+ static_assert(!AppendKV || kBlockM % CUTE_STATIC_V(shape<0>(GmemLayoutAtomAppend{})) == 0, "kBlockM must be a multiple of NumMmaThreads / kGmemThreadsPerRowAppend");
+ using GmemTiledCopyAppendKV = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ GmemLayoutAtomAppend{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per store
+
+ using ShapeQKV = cute::Shape<int32_t, int32_t, int32_t, int32_t>; // (seqlen, d, head, batch)
+ using StrideQK = cute::Stride<int64_t, _1, int64_t, int64_t>;
+ using StrideV = StrideQK;
+ // ((qhead_per_khead, seqlen_q), d, nheads_kv, batch, num_splits)
+ using ShapeQPacked = std::conditional_t<!PackGQA, ShapeQKV, cute::Shape<cute::Shape<int32_t, int32_t>, int32_t, int32_t, int32_t>>;
+ using StrideQPacked = std::conditional_t<!PackGQA, StrideQK, cute::Stride<cute::Stride<int64_t, int64_t>, _1, int64_t, int64_t>>;
+ using ShapePageTable = cute::Shape<int32_t, int32_t>; // (batch, max_num_pages_per_seq)
+ using StridePageTable = cute::Stride<int64_t, _1>;
+ using ShapeRotary = cute::Shape<int32_t, int32_t>; // (seqlen_ro, rotary_dim // 2)
+ using StrideRotary = cute::Stride<int64_t, _1>;
+ using StrideDescale = cute::Stride<int64_t, int64_t>;
+
+ static constexpr bool Share_QV_Smem = Q_in_regs;
+
+ struct TensorStorageSharedQV : cute::aligned_struct<128> {
+ union {
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutV>> smem_v;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutQ>> smem_q;
+ };
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutK>> smem_k;
+ };
+
+ struct TensorStorageSeparateQV : cute::aligned_struct<128> {
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutV>> smem_v;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutK>> smem_k;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutQ>> smem_q;
+ };
+
+ using TensorStorage = std::conditional_t<Share_QV_Smem, TensorStorageSharedQV, TensorStorageSeparateQV>;
+
+ // Host side kernel arguments
+ struct Arguments {
+ Element const* const ptr_Q;
+ ShapeQKV const shape_Q;
+ StrideQK const stride_Q;
+ Element* const ptr_K; // Not Element const* since we might append to KV cache in-place
+ ShapeQKV const shape_K;
+ StrideQK const stride_K;
+ Element* const ptr_V;
+ StrideV const stride_V;
+ Element const* const ptr_K_new;
+ ShapeQKV const shape_K_new;
+ StrideQK const stride_K_new;
+ Element const* const ptr_V_new;
+ StrideV const stride_V_new;
+ Element const* const ptr_rotary_cos;
+ ShapeRotary const shape_rotary;
+ StrideRotary const stride_rotary_cos;
+ Element const* const ptr_rotary_sin;
+ StrideRotary const stride_rotary_sin;
+ bool const is_rotary_interleaved;
+ int const* const ptr_pagetable;
+ ShapePageTable const shape_pagetable;
+ StridePageTable const stride_pagetable;
+ float const softmax_scale;
+ float const* ptr_q_descale, *ptr_k_descale, *ptr_v_descale;
+ StrideDescale const stride_q_descale, stride_k_descale, stride_v_descale;
+ int const window_size_left = -1, window_size_right = -1, sink_token_length = 0;
+ float const softcap_val;
+ int const num_splits;
+ int const* const kv_batch_idx = nullptr;
+ int const* const cu_seqlens_q = nullptr;
+ int const* const cu_seqlens_k = nullptr;
+ int const* const cu_seqlens_k_new = nullptr;
+ int const* const seqused_q = nullptr;
+ int const* const seqused_k = nullptr;
+ int const* const leftpad_k = nullptr;
+ };
+
+ // Device side kernel params
+ struct Params {
+ Element const* const ptr_Q;
+ ShapeQKV const shape_Q;
+ StrideQK const stride_Q;
+ ShapeQPacked const shape_Q_packed;
+ StrideQPacked const stride_Q_packed;
+ Element* const ptr_K;
+ ShapeQKV const shape_K;
+ StrideQK const stride_K;
+ Element* const ptr_V;
+ StrideV const stride_V;
+ Element const* const ptr_K_new;
+ ShapeQKV const shape_K_new;
+ StrideQK const stride_K_new;
+ Element const* const ptr_V_new;
+ StrideV const stride_V_new;
+ Element const* const ptr_rotary_cos;
+ ShapeRotary const shape_rotary;
+ StrideRotary const stride_rotary_cos;
+ Element const* const ptr_rotary_sin;
+ StrideRotary const stride_rotary_sin;
+ bool const is_rotary_interleaved;
+ int const* const ptr_pagetable;
+ ShapePageTable const shape_pagetable;
+ StridePageTable const stride_pagetable;
+ cutlass::FastDivmod page_size_divmod;
+ cutlass::FastDivmod qhead_per_khead_divmod;
+ float const softmax_scale_log2;
+ float const* ptr_q_descale, *ptr_k_descale, *ptr_v_descale;
+ StrideDescale const stride_q_descale, stride_k_descale, stride_v_descale;
+ float const softcap_val;
+ int const window_size_left, window_size_right, sink_token_length;
+ int const num_splits;
+ int const* const kv_batch_idx = nullptr;
+ int const* const cu_seqlens_q = nullptr;
+ int const* const cu_seqlens_k = nullptr;
+ int const* const cu_seqlens_k_new = nullptr;
+ int const* const seqused_q = nullptr;
+ int const* const seqused_k = nullptr;
+ int const* const leftpad_k = nullptr;
+ };
+
+ static Params
+ to_underlying_arguments(Arguments const& args) {
+ // If PackGQA, reshape Q to be ((qhead_per_khead, seqlen_q), head_size, nhead_k, batch_size)
+ int const qhead_per_khead = !PackGQA ? 1 : cute::ceil_div(get<2>(args.shape_Q), get<2>(args.shape_K));
+ auto const shape_Q_packed = cute::conditional_return<!PackGQA>(
+ args.shape_Q,
+ make_shape(make_shape(qhead_per_khead, get<0>(args.shape_Q)), get<1>(args.shape_Q), get<2>(args.shape_K), get<3>(args.shape_Q))
+ );
+ auto const stride_Q_packed = cute::conditional_return<!PackGQA>(
+ args.stride_Q,
+ make_stride(make_stride(get<2>(args.stride_Q), get<0>(args.stride_Q)), get<1>(args.stride_Q), get<2>(args.stride_Q) * qhead_per_khead, get<3>(args.stride_Q))
+ );
+ if (get<1>(args.shape_rotary) > 0) {
+ assert(args.ptr_rotary_cos != nullptr && args.ptr_rotary_sin != nullptr);
+ }
+ assert(args.num_splits >= 1);
+ // If there's tanh softcapping, we do tanh(scores * softmax_scale / softcap_val) * softcap_val.
+ // Right after this, we multiply by log2(e) before applying exp2.
+ // To reduce the number of instructions, we instead pre-multiply softmax_scale / softcap_val
+ // (assigning it to params.softcap_val) and pre-multiply softcap_val * log2(e)
+ // (assigning it to params.softmax_scale_log2).
+ return {args.ptr_Q, args.shape_Q, args.stride_Q, shape_Q_packed, stride_Q_packed,
+ args.ptr_K, args.shape_K, args.stride_K, args.ptr_V, args.stride_V,
+ args.ptr_K_new, args.shape_K_new, args.stride_K_new, args.ptr_V_new, args.stride_V_new,
+ args.ptr_rotary_cos, args.shape_rotary, args.stride_rotary_cos,
+ args.ptr_rotary_sin, args.stride_rotary_sin, args.is_rotary_interleaved,
+ args.ptr_pagetable, args.shape_pagetable, args.stride_pagetable,
+ cutlass::FastDivmod(int(get<0>(args.shape_K))),
+ cutlass::FastDivmod(cute::ceil_div(get<2>(args.shape_Q), get<2>(args.shape_K))),
+ !Has_softcap ? float(args.softmax_scale * M_LOG2E) : float(args.softcap_val * M_LOG2E),
+ args.ptr_q_descale, args.ptr_k_descale, args.ptr_v_descale,
+ args.stride_q_descale, args.stride_k_descale, args.stride_v_descale,
+ !Has_softcap ? 0.f : args.softmax_scale / args.softcap_val,
+ args.window_size_left, args.window_size_right, args.sink_token_length,
+ !Split ? 1 : args.num_splits,
+ args.kv_batch_idx,
+ args.cu_seqlens_q, args.cu_seqlens_k, args.cu_seqlens_k_new,
+ args.seqused_q, args.seqused_k, args.leftpad_k};
+ }
+
+ CUTLASS_DEVICE
+ cute::tuple<int, int> get_n_block_min_max(Params const& params, SeqlenInfo_t const& seqlen_info,
+ int m_block, int bidb, int split_idx=0, int num_splits=1) {
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ int const seqlen_k = seqlen_info.seqlen_k;
+ int const seqlen_q = seqlen_info.seqlen_q;
+ int n_block_max = cute::ceil_div(seqlen_k, kBlockN);
+ if constexpr (Is_causal || Is_local) {
+ int m_idx_max = (m_block + 1) * kBlockM;
+ if (PackGQA) { m_idx_max = params.qhead_per_khead_divmod.divide(m_idx_max - 1) + 1 ; }
+ n_block_max = std::min(n_block_max,
+ cute::ceil_div(m_idx_max + seqlen_k - seqlen_q + params.window_size_right, kBlockN));
+ }
+ int n_block_min = 0;
+ if constexpr (Is_local) {
+ int m_idx_min = m_block * kBlockM;
+ if (PackGQA) { m_idx_min = params.qhead_per_khead_divmod.divide(m_idx_min); }
+ n_block_min = std::max(int(0), (m_idx_min + seqlen_k - seqlen_q - params.window_size_left) / kBlockN);
+ }
+ // if (threadIdx.x == 128) { printf("Inside, bid.x = %d, bid.y = %d, bid.z = %d, split_idx = %d, n_block_min: %d, n_block_max: %d\n", blockIdx.x, blockIdx.y, blockIdx.z, split_idx, n_block_min, n_block_max); }
+ if constexpr (Split) {
+ int num_n_blocks_per_split = n_block_max <= n_block_min ? 0 : cute::ceil_div(n_block_max - n_block_min, num_splits);
+ n_block_min = n_block_min + split_idx * num_n_blocks_per_split;
+ n_block_max = std::min(n_block_min + num_n_blocks_per_split, n_block_max);
+ }
+ // if (threadIdx.x == 128) { printf("After split, inside, bid.y = %d, bid.z = %d, split_idx = %d, n_block_min: %d, n_block_max: %d\n", blockIdx.y, blockIdx.z, split_idx, n_block_min, n_block_max); }
+ return {n_block_min, n_block_max};
+ }
+
+ template <typename SharedStorage, typename FrgTensorO, typename Softmax>
+ CUTLASS_DEVICE bool
+ mma(Params const& params,
+ FrgTensorO& tOrO,
+ Softmax& softmax,
+ int const thread_idx,
+ SeqlenInfo_t const& seqlen_info,
+ cute::tuple<int32_t, int32_t, int32_t, int32_t> block_coord,
+ SharedStorage& shared_storage
+ ) {
+ static_assert(is_rmem<FrgTensorO>::value, "O tensor must be rmem resident.");
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+
+ // can't use auto [m_block, ...] = block_coord since structured binding cannot be captured in lambda
+ int const m_block = get<0>(block_coord);
+ int const bidh = get<1>(block_coord);
+ int const bidb = get<2>(block_coord);
+ int const split_idx = get<3>(block_coord);
+ int const bidh_kv = !PackGQA ? params.qhead_per_khead_divmod.divide(bidh) : bidh;
+ auto n_block_min_max = get_n_block_min_max(params, seqlen_info, m_block, bidb, split_idx, params.num_splits);
+ int const n_block_min = get<0>(n_block_min_max);
+ int const n_block_max = get<1>(n_block_min_max);
+ // It's possible to have n_block_max <= n_block_min. We don't want to load Q or change any barrier
+ if constexpr (Is_causal || Is_local || Varlen || Split) {
+ if (n_block_max <= n_block_min) { return false; }
+ }
+
+ Tensor sQ = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), SmemLayoutQ{});
+ Tensor sK = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), SmemLayoutK{});
+ Tensor sV = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutV{});
+ Tensor sVt = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutVt{});
+
+ bool const is_varlen_q = Varlen && params.cu_seqlens_q;
+ bool const is_varlen_k = Varlen && params.cu_seqlens_k;
+
+ int const bidb_kv = params.kv_batch_idx == nullptr ? bidb : params.kv_batch_idx[bidb];
+ Tensor mQ = make_tensor(make_gmem_ptr(params.ptr_Q + seqlen_info.offset_q * get<0>(params.stride_Q)), params.shape_Q_packed, params.stride_Q_packed)(_, _, bidh, !is_varlen_q ? bidb : 0);
+ Tensor gQ = local_tile(mQ, select<0, 2>(TileShape_MNK{}), make_coord(m_block, _0{})); // (M, K)
+ Tensor mK = make_tensor(make_gmem_ptr(params.ptr_K + seqlen_info.offset_k * get<0>(params.stride_K)), params.shape_K, params.stride_K)(_, _, bidh_kv, !is_varlen_k ? bidb_kv : 0);
+ Tensor gK = local_tile(mK, select<1, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (N, K, _)
+ Tensor mV = make_tensor(make_gmem_ptr(params.ptr_V + seqlen_info.offset_k * get<0>(params.stride_V)), params.shape_K, params.stride_V)(_, _, bidh_kv, !is_varlen_k ? bidb_kv : 0);
+ Tensor gV = local_tile(mV, select<1, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (N, K, _)
+
+ GmemTiledCopyQKV gmem_tiled_copy_QKV;
+ auto gmem_thr_copy_QKV = gmem_tiled_copy_QKV.get_thread_slice(thread_idx);
+ auto gmem_thr0_copy_QKV = gmem_tiled_copy_QKV.get_thread_slice(_0{}); // For index calculation
+
+ Tensor tKgK = gmem_thr_copy_QKV.partition_S(gK); // (KCPY, KCPY_N, KCPY_K, nblocksN)
+ Tensor tKsK = gmem_thr_copy_QKV.partition_D(sK);
+ Tensor tVgV = gmem_thr_copy_QKV.partition_S(gV); // (VCPY, VCPY_N, VCPY_K, nblocksN)
+ Tensor tVsV = gmem_thr_copy_QKV.partition_D(sV);
+
+ TiledMma tiled_mma;
+ auto thr_mma = tiled_mma.get_slice(thread_idx);
+
+ // Allocate "fragments/descriptors"
+ Tensor tSrQ = thr_mma.partition_fragment_A(sQ);
+
+ // Copy Atom retiling
+ auto smem_tiled_copy_Q = make_tiled_copy_A(SmemCopyAtom{}, tiled_mma);
+ auto smem_thr_copy_Q = smem_tiled_copy_Q.get_thread_slice(thread_idx);
+ auto smem_tiled_copy_K = make_tiled_copy_B(SmemCopyAtom{}, tiled_mma);
+ auto smem_thr_copy_K = smem_tiled_copy_K.get_thread_slice(thread_idx);
+ auto smem_tiled_copy_V = make_tiled_copy_B(SmemCopyAtomTransposed{}, tiled_mma);
+ auto smem_thr_copy_V = smem_tiled_copy_V.get_thread_slice(thread_idx);
+ Tensor tSsQ = smem_thr_copy_Q.partition_S(sQ);
+ Tensor tSsK = smem_thr_copy_K.partition_S(sK);
+ Tensor tOsVt = smem_thr_copy_V.partition_S(sVt);
+
+ // Predicates
+ Tensor cKV = cute::make_identity_tensor(select<1, 2>(TileShape_MNK{}));
+ Tensor tKVcKV = gmem_thr_copy_QKV.partition_S(cKV);
+ Tensor t0KVcKV = gmem_thr0_copy_QKV.partition_S(cKV);
+ Tensor tKVpKV = make_tensor<bool>(make_shape(size<2>(tKsK)));
+ #pragma unroll
+ for (int k = 0; k < size(tKVpKV); ++k) { tKVpKV(k) = get<1>(tKVcKV(_0{}, _0{}, k)) < get<1>(params.shape_K); }
+
+ int const seqlen_q = seqlen_info.seqlen_q;
+ int const seqlen_k = seqlen_info.seqlen_k;
+ int n_block = n_block_max - 1;
+
+ // Prologue: load Q, K, V
+ // If persistent, we don't need to wait for the previous work_idx to finish
+ // since we assume that all MMA threads sync in the epilogue before writing to smem_o.
+ // So any thread gets there, all threads must have finished the previous MMA and at least started
+ // writing to smem_o.
+ // If persistent, need to sync to make sure all threads have finished with smem_o before writing to smem_v
+ if constexpr (Share_QV_Smem) { __syncthreads(); }
+ if constexpr (!PackGQA) {
+ Tensor tQgQ = gmem_thr_copy_QKV.partition_S(gQ);
+ Tensor tQsQ = gmem_thr_copy_QKV.partition_D(sQ);
+ Tensor cQ = cute::make_identity_tensor(select<0, 2>(TileShape_MNK{}));
+ Tensor tQcQ = gmem_thr_copy_QKV.partition_S(cQ);
+ Tensor t0QcQ = gmem_thr0_copy_QKV.partition_S(cQ);
+ Tensor tQpQ = make_tensor<bool>(make_shape(size<2>(tQsQ)));
+ #pragma unroll
+ for (int k = 0; k < size(tQpQ); ++k) { tQpQ(k) = get<1>(tQcQ(_0{}, _0{}, k)) < get<1>(params.shape_Q); }
+ // Instead of passing in tQcQ, we pass in t0QcQ and subtract the offset from the limit
+ // (seqlen_q - m_block * kBlockM). This is because the entries of t0QcQ are known at compile time.
+ // We don't need to clear the sQ smem tiles since we'll only write out the valid outputs
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/true>(
+ gmem_tiled_copy_QKV, tQgQ, tQsQ, t0QcQ, tQpQ, seqlen_info.seqlen_q - m_block * kBlockM - get<0>(tQcQ(_0{}, _0{}, _0{}))
+ );
+ } else {
+ using PackGQAt = flash::PackGQAManager<get<0>(TileShape_MNK{}), get<2>(TileShape_MNK{}), NumMmaThreads, Element>;
+ PackGQAt::load_Q(mQ, sQ, params.qhead_per_khead_divmod, thread_idx, seqlen_q, m_block);
+ }
+ cute::cp_async_fence();
+
+ using PagedKVManager_t = PagedKVManager<get<1>(TileShape_MNK{}), get<2>(TileShape_MNK{}), NumMmaThreads, Element, true /*KV_Same_Iter*/>;
+ PagedKVManager_t paged_kv_manager(
+ params.ptr_pagetable, params.shape_pagetable, params.stride_pagetable,
+ params.ptr_K, params.shape_K, params.stride_K,
+ params.ptr_V, params.stride_V,
+ params.page_size_divmod, bidb_kv, bidh_kv, thread_idx, seqlen_info.seqlen_k, seqlen_info.leftpad_k
+ );
+
+ auto load_K = [&] (int const n_block, int const smem_pipe_write, auto need_seqlenk_masking_type) {
+ static constexpr bool Seqlenk_mask = decltype(need_seqlenk_masking_type)::value;
+ if constexpr (!PagedKV) {
+ // Do we need bound check to make sure the row doesn't go above kBlockN
+ static constexpr bool EvenN = kBlockN % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0;
+ Tensor tKsK_cur = tKsK(_, _, _, smem_pipe_write);
+ // Instead of passing in tKVcKV, we pass in t0KVcKV and subtract the offset from the limit
+ // (seqlen_k - n_block * kBlockN). This is because the entries of t0KVcKV are known at compile time.
+ int const seqlenk_row_limit = -int(get<0>(tKVcKV(_0{}, _0{}, _0{}))) + (EvenN
+ ? seqlen_info.seqlen_k - n_block * kBlockN
+ : (!Seqlenk_mask ? kBlockN : std::min(seqlen_info.seqlen_k - n_block * kBlockN, kBlockN)));
+ // We don't need to clear the sK smem tiles since we'll mask out the scores anyway.
+ flash::copy</*Is_even_MN=*/!Seqlenk_mask && EvenN, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/true>(
+ gmem_tiled_copy_QKV, tKgK(_, _, _, n_block), tKsK_cur, t0KVcKV, tKVpKV, seqlenk_row_limit);
+ } else {
+ paged_kv_manager.template load_page_table<Seqlenk_mask>(n_block);
+ paged_kv_manager.template load_K<Seqlenk_mask>(n_block, sK(_, _, smem_pipe_write));
+ }
+ };
+
+ auto load_V = [&] (int const n_block, int const smem_pipe_write, auto need_seqlenk_masking_type) {
+ static constexpr bool Seqlenk_mask = decltype(need_seqlenk_masking_type)::value;
+ if constexpr (!PagedKV) {
+ // Do we need bound check to make sure the row doesn't go above kBlockN
+ static constexpr bool EvenN = kBlockN % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0;
+ Tensor tVsV_cur = tVsV(_, _, _, smem_pipe_write);
+ // We don't call flash::copy since it doesn't support bound checking
+ // to not overshot kBlockN when writing to smem.
+ Tensor tVgV_cur = tVgV(_, _, _, n_block);
+ int const seqlenk_row_limit = seqlen_info.seqlen_k - n_block * kBlockN - get<0>(tKVcKV(_0{}, _0{}, _0{}));
+ #pragma unroll
+ for (int m = 0; m < size<1>(tVsV); ++m) {
+ // If kBlockN doesn't evenly divide the tiled copy, only the last `m` needs to be checked
+ if (EvenN || m < size<1>(tVsV) - 1 || get<0>(tKVcKV(_0{}, m, _0{})) < kBlockN) {
+ bool const predicate_n = !Seqlenk_mask || get<0>(t0KVcKV(_0{}, m, _0{})) < seqlenk_row_limit;
+ #pragma unroll
+ for (int k = 0; k < size<2>(tVsV); ++k) {
+ cute::copy(gmem_tiled_copy_QKV.with(tKVpKV(k) && predicate_n), tVgV_cur(_, m, k), tVsV_cur(_, m, k));
+ }
+ }
+ }
+ } else {
+ paged_kv_manager.template load_V<Seqlenk_mask>(n_block, sV(_, _, smem_pipe_write));
+ }
+ };
+
+ auto preprocess_Q = [&] {
+ if constexpr (!AppendKV) {
+ flash::cp_async_wait<Share_QV_Smem ? 1 : kStages * 2 - 1>();
+ } else {
+ if (get<1>(params.shape_rotary) > 0) { // Apply rotary to Q
+ int const offset_rotary = seqlen_info.seqlen_k_og + seqlen_info.leftpad_k;
+ using Rotary_t = Rotary<kBlockM, kHeadDim, NumMmaThreads, Element, !(Is_causal || Is_local) /*FixedPosition*/>;
+ Rotary_t rotary(params.ptr_rotary_cos, params.shape_rotary, params.stride_rotary_cos,
+ params.ptr_rotary_sin, params.stride_rotary_sin,
+ params.is_rotary_interleaved, thread_idx, seqlen_q, offset_rotary);
+ int const qhead_per_khead = !PackGQA ? 1 : params.qhead_per_khead_divmod.divisor;
+ if (params.is_rotary_interleaved) {
+ auto [tRrCos, tRrSin] = cute::conditional_return<!PackGQA>(
+ rotary.template load_cos_sin<true /*kInterleaved*/>(m_block),
+ rotary.template load_cos_sin_packgqa<true /*kInterleaved*/>(m_block, params.qhead_per_khead_divmod)
+ );
+ flash::cp_async_wait<Share_QV_Smem ? 1 : kStages * 2 - 1>();
+ __syncthreads();
+ rotary.apply_Q_interleaved(sQ, tRrCos, tRrSin, m_block, qhead_per_khead);
+ } else {
+ auto [tRrCosCont, tRrSinCont] = cute::conditional_return<!PackGQA>(
+ rotary.template load_cos_sin<false /*kInterleaved*/>(m_block),
+ rotary.template load_cos_sin_packgqa<false /*kInterleaved*/>(m_block, params.qhead_per_khead_divmod)
+ );
+ flash::cp_async_wait<Share_QV_Smem ? 1 : kStages * 2 - 1>();
+ __syncthreads();
+ rotary.apply_Q_contiguous(sQ, tRrCosCont, tRrSinCont, m_block, qhead_per_khead);
+ }
+ } else {
+ flash::cp_async_wait<Share_QV_Smem ? 1 : kStages * 2 - 1>();
+ }
+ }
+
+ if constexpr (Q_in_regs) {
+ __syncthreads();
+ Tensor tSrQ_copy_view = smem_thr_copy_Q.retile_D(tSrQ);
+ Tensor tSsQ_copy_view = smem_thr_copy_Q.partition_S(sQ);
+ cute::copy(smem_tiled_copy_Q, tSsQ_copy_view, tSrQ_copy_view);
+ }
+ };
+
+ // If Share_QV_Smem, we load Q, then load 1 stage of K, then (optionally) rotate Q and
+ // read from smem_q to registers, then load V.
+ // If !Share_QV, Smem, we load Q, load all stages of K & V, then (optionally) rotate Q.
+
+ if constexpr (Share_QV_Smem) {
+ load_K(n_block, 0, cute::true_type{} /*Seqlenk_mask*/);
+ cute::cp_async_fence();
+ preprocess_Q();
+ __syncthreads(); // Make sure all threads have read smem_q before loading V
+ }
+
+ // For persistent, make sure all threads have finished reading smem_o
+ if constexpr (!Share_QV_Smem) { __syncthreads(); }
+ // Note, using the for_each() function here to ensure `stage` is of type Int<x>.
+ for_each(make_int_sequence<kStages>{}, [&] (auto stage) {
+ static constexpr bool Is_first_stage = CUTE_STATIC_V(stage) == 0;
+ static constexpr bool Is_last_stage = CUTE_STATIC_V(stage) == kStages - 1;
+ if constexpr (!Share_QV_Smem || !Is_first_stage) {
+ if (Is_first_stage || n_block - stage >= n_block_min) {
+ load_K(n_block - stage, stage, cute::bool_constant<Is_first_stage>{} /*Seqlenk_mask*/);
+ }
+ // We want the fence outside the if statement to have a fixed number of cp.async commits.
+ // so that we can wait with the correct number of outstanding commits.
+ cute::cp_async_fence();
+ }
+ if constexpr (!Is_last_stage) {
+ if (Is_first_stage || n_block - stage >= n_block_min) {
+ load_V(n_block - stage, stage, cute::bool_constant<Is_first_stage>{} /*Seqlenk_mask*/);
+ }
+ cute::cp_async_fence();
+ }
+ });
+
+ if constexpr (!Share_QV_Smem) { preprocess_Q(); }
+
+ flash::Mask<kBlockM, kBlockN, PackGQA, TiledMma> mask(
+ thread_idx, seqlen_q, seqlen_k, params.window_size_left, params.window_size_right, params.sink_token_length,
+ params.qhead_per_khead_divmod
+ );
+
+ float softcap_val = params.softcap_val;
+ if constexpr (Has_softcap && Is_FP8) {
+ float const q_descale = params.ptr_q_descale == nullptr ? 1.0f : params.ptr_q_descale[bidb * get<0>(params.stride_q_descale) + bidh_kv * get<1>(params.stride_q_descale)];
+ float const k_descale = params.ptr_k_descale == nullptr ? 1.0f : params.ptr_k_descale[bidb * get<0>(params.stride_k_descale) + bidh_kv * get<1>(params.stride_k_descale)];
+ softcap_val *= q_descale * k_descale;
+ }
+ // Softcapping needs to happen before masking since if we apply after masking, softcapping can turn
+ // -inf to e.g. -50.0, which can affect the attention softmax.
+ auto scoremod_premask_fn = [&](auto& tSrS) {
+ if constexpr (Has_softcap) { flash::apply_softcap(tSrS, softcap_val); }
+ };
+
+ int smem_pipe_read = 0, smem_pipe_write = kStages - 1;
+
+ auto load_K_next = [&] {
+ if (n_block - kStages >= n_block_min) {
+ load_K(n_block - kStages, kStages > 1 ? smem_pipe_write : 0, cute::false_type{} /*Seqlenk_mask*/);
+ }
+ cute::cp_async_fence();
+ };
+
+ auto sync = [&] {
+ flash::cp_async_wait<kStages * 2 - 2>();
+ __syncthreads();
+ };
+
+ clear(tOrO);
+
+ auto fwd_step = [&](int const n_block, auto mask_fn, auto is_first_iter_type, auto check_inf_type) {
+ static constexpr bool Is_first_iter = decltype(is_first_iter_type)::value;
+ static constexpr bool Check_inf = decltype(check_inf_type)::value;
+ Tensor tSrS = partition_fragment_C(tiled_mma, select<0, 1>(TileShape_MNK{}));
+ clear(tSrS);
+ sync();
+ auto load_V_next = [&] {
+ if (n_block - kStages + 1 >= n_block_min) {
+ load_V(n_block - kStages + 1, kStages > 1 ? smem_pipe_write : 0, cute::bool_constant<Is_first_iter && kStages == 1>{} /*Seqlenk_mask*/);
+ }
+ cute::cp_async_fence();
+ };
+ Tensor tSrQ_cur = cute::conditional_return<Q_in_regs>(tSrQ, thr_mma.partition_fragment_A(sQ));
+ Tensor tSrK = thr_mma.partition_fragment_B(sK(_, _, _0{}));
+ flash::gemm_sm80<Q_in_regs>(
+ tSrS, tSrQ_cur, tSrK, tSsQ, tSsK(_, _, _, kStages > 1 ? smem_pipe_read : 0),
+ tiled_mma, smem_tiled_copy_Q, smem_tiled_copy_K, smem_thr_copy_Q, smem_thr_copy_K, load_V_next
+ );
+ smem_pipe_write = smem_pipe_write < kStages - 1 ? smem_pipe_write + 1 : 0;
+ scoremod_premask_fn(tSrS);
+ // Faster to load_K before gemm if we only have 1 stage
+ if constexpr (kStages == 1) { sync(); load_K_next(); }
+ mask_fn(tSrS, n_block);
+ Tensor scores_scale = softmax.template max_get_scale</*Is_first=*/Is_first_iter, Check_inf>(tSrS);
+ softmax.template online_softmax</*Is_first=*/Is_first_iter, Check_inf>(tSrS);
+ if constexpr (Is_FP8) { flash::permute_Cregs_fp8(tSrS); }
+ Tensor tOrP_acc = make_tensor(tSrS.data(), flash::convert_layout_acc_Aregs<TiledMma>(tSrS.layout()));
+ Tensor tOrP = make_tensor_like<Element>(tOrP_acc);
+ convert_type_out(tOrP_acc, tOrP);
+ if constexpr (!Is_first_iter) { softmax.rescale_o(tOrO, scores_scale); }
+ if constexpr (kStages > 1) { sync(); }
+ Tensor tOrV = thr_mma.partition_fragment_B(sVt(_, _, _0{}));
+ flash::gemm_rs_sm80(tOrO, tOrP, tOrV, tOsVt(_, _, _, kStages > 1 ? smem_pipe_read : 0), tiled_mma, smem_tiled_copy_V, smem_thr_copy_V);
+ if constexpr (kStages > 1) { load_K_next(); }
+ smem_pipe_read = smem_pipe_read < kStages - 1 ? smem_pipe_read + 1 : 0;
+ };
+
+ auto first_iter_mask_fn = [&](auto& tSrS, int n_block) { mask.template apply<true /*Seqlenk_mask*/, Is_causal, Is_local>(tSrS, m_block, n_block); };
+ fwd_step(n_block, first_iter_mask_fn, cute::true_type{} /*is_first_iter*/, cute::true_type{} /*check_inf*/);
+ --n_block;
+ if constexpr (Is_causal || Is_local) { // Separate iterations with causal or local masking
+ auto mask_fn = [&](auto& tSrS, int n_block) { mask.template apply<false /*Seqlenk_mask*/, Is_causal, Is_local>(tSrS, m_block, n_block); };
+ int const m_idx_min = !PackGQA ? m_block * kBlockM : params.qhead_per_khead_divmod.divide(m_block * kBlockM);
+ int const n_block_min_causal_local_mask =
+ std::max(n_block_min, (m_idx_min + seqlen_k - seqlen_q + params.window_size_right) / kBlockN);
+ #pragma unroll 1
+ for (; n_block >= n_block_min_causal_local_mask; --n_block) {
+ fwd_step(n_block, mask_fn, cute::false_type{} /*is_first_iter*/, cute::true_type{} /*check_inf*/);
+ }
+ }
+ int const m_idx_max = !PackGQA ? (m_block + 1) * kBlockM : params.qhead_per_khead_divmod.divide((m_block + 1) * kBlockM - 1) + 1;
+ int const n_block_min_before_local_mask = !Is_local
+ ? n_block_min
+ : std::max(n_block_min,
+ cute::ceil_div(m_idx_max + seqlen_k - seqlen_q - params.window_size_left, kBlockN));
+ auto no_mask_fn = [](auto& tSrS, int n_block) { };
+ #pragma unroll 1
+ for (; n_block >= n_block_min_before_local_mask; --n_block) {
+ fwd_step(n_block, no_mask_fn, cute::false_type{} /*is_first_iter*/, cute::false_type{} /*check_inf*/);
+ }
+ // Separate masking iterations on the left for local attention
+ if constexpr (Is_local) {
+ auto local_mask_fn = [&](auto& tSrS, int n_block) { mask.template apply<false /*Seqlenk_mask*/, false /*Causal_mask*/, Is_local>(tSrS, m_block, n_block); };
+ #pragma unroll 1
+ for (; n_block >= n_block_min; --n_block) {
+ fwd_step(n_block, local_mask_fn, cute::false_type{} /*is_first_iter*/, cute::bool_constant<Is_local>{} /*check_inf*/);
+ }
+ // Disable sink token code for now
+ // int n_block_sink_max = cute::ceil_div(params.sink_token_length, kBlockN);
+ // #pragma unroll 1
+ // for (n_block = std::min(n_block, n_block_sink_max - 1); n_block >= 0; --n_block) {
+ // fwd_step(n_block, local_mask_fn, cute::false_type{} /*is_first_iter*/, cute::bool_constant<Is_local>{} /*check_inf*/);
+ // }
+ }
+ float const v_descale = !Is_FP8 || params.ptr_v_descale == nullptr ? 1.0f : params.ptr_v_descale[bidb * get<0>(params.stride_v_descale) + bidh_kv * get<1>(params.stride_v_descale)];
+ Tensor scores_scale = softmax.finalize(v_descale);
+ softmax.rescale_o(tOrO, scores_scale);
+ if constexpr (Is_FP8) { flash::permute_output_fp8(tOrO); }
+ return true;
+ }
+
+ CUTLASS_DEVICE
+ cute::tuple<int, int> get_n_block_k_new_min_max(Params const& params, SeqlenInfo_t const& seqlen_info,
+ int m_block, int bidb, int split_idx=0, int num_splits=1) {
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ auto [n_block_min, n_block_max] = get_n_block_min_max(params, seqlen_info, m_block, bidb, split_idx, num_splits);
+ int const idx_k_new_min = std::max(n_block_min * kBlockN - seqlen_info.seqlen_k_og, 0);
+ int const idx_k_new_max = std::min(n_block_max * kBlockN - seqlen_info.seqlen_k_og, seqlen_info.seqlen_k_new);
+ int const n_block_new_min = idx_k_new_min / kBlockN;
+ int const n_block_new_max = idx_k_new_max > idx_k_new_min ? cute::ceil_div(idx_k_new_max, kBlockN) : n_block_new_min;
+ // if (threadIdx.x == 128 && m_block == 0) { printf("bidb = %d, seqlen_k_new = %d, seqlen_k_og = %d, n_block_min = %d, n_block_max = %d, idx_k_new_min = %d, idx_k_new_max = %d, n_block_new_min = %d, n_block_new_max = %d\n", bidb, seqlen_k_new, seqlen_k_og, n_block_min, n_block_max, idx_k_new_min, idx_k_new_max, n_block_new_min, n_block_new_max);}
+ return {n_block_new_min, n_block_new_max};
+ }
+
+ template <typename SharedStorage>
+ CUTLASS_DEVICE bool
+ store_kv_new(Params const& params,
+ int const thread_idx,
+ SharedStorage &shared_storage,
+ SeqlenInfo_t const& seqlen_info,
+ cute::tuple<int32_t, int32_t, int32_t, int32_t> block_coord
+ ) {
+ auto [m_block, bidh, bidb, split_idx] = block_coord;
+ auto n_block_new_min_max = get_n_block_k_new_min_max(params, seqlen_info, m_block, bidb, split_idx, params.num_splits);
+ int const n_block_new_min = get<0>(n_block_new_min_max);
+ int const n_block_new_max = get<1>(n_block_new_min_max);
+ if (n_block_new_max <= n_block_new_min) { return false; }
+
+ Tensor sK = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), SmemLayoutK{});
+ Tensor sV = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutV{});
+
+ int const bidh_kv = !PackGQA ? params.qhead_per_khead_divmod.divide(bidh) : bidh;
+ int const bidb_kv = params.kv_batch_idx == nullptr ? bidb : params.kv_batch_idx[bidb];
+
+ bool const is_varlen_k_new = Varlen && params.cu_seqlens_k_new;
+ Tensor mKnew = make_tensor(make_gmem_ptr(params.ptr_K_new), params.shape_K_new, params.stride_K_new)(_, _, bidh_kv, !is_varlen_k_new ? bidb : 0);
+ Tensor mVnew = make_tensor(make_gmem_ptr(params.ptr_V_new), params.shape_K_new, params.stride_V_new)(_, _, bidh_kv, !is_varlen_k_new ? bidb : 0);
+
+ bool const is_varlen_k = Varlen && params.cu_seqlens_k;
+ Tensor mK = make_tensor(make_gmem_ptr(params.ptr_K), params.shape_K, params.stride_K)(_, _, bidh_kv, !is_varlen_k ? bidb_kv : 0);
+ Tensor mV = make_tensor(make_gmem_ptr(params.ptr_V), params.shape_K, params.stride_V)(_, _, bidh_kv, !is_varlen_k ? bidb_kv : 0);
+
+ Tensor gKnew = local_tile(domain_offset(make_coord(seqlen_info.offset_k_new, _0{}), mKnew), select<1, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (N, K, _)
+ Tensor gVnew = local_tile(domain_offset(make_coord(seqlen_info.offset_k_new, _0{}), mVnew), select<1, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (N, K, _)
+ int const offset_k = seqlen_info.offset_k + seqlen_info.seqlen_k_og;
+ Tensor gK = local_tile(domain_offset(make_coord(offset_k, _0{}), mK), select<1, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (N, K, _)
+ Tensor gV = local_tile(domain_offset(make_coord(offset_k, _0{}), mV), select<1, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (N, K, _)
+
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ static constexpr int kHeadDim = get<2>(TileShape_MNK{});
+ int const offset_rotary = seqlen_info.seqlen_k_og + seqlen_info.leftpad_k;
+ int const seqlen_k_new = seqlen_info.seqlen_k_new;
+ using Rotary_t = Rotary<kBlockN, kHeadDim, NumMmaThreads, Element>;
+ Rotary_t rotary(params.ptr_rotary_cos, params.shape_rotary, params.stride_rotary_cos,
+ params.ptr_rotary_sin, params.stride_rotary_sin,
+ params.is_rotary_interleaved, thread_idx, seqlen_k_new, offset_rotary);
+
+ using PagedKVManager_t = PagedKVManager<get<1>(TileShape_MNK{}), get<2>(TileShape_MNK{}), NumMmaThreads, Element, true /*KV_Same_Iter*/, 2 /*LoadsPerRow_LB*/>;
+ PagedKVManager_t paged_kv_manager(
+ params.ptr_pagetable, params.shape_pagetable, params.stride_pagetable,
+ params.ptr_K, params.shape_K, params.stride_K,
+ params.ptr_V, params.stride_V,
+ params.page_size_divmod, bidb_kv, bidh_kv, thread_idx, seqlen_k_new, offset_k
+ // passing offset_k instead of leftpad_k will move the PageTable pointer to the right position
+ );
+
+ static_assert(std::is_same_v<GmemLayoutAtomAppend, typename Rotary_t::LayoutAtom>);
+ static_assert(!PagedKV || std::is_same_v<GmemLayoutAtomAppend, typename PagedKVManager_t::GmemLayoutAtomKVCpAsync>);
+ GmemTiledCopyQKV gmem_tiled_copy_kv_g2s;
+ auto gmem_thr_copy_kv_g2s = gmem_tiled_copy_kv_g2s.get_thread_slice(thread_idx);
+ auto gmem_thr0_copy_kv_g2s = gmem_tiled_copy_kv_g2s.get_thread_slice(_0{}); // Only for index calculation
+ GmemTiledCopyAppendKV gmem_tiled_copy_kv_s2g;
+ auto gmem_thr_copy_kv_s2g = gmem_tiled_copy_kv_s2g.get_thread_slice(thread_idx);
+ auto gmem_thr0_copy_kv_s2g = gmem_tiled_copy_kv_s2g.get_thread_slice(_0{}); // Only for index calculation
+ Tensor tKgKnew = gmem_thr_copy_kv_g2s.partition_S(gKnew);
+ Tensor tKsKg2s = gmem_thr_copy_kv_g2s.partition_S(sK);
+ Tensor tKsKs2g = gmem_thr_copy_kv_s2g.partition_S(sK); // ((Atom,AtomNum),ATOM_M,ATOM_N)
+ Tensor tKgK = gmem_thr_copy_kv_s2g.partition_D(gK);
+ Tensor tVgVnew = gmem_thr_copy_kv_g2s.partition_S(gVnew); // ((Atom,AtomNum),ATOM_M,ATOM_N)
+ Tensor tVsVg2s = gmem_thr_copy_kv_g2s.partition_S(sV); // ((Atom,AtomNum),ATOM_M,ATOM_N)
+ Tensor tVsVs2g = gmem_thr_copy_kv_s2g.partition_S(sV); // ((Atom,AtomNum),ATOM_M,ATOM_N)
+ Tensor tVgV = gmem_thr_copy_kv_s2g.partition_D(gV);
+ Tensor cK = cute::make_identity_tensor(select<1, 2>(TileShape_MNK{})); // (BLK_N,BLK_K) -> (blk_n,blk_k)
+ Tensor tKcKg2s = gmem_thr_copy_kv_g2s.partition_D(cK);
+ Tensor t0KcKg2s = gmem_thr0_copy_kv_g2s.partition_D(cK);
+ Tensor tKpKg2s = make_tensor<bool>(make_shape(size<2>(tKsKg2s)));
+ Tensor tKcKs2g = gmem_thr_copy_kv_s2g.partition_D(cK);
+ Tensor t0KcKs2g = gmem_thr0_copy_kv_s2g.partition_D(cK);
+ Tensor tKpKs2g = make_tensor<bool>(make_shape(size<2>(tKsKs2g)));
+ #pragma unroll
+ for (int k = 0; k < size(tKpKg2s); ++k) { tKpKg2s(k) = get<1>(tKcKg2s(_0{}, _0{}, k)) < get<1>(params.shape_K); }
+ #pragma unroll
+ for (int k = 0; k < size(tKpKs2g); ++k) { tKpKs2g(k) = get<1>(tKcKs2g(_0{}, _0{}, k)) < get<1>(params.shape_K); }
+
+ auto load_K_new = [&] (int const n_block, int const smem_pipe_write, auto need_seqlenk_masking_type) {
+ static constexpr bool Seqlenk_mask = decltype(need_seqlenk_masking_type)::value;
+ static constexpr bool EvenN = kBlockN % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0;
+ Tensor tKsK_cur = tKsKg2s(_, _, _, smem_pipe_write);
+ int const seqlenk_row_limit = -int(get<0>(tKcKg2s(_0{}, _0{}, _0{}))) + (EvenN
+ ? seqlen_k_new - n_block * kBlockN
+ : (!Seqlenk_mask ? kBlockN : std::min(seqlen_k_new - n_block * kBlockN, kBlockN)));
+ // We don't need to clear the sK smem tiles since we won't write them out
+ flash::copy</*Is_even_MN=*/!Seqlenk_mask && EvenN, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/true>(
+ gmem_tiled_copy_kv_g2s, tKgKnew(_, _, _, n_block), tKsK_cur, t0KcKg2s, tKpKg2s, seqlenk_row_limit);
+ };
+
+ auto load_V_new = [&] (int const n_block, int const smem_pipe_write, auto need_seqlenk_masking_type) {
+ static constexpr bool Seqlenk_mask = decltype(need_seqlenk_masking_type)::value;
+ static constexpr bool EvenN = kBlockN % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0;
+ Tensor tVsV_cur = tVsVg2s(_, _, _, smem_pipe_write);
+ int const seqlenk_row_limit = -int(get<0>(tKcKg2s(_0{}, _0{}, _0{}))) + (EvenN
+ ? seqlen_k_new - n_block * kBlockN
+ : (!Seqlenk_mask ? kBlockN : std::min(seqlen_k_new - n_block * kBlockN, kBlockN)));
+ // We don't need to clear the sV smem tiles since we won't write them out
+ flash::copy</*Is_even_MN=*/!Seqlenk_mask && EvenN, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/true>(
+ gmem_tiled_copy_kv_g2s, tVgVnew(_, _, _, n_block), tVsV_cur, t0KcKg2s, tKpKg2s, seqlenk_row_limit);
+ };
+
+ auto store_K = [&] (int const n_block, int const smem_pipe_read) {
+ int const n_limit = std::min(seqlen_k_new - n_block * kBlockN, kBlockN);
+ if (get<1>(params.shape_rotary) <= 0) {
+ Tensor tKsK_cur = tKsKs2g(_, _, _, smem_pipe_read);
+ if constexpr (!PagedKV) {
+ Tensor tKgK_cur = tKgK(_, _, _, n_block);
+ // Clear_OOB_K must be false since we don't want to write zeros to gmem
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/false>(
+ gmem_tiled_copy_kv_s2g, tKsK_cur, tKgK_cur, tKcKs2g, tKpKs2g, std::min(seqlen_k_new - n_block * kBlockN, kBlockN)
+ );
+ } else {
+ paged_kv_manager.store_K(n_block, tKsK_cur);
+ }
+ } else {
+ Tensor gK_cur = gK(_, _, n_block);
+ auto tPrKPtr = cute::conditional_return<PagedKV>(paged_kv_manager.compute_K_ptr(), nullptr);
+ if (params.is_rotary_interleaved) {
+ auto [tRrCos, tRrSin] = rotary.template load_cos_sin<true /*kInterleaved*/>(n_block);
+ rotary.template apply_K_interleaved<PagedKV>(sK(_, _, smem_pipe_read), gK_cur, tKpKs2g, tRrCos, tRrSin, tPrKPtr, n_block);
+ } else {
+ auto [tRrCosCont, tRrSinCont] = rotary.template load_cos_sin<false /*kInterleaved*/>(n_block);
+ rotary.template apply_K_contiguous<PagedKV>(sK(_, _, smem_pipe_read), gK_cur, tKpKs2g, tRrCosCont, tRrSinCont, tPrKPtr, n_block, get<1>(params.shape_K));
+ }
+ }
+ };
+
+ auto store_V = [&] (int const n_block, int const smem_pipe_read) {
+ int const n_limit = std::min(seqlen_k_new - n_block * kBlockN, kBlockN);
+ Tensor tVsV_cur = tVsVs2g(_, _, _, smem_pipe_read);
+ if constexpr (!PagedKV) {
+ Tensor tVgV_cur = tVgV(_, _, _, n_block);
+ // Clear_OOB_K must be false since we don't want to write zeros to gmem
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/false>(
+ gmem_tiled_copy_kv_s2g, tVsV_cur, tVgV_cur, tKcKs2g, tKpKs2g, n_limit);
+ } else {
+ paged_kv_manager.store_V(n_block, tVsV_cur);
+ }
+ };
+
+ int n_block = n_block_new_max - 1;
+ // Note, using the for_each() function here to ensure `stage` is of type Int<x>.
+ for_each(make_int_sequence<kStages>{}, [&] (auto stage) {
+ static constexpr bool Is_first_stage = CUTE_STATIC_V(stage) == 0;
+ static constexpr bool Is_last_stage = CUTE_STATIC_V(stage) == kStages - 1;
+ if (Is_first_stage || n_block - stage >= n_block_new_min) {
+ load_K_new(n_block - stage, stage, cute::bool_constant<Is_first_stage>{} /*Seqlenk_mask*/);
+ }
+ cute::cp_async_fence();
+ // If persistent, need to sync to make sure all threads have finished with smem_o before writing to smem_v
+ if constexpr (Is_first_stage) { __syncthreads(); }
+ if constexpr (!Is_last_stage) {
+ if (Is_first_stage || n_block - stage >= n_block_new_min) {
+ load_V_new(n_block - stage, stage, cute::bool_constant<Is_first_stage>{} /*Seqlenk_mask*/);
+ }
+ cute::cp_async_fence();
+ }
+ });
+
+ int smem_pipe_read = 0, smem_pipe_write = kStages - 1;
+ #pragma unroll 1
+ for (; n_block >= n_block_new_min; --n_block) {
+ if constexpr (PagedKV) { paged_kv_manager.template load_page_table<true /*Seqlenk_mask*/>(n_block); }
+ flash::cp_async_wait<kStages * 2 - 2>();
+ __syncthreads();
+ store_K(n_block, kStages > 1 ? smem_pipe_read : 0);
+ if (n_block - kStages + 1 >= n_block_new_min) {
+ load_V_new(n_block - kStages + 1, kStages > 1 ? smem_pipe_write : 0, cute::bool_constant<kStages == 1>{} /*Seqlenk_mask*/);
+ }
+ cute::cp_async_fence();
+ smem_pipe_write = smem_pipe_write < kStages - 1 ? smem_pipe_write + 1 : 0;
+ flash::cp_async_wait<kStages * 2 - 2>();
+ __syncthreads();
+ store_V(n_block, kStages > 1 ? smem_pipe_read : 0);
+ smem_pipe_read = smem_pipe_read < kStages - 1 ? smem_pipe_read + 1 : 0;
+ if (n_block - kStages >= n_block_new_min) {
+ load_K_new(n_block - kStages, kStages > 1 ? smem_pipe_write : 0, cute::false_type{} /*Seqlenk_mask*/);
+ }
+ cute::cp_async_fence();
+ }
+
+ return true;
+
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/mainloop_fwd_sm90_tma_gmma_ws.hpp b/src/solarwm/backends/minimax_h3/hopper_src/mainloop_fwd_sm90_tma_gmma_ws.hpp
new file mode 100644
index 0000000..dbbf2f8
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/mainloop_fwd_sm90_tma_gmma_ws.hpp
@@ -0,0 +1,1428 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cutlass/cutlass.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_types.h>
+#include <cutlass/numeric_conversion.h>
+#include "cutlass/pipeline/pipeline.hpp"
+
+#include "cute/tensor.hpp"
+
+#include "cutlass/gemm/collective/builders/sm90_common.inl"
+
+#include "named_barrier.hpp"
+#include "seqlen.h"
+#include "mask.h"
+#include "pack_gqa.h"
+#include "paged_kv.h"
+#include "rotary.h"
+#include "utils.h"
+#include "sm90_pipeline_no_cluster.hpp"
+
+namespace flash {
+
+using namespace cute;
+
+template <int Stages, class ClusterShape_, class TileShape_MNK_, class Element_, class ElementAccum_, class ArchTag_,
+ bool Is_causal_, bool Is_local_, bool Has_softcap_, bool Varlen_, bool PagedKV_, bool AppendKV_,
+ bool Mma1_is_RS, bool IntraWGOverlap, bool PackGQA_, bool Split_, bool V_colmajor_>
+struct CollectiveMainloopFwdSm90 {
+
+ static constexpr int kStages = Stages;
+ using ClusterShape = ClusterShape_;
+ using TileShape_MNK = TileShape_MNK_;
+ using Element = Element_;
+ using ElementAccum = ElementAccum_;
+ using ArchTag = ArchTag_;
+ static constexpr bool Is_FP8 = cute::is_same_v<Element, cutlass::float_e4m3_t> || cute::is_same_v<Element, cutlass::float_e5m2_t>;;
+ static constexpr bool Is_causal = Is_causal_;
+ static constexpr bool Is_local = Is_local_;
+ static constexpr bool Has_softcap = Has_softcap_;
+ static constexpr bool Varlen = Varlen_;
+ static constexpr bool PagedKV = PagedKV_;
+ static constexpr bool AppendKV = AppendKV_;
+ static constexpr bool PackGQA = PackGQA_;
+ static constexpr bool Split = Split_;
+ static constexpr bool V_colmajor = V_colmajor_;
+ static constexpr bool Transpose_V = Is_FP8 && !V_colmajor;
+ static constexpr bool Use_TMA_Q = !PackGQA;
+ static constexpr bool Use_TMA_KV = !PagedKV;
+ static_assert(Use_TMA_KV || CUTE_STATIC_V(size(ClusterShape{})) == 1, "If not using TMA for KV, ClusterShape must be 1");
+ static_assert(Use_TMA_KV || !V_colmajor, "If not using TMA for KV, V_colmajor is not supported");
+ using SeqlenInfo_t = flash::SeqlenInfoQKNewK<Varlen, AppendKV>;
+
+ static_assert(ArchTag::kMinComputeCapability >= 90);
+
+ static constexpr cute::GMMA::Major MmaMajorV = !Is_FP8 && !V_colmajor ? GMMA::Major::MN : GMMA::Major::K;
+ static constexpr cute::GMMA::Major TmaMajorV = !V_colmajor ? GMMA::Major::MN : GMMA::Major::K;
+
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ static constexpr int kHeadDim = get<2>(TileShape_MNK{});
+
+ // Register bandwidth is actually a bottleneck so we don't want Q to be in registers.
+ // Leaving this option here for reference.
+ static constexpr bool Mma0_is_RS = false;
+ // We can have Mma1 (P @ V) with P in smem in rmem to reduce register pressure at the cost of more smem.
+ static_assert(!(!Mma1_is_RS && !IntraWGOverlap), "Mma1 must be RS if IntraWGOverlap is enabled");
+ static_assert(!(!Mma1_is_RS && Is_FP8), "Mma1 must be RS if FP8");
+ static_assert(!(!Mma1_is_RS && Transpose_V), "Mma1 must be RS if Transpose_V");
+
+ using AtomLayoutMNK = Layout<Shape<Int<kBlockM / 64>, _1, _1>>;
+ using TiledMma0 = decltype(cute::make_tiled_mma(
+ std::conditional_t<
+ !Mma0_is_RS,
+ decltype(cute::GMMA::ss_op_selector<Element, Element, ElementAccum, TileShape_MNK>()),
+ decltype(cute::GMMA::rs_op_selector<Element, Element, ElementAccum, TileShape_MNK>())
+ >{},
+ AtomLayoutMNK{}));
+ using TiledMma1 = decltype(cute::make_tiled_mma(
+ std::conditional_t<
+ !Mma1_is_RS,
+ decltype(cute::GMMA::ss_op_selector<Element, Element, ElementAccum,
+ decltype(select<0, 2, 1>(TileShape_MNK{})), GMMA::Major::K, MmaMajorV>()),
+ decltype(cute::GMMA::rs_op_selector<Element, Element, ElementAccum,
+ decltype(select<0, 2, 1>(TileShape_MNK{})), GMMA::Major::K, MmaMajorV>())
+ >{},
+ AtomLayoutMNK{}));
+
+ static constexpr int NumMmaThreads = size(TiledMma0{});
+ static constexpr int NumProducerThreads = !Transpose_V && Use_TMA_KV && Use_TMA_Q ? cutlass::NumThreadsPerWarp : cutlass::NumThreadsPerWarpGroup;
+ static_assert(NumMmaThreads % cutlass::NumThreadsPerWarpGroup == 0);
+ static constexpr int NumMmaWarpGroups = NumMmaThreads / cutlass::NumThreadsPerWarpGroup;
+ static_assert(NumMmaWarpGroups == 1 || NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3);
+
+ using SmemLayoutAtomQ = decltype(cutlass::gemm::collective::detail::ss_smem_selector<GMMA::Major::K, Element,
+ decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{}))>());
+ using SmemLayoutQ = decltype(tile_to_shape(SmemLayoutAtomQ{}, select<0, 2>(TileShape_MNK{})));
+
+ using SmemLayoutAtomK = decltype(cutlass::gemm::collective::detail::ss_smem_selector<GMMA::Major::K, Element,
+ decltype(cute::get<1>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{}))>());
+ using SmemLayoutK = decltype(tile_to_shape(
+ SmemLayoutAtomK{},
+ make_shape(shape<1>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int<kStages>{})));
+
+ using SmemLayoutAtomVt = decltype(cutlass::gemm::collective::detail::ss_smem_selector<TmaMajorV, Element,
+ decltype(cute::get<2>(TileShape_MNK{})), decltype(cute::get<1>(TileShape_MNK{}))>());
+ using SmemLayoutVt = decltype(tile_to_shape(
+ SmemLayoutAtomVt{},
+ make_shape(shape<2>(TileShape_MNK{}), shape<1>(TileShape_MNK{}), Int<kStages>{}),
+ std::conditional_t<TmaMajorV == GMMA::Major::K, cute::Step<_1, _2, _3>, cute::Step<_2, _1, _3>>{}));
+
+ using SmemLayoutAtomVtMma = decltype(cutlass::gemm::collective::detail::ss_smem_selector<MmaMajorV, Element,
+ decltype(cute::get<2>(TileShape_MNK{})), decltype(cute::get<1>(TileShape_MNK{}))>());
+ using SmemLayoutVtMma = decltype(tile_to_shape(
+ SmemLayoutAtomVtMma{},
+ make_shape(shape<2>(TileShape_MNK{}), shape<1>(TileShape_MNK{}), Int<kStages>{}),
+ std::conditional_t<MmaMajorV == GMMA::Major::K, cute::Step<_1, _2, _3>, cute::Step<_2, _1, _3>>{}));
+
+ // Only used if we're using cp.async to load V
+ using SmemLayoutAtomVCpAsync = decltype(cutlass::gemm::collective::detail::ss_smem_selector<GMMA::Major::K, Element,
+ decltype(cute::get<1>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{}))>());
+ using SmemLayoutVCpAsync = decltype(tile_to_shape(
+ SmemLayoutAtomVCpAsync{},
+ make_shape(shape<1>(TileShape_MNK{}), shape<2>(TileShape_MNK{}), Int<kStages>{})));
+
+ using SmemLayoutAtomP = decltype(cutlass::gemm::collective::detail::ss_smem_selector<GMMA::Major::K, Element,
+ decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<1>(TileShape_MNK{}))>());
+ using SmemLayoutP = decltype(tile_to_shape(SmemLayoutAtomP{}, select<0, 1>(TileShape_MNK{})));
+
+ using SmemCopyAtomP = Copy_Atom<cute::SM90_U32x4_STSM_N, Element>;
+
+ // Use LDSM.T and STSM to transpose V in the case of FP8 and V being row-major.
+ // For FP16/BF16 we don't do any transposing.
+ static_assert(!Transpose_V || (kHeadDim % 32 == 0 && kBlockN % 32 == 0));
+ static constexpr bool kHeadDim_multiple_64 = kHeadDim % 64 == 0;
+ // Either kHeadDim is a multiple of 64 (in which case we use a block size of 64 x 32 for the transpose),
+ // or we need kBlockN to be a multiple of 64 (in which case we use a block size of 32 x 64 for the transpose).
+ static_assert(!Transpose_V || (kHeadDim_multiple_64 || kBlockN % 64 == 0));
+ using LDSM_thread_shape = std::conditional_t<kHeadDim_multiple_64, Shape<_32, _4, _1, _1>, Shape<_16, _4, _1, _2>>;
+ using LDSM_thread_stride = std::conditional_t<kHeadDim_multiple_64, Stride<_4, _1, _0, _0>, Stride<_4, _1, _0, _64>>;
+ using LDSM_value_shape = Shape<_2, _2, _1, _4>;
+ using LDSM_value_stride = Stride<_1, _2, _16, _4>;
+ using LDSM_divide_shape = std::conditional_t<kHeadDim_multiple_64, Shape<_64, _8>, Shape<_32, _8>>;
+ using S2RTiledCopyVt = decltype(make_tiled_copy(
+ Copy_Atom<SM75_U16x8_LDSM_T, Element>{}, Layout<LDSM_thread_shape, LDSM_thread_stride>{},
+ Layout<LDSM_value_shape, LDSM_value_stride>{}));
+
+ using STSM_thread_shape = std::conditional_t<kHeadDim_multiple_64, Shape<_8, _4, _4, _1>, Shape<_8, _4, _2, _2>>;
+ using STSM_thread_stride = std::conditional_t<kHeadDim_multiple_64, Stride<_4, _1, _32, _0>, Stride<_4, _1, _32, _64>>;
+ using STSM_value_shape = Shape<_1, _4, _2, _2>;
+ using STSM_value_stride = Stride<_0, _1, _4, _8>;
+ using STSM_divide_shape = Shape<_8, _16>;
+ // These will not permute the columns of V (the kHeadDim dimension) but incur bank conflicts
+ // so a little slower (e.g. 1150 TFLOPS for hdim 256 instead of 1200 TFLOPS).
+ // Instead we will permute the cols of V, and un-permute the cols of O in the epilogue.
+ // using STSM_value_shape = Shape<_2, _4, _1, _2>;
+ // using STSM_value_stride = Stride<_4, _1, _0, _8>;
+ // using STSM_divide_shape = Shape<_16, _16>;
+ using R2STiledCopyV = decltype(make_tiled_copy(
+ Copy_Atom<SM90_U32x4_STSM_N, Element>{}, Layout<STSM_thread_shape, STSM_thread_stride>{},
+ Layout<STSM_value_shape, STSM_value_stride>{}));
+
+ using GmemTiledCopyQ = cute::SM90_TMA_LOAD;
+ using GmemTiledCopyKV = decltype(cutlass::gemm::collective::detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape{})));
+
+ // We use CpAsync for K and V if PagedKV and AppendKV, since TMA doesn't work there
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(Element);
+ static_assert(kHeadDim % kGmemElemsPerLoad == 0, "Headdim must be a multiple of kGmemElemsPerLoad");
+ // We want each "row" to have 64 elements (128 bytes, i.e. 1 cache line). E.g. if hdim=128, we want each
+ // thread to have 4 loads in the M direction and 2 vectorized load in the K direction.
+ // We want each thread to have at least 2 loads in the K direction since in the case of non-interleaved
+ // rotary (combining elements at indices 0 and rotary_dim/2, 1 and rotary_dim/2+1, etc), each thread will
+ // load twice from the same row.
+ static constexpr int kBytePerHalfRow = kHeadDim / 2 * sizeof(Element);
+ static constexpr int kBlockKGmem = (kBytePerHalfRow % 128 == 0 ? 128 : (kBytePerHalfRow % 64 == 0 ? 64 : 32)) / sizeof(Element);
+ static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad;
+ static_assert(NumMmaThreads % kGmemThreadsPerRow == 0, "NumMmaThreads must be a multiple of kGmemThreadsPerRow");
+ // We assume threads loading the same row are in the same warp. This is for an optimization in PagedKV where
+ // these threads share the same page table entry and share the work of computing pointers to paged K and paged V.
+ static_assert(cutlass::NumThreadsPerWarp % kGmemThreadsPerRow == 0, "kGmemThreadsPerRow must divide NumThreadsPerWarp");
+ using GmemLayoutAtom = Layout<Shape <Int<NumMmaThreads / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+ // If AppendKV, we'll be loading Q for rotary, and we assume divisibility to avoid predication
+ static_assert(!AppendKV || kBlockM % CUTE_STATIC_V(shape<0>(GmemLayoutAtom{})) == 0, "kBlockM must be a multiple of NumMmaThreads / kGmemThreadsPerRow");
+ using GmemTiledCopyAppendKV = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per store
+
+ using ShapeQKV = cute::Shape<int32_t, int32_t, int32_t, int32_t>; // (seqlen, d, head, batch)
+ using StrideQK = cute::Stride<int64_t, _1, int64_t, int64_t>;
+ using StrideV = std::conditional_t<!V_colmajor, StrideQK, cute::Stride<_1, int64_t, int64_t, int64_t>>;
+ // ((qhead_per_khead, seqlen_q), d, nheads_kv, batch, num_splits)
+ using ShapeQPacked = std::conditional_t<!PackGQA, ShapeQKV, cute::Shape<cute::Shape<int32_t, int32_t>, int32_t, int32_t, int32_t>>;
+ using StrideQPacked = std::conditional_t<!PackGQA, StrideQK, cute::Stride<cute::Stride<int64_t, int64_t>, _1, int64_t, int64_t>>;
+ using ShapePageTable = cute::Shape<int32_t, int32_t>; // (batch, max_num_pages_per_seq)
+ using StridePageTable = cute::Stride<int64_t, _1>;
+ using ShapeRotary = cute::Shape<int32_t, int32_t>; // (seqlen_ro, rotary_dim // 2)
+ using StrideRotary = cute::Stride<int64_t, _1>;
+ using StrideDescale = cute::Stride<int64_t, int64_t>;
+
+ using TMA_Q = decltype(make_tma_copy_A_sm90(
+ GmemTiledCopyQ{},
+ make_tensor(make_gmem_ptr(static_cast<Element const*>(nullptr)), ShapeQKV{}, StrideQK{}),
+ SmemLayoutQ{},
+ TileShape_MNK{},
+ ClusterShape{}));
+
+ using TMA_K = decltype(make_tma_copy_B_sm90(
+ GmemTiledCopyKV{},
+ make_tensor(make_gmem_ptr(static_cast<Element const*>(nullptr)), ShapeQKV{}, StrideQK{}),
+ take<0, 2>(SmemLayoutK{}),
+ TileShape_MNK{},
+ ClusterShape{})); // mcast along M mode for this N load, if any
+
+ using TMA_V = decltype(make_tma_copy(
+ GmemTiledCopyKV{},
+ make_tensor(make_gmem_ptr(static_cast<Element const*>(nullptr)), ShapeQKV{}, select<1, 0, 2, 3>(StrideV{})),
+ take<0, 2>(SmemLayoutVt{}),
+ select<2, 1>(TileShape_MNK{}),
+ size<0>(ClusterShape{}))); // mcast along M mode for this N load, if any
+
+ // Set the bytes transferred in this TMA transaction (may involve multiple issues)
+ static constexpr uint32_t TmaTransactionBytesQ = static_cast<uint32_t>(size(SmemLayoutQ{}) * cutlass::sizeof_bits_v<Element> / 8);
+ static constexpr uint32_t TmaTransactionBytesK = static_cast<uint32_t>(size(take<0, 2>(SmemLayoutK{})) * cutlass::sizeof_bits_v<Element> / 8);
+ static constexpr uint32_t TmaTransactionBytesV = static_cast<uint32_t>(size(take<0, 2>(SmemLayoutVt{})) * cutlass::sizeof_bits_v<Element> / 8);
+ static_assert(TmaTransactionBytesK == TmaTransactionBytesV);
+
+ using PipelineTmaAsync = std::conditional_t<CUTE_STATIC_V(size(ClusterShape{})) == 1, typename cutlass::PipelineTmaAsyncNoCluster<kStages>, typename cutlass::PipelineTmaAsync<kStages>>;
+ using MainloopPipelineK = std::conditional_t<Use_TMA_KV, PipelineTmaAsync, typename cutlass::PipelineAsync<kStages>>;
+ using MainloopPipelineV = std::conditional_t<!Transpose_V && Use_TMA_KV, PipelineTmaAsync, typename cutlass::PipelineAsync<kStages>>;
+ using MainloopPipelineVt = std::conditional_t<Use_TMA_KV, PipelineTmaAsync, typename cutlass::PipelineAsync<kStages>>;
+ // We always use TMA for K_new and V_new
+ using MainloopPipelineKVNew = PipelineTmaAsync;
+ using PipelineState = cutlass::PipelineState<kStages>;
+
+ // If PackGQA, we use cp.async (instead of TMA) to load Q, so we want smem_q to be aligned
+ // and have sQ being position_independent_swizzle_tensor.
+ // If !Use_TMA_KV, we use cp.async (instead of TMA) to load K & V, so we want smem_k and smem_v to be aligned.
+ static constexpr size_t SmemAlignmentQ = Use_TMA_Q && !AppendKV && !Mma0_is_RS ? 128 : cutlass::detail::alignment_for_swizzle(SmemLayoutQ{});
+ static constexpr size_t SmemAlignmentK = Use_TMA_KV && !AppendKV ? 128 : cutlass::detail::alignment_for_swizzle(SmemLayoutK{});
+ static constexpr size_t SmemAlignmentVtNoTranspose = cutlass::detail::alignment_for_swizzle(SmemLayoutVt{});
+ static_assert(SmemAlignmentQ >= 128 and SmemAlignmentK >= 128 && SmemAlignmentVtNoTranspose >= 128, "Require at least 128B alignment");
+ static constexpr size_t SmemAlignmentP = cutlass::detail::alignment_for_swizzle(SmemLayoutP{});
+ static_assert(SmemAlignmentP >= 128, "Require at least 128B alignment");
+
+ using SmemP_t = std::conditional_t<Mma1_is_RS, cute::array<Element, 0>, cute::array_aligned<Element, cute::cosize_v<SmemLayoutP>, SmemAlignmentP>>;
+ // Sometimes even with SmemP_t = cute::array<Element, 0>, putting it in the TensorStorage struct causes
+ // smem size to go from 227KB to 228KB and we get "invalid argument".
+
+ struct TensorStorageWithoutPNoTranspose : cute::aligned_struct<cute::max(SmemAlignmentQ, SmemAlignmentK, SmemAlignmentVtNoTranspose)> {
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutVt>, SmemAlignmentVtNoTranspose> smem_v;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutQ>, SmemAlignmentQ> smem_q;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutK>, SmemAlignmentK> smem_k;
+ };
+
+ struct TensorStorageWithPNoTranspose : cute::aligned_struct<cute::max(SmemAlignmentQ, SmemAlignmentK, SmemAlignmentVtNoTranspose, SmemAlignmentP)> {
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutVt>, SmemAlignmentVtNoTranspose> smem_v;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutQ>, SmemAlignmentQ> smem_q;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutK>, SmemAlignmentK> smem_k;
+ SmemP_t smem_p;
+ };
+
+ using TensorStorageNoTranspose = std::conditional_t<Mma1_is_RS, TensorStorageWithoutPNoTranspose, TensorStorageWithPNoTranspose>;
+
+ static constexpr size_t SmemAlignmentVt = cutlass::detail::alignment_for_swizzle(SmemLayoutVt{});
+ static constexpr size_t SmemAlignmentV = cutlass::detail::alignment_for_swizzle(SmemLayoutVtMma{});
+ static_assert(SmemAlignmentVt >= 128 and SmemAlignmentV >= 128, "Require at least 128B alignment");
+ struct TensorStorageTransposeV : cute::aligned_struct<cute::max(SmemAlignmentQ, SmemAlignmentK, SmemAlignmentV)> {
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutVtMma>, SmemAlignmentV> smem_v;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutVt>, SmemAlignmentVt> smem_vt;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutQ>, SmemAlignmentQ> smem_q;
+ cute::array_aligned<Element, cute::cosize_v<SmemLayoutK>, SmemAlignmentK> smem_k;
+ };
+
+ using TensorStorage = std::conditional_t<!Transpose_V, TensorStorageNoTranspose, TensorStorageTransposeV>;
+
+ // These are tuned for speed. They don't affect correctness.
+ static constexpr bool UseSchedulerBarrier = IntraWGOverlap
+ ? (NumMmaWarpGroups >= 2) && (!Is_FP8 ? kHeadDim <= 128 : kHeadDim >= 128)
+ : NumMmaWarpGroups == 2;
+ static constexpr bool RescaleOBeforeGemm = kHeadDim > 128 && (!Is_FP8 || V_colmajor);
+
+ // Host side kernel arguments
+ struct Arguments {
+ Element const* const ptr_Q;
+ ShapeQKV const shape_Q;
+ StrideQK const stride_Q;
+ Element* const ptr_K; // Not Element const* since we might append to KV cache in-place
+ ShapeQKV const shape_K;
+ StrideQK const stride_K;
+ Element* const ptr_V;
+ StrideV const stride_V;
+ Element const* const ptr_K_new;
+ ShapeQKV const shape_K_new;
+ StrideQK const stride_K_new;
+ Element const* const ptr_V_new;
+ StrideV const stride_V_new;
+ Element const* const ptr_rotary_cos;
+ ShapeRotary const shape_rotary;
+ StrideRotary const stride_rotary_cos;
+ Element const* const ptr_rotary_sin;
+ StrideRotary const stride_rotary_sin;
+ bool const is_rotary_interleaved;
+ int const* const ptr_pagetable;
+ ShapePageTable const shape_pagetable;
+ StridePageTable const stride_pagetable;
+ float const softmax_scale;
+ float const* ptr_q_descale, *ptr_k_descale, *ptr_v_descale;
+ StrideDescale const stride_q_descale, stride_k_descale, stride_v_descale;
+ int const window_size_left = -1, window_size_right = -1, sink_token_length = 0;
+ float const softcap_val;
+ int const num_splits;
+ int const* const kv_batch_idx = nullptr;
+ int const* const cu_seqlens_q = nullptr;
+ int const* const cu_seqlens_k = nullptr;
+ int const* const cu_seqlens_k_new = nullptr;
+ int const* const seqused_q = nullptr;
+ int const* const seqused_k = nullptr;
+ int const* const leftpad_k = nullptr;
+ };
+
+ // Device side kernel params
+ struct Params {
+ Element const* const ptr_Q;
+ ShapeQKV const shape_Q;
+ StrideQK const stride_Q;
+ ShapeQPacked const shape_Q_packed;
+ StrideQPacked const stride_Q_packed;
+ Element* const ptr_K;
+ ShapeQKV const shape_K;
+ StrideQK const stride_K;
+ Element* const ptr_V;
+ StrideV const stride_V;
+ Element const* const ptr_K_new;
+ ShapeQKV const shape_K_new;
+ StrideQK const stride_K_new;
+ Element const* const ptr_V_new;
+ StrideV const stride_V_new;
+ Element const* const ptr_rotary_cos;
+ ShapeRotary const shape_rotary;
+ StrideRotary const stride_rotary_cos;
+ Element const* const ptr_rotary_sin;
+ StrideRotary const stride_rotary_sin;
+ bool const is_rotary_interleaved;
+ int const* const ptr_pagetable;
+ ShapePageTable const shape_pagetable;
+ StridePageTable const stride_pagetable;
+ cutlass::FastDivmod page_size_divmod;
+ cutlass::FastDivmod qhead_per_khead_divmod;
+ TMA_Q tma_load_Q;
+ TMA_K tma_load_K;
+ TMA_V tma_load_V;
+ TMA_K tma_load_K_new;
+ TMA_V tma_load_V_new;
+ float const softmax_scale_log2;
+ float const* ptr_q_descale, *ptr_k_descale, *ptr_v_descale;
+ StrideDescale const stride_q_descale, stride_k_descale, stride_v_descale;
+ float const softcap_val;
+ int const window_size_left, window_size_right, sink_token_length;
+ int const num_splits;
+ int const* const kv_batch_idx = nullptr;
+ int const* const cu_seqlens_q = nullptr;
+ int const* const cu_seqlens_k = nullptr;
+ int const* const cu_seqlens_k_new = nullptr;
+ int const* const seqused_q = nullptr;
+ int const* const seqused_k = nullptr;
+ int const* const leftpad_k = nullptr;
+ };
+
+ static Params
+ to_underlying_arguments(Arguments const& args) {
+ Tensor mQ = make_tensor(make_gmem_ptr(args.ptr_Q), args.shape_Q, args.stride_Q);
+ TMA_Q tma_load_Q = make_tma_copy_A_sm90(
+ GmemTiledCopyQ{},
+ mQ,
+ SmemLayoutQ{},
+ TileShape_MNK{},
+ ClusterShape{}); // no mcast for Q
+ Tensor mK = make_tensor(make_gmem_ptr(args.ptr_K), args.shape_K, args.stride_K);
+ TMA_K tma_load_K = make_tma_copy_B_sm90(
+ GmemTiledCopyKV{},
+ mK,
+ take<0, 2>(SmemLayoutK{}),
+ TileShape_MNK{},
+ ClusterShape{}); // mcast along M mode for this N load, if any
+ Tensor mV = make_tensor(make_gmem_ptr(args.ptr_V), select<1, 0, 2, 3>(args.shape_K), select<1, 0, 2, 3>(args.stride_V));
+ TMA_V tma_load_V = make_tma_copy(
+ GmemTiledCopyKV{},
+ mV,
+ take<0, 2>(SmemLayoutVt{}),
+ select<2, 1>(TileShape_MNK{}),
+ size<0>(ClusterShape{})); // mcast along M mode for this N load, if any
+ Tensor mKnew = make_tensor(make_gmem_ptr(args.ptr_K_new), args.shape_K_new, args.stride_K_new);
+ TMA_K tma_load_K_new = make_tma_copy_B_sm90(
+ GmemTiledCopyKV{},
+ cute::conditional_return<AppendKV>(mKnew, mK),
+ take<0, 2>(SmemLayoutK{}),
+ TileShape_MNK{},
+ ClusterShape{}); // mcast along M mode for this N load, if any
+ Tensor mVnew = make_tensor(make_gmem_ptr(args.ptr_V_new), select<1, 0, 2, 3>(args.shape_K_new), select<1, 0, 2, 3>(args.stride_V_new));
+ TMA_V tma_load_V_new = make_tma_copy(
+ GmemTiledCopyKV{},
+ cute::conditional_return<AppendKV>(mVnew, mV),
+ take<0, 2>(SmemLayoutVt{}),
+ select<2, 1>(TileShape_MNK{}),
+ size<0>(ClusterShape{})); // mcast along M mode for this N load, if any
+ // If PackGQA, reshape Q to be ((qhead_per_khead, seqlen_q), head_size, nhead_k, batch_size)
+ int const qhead_per_khead = !PackGQA ? 1 : cute::ceil_div(get<2>(args.shape_Q), get<2>(args.shape_K));
+ auto const shape_Q_packed = cute::conditional_return<!PackGQA>(
+ args.shape_Q,
+ make_shape(make_shape(qhead_per_khead, get<0>(args.shape_Q)), get<1>(args.shape_Q), get<2>(args.shape_K), get<3>(args.shape_Q))
+ );
+ auto const stride_Q_packed = cute::conditional_return<!PackGQA>(
+ args.stride_Q,
+ make_stride(make_stride(get<2>(args.stride_Q), get<0>(args.stride_Q)), get<1>(args.stride_Q), get<2>(args.stride_Q) * qhead_per_khead, get<3>(args.stride_Q))
+ );
+ if (get<1>(args.shape_rotary) > 0) {
+ assert(args.ptr_rotary_cos != nullptr && args.ptr_rotary_sin != nullptr);
+ }
+ assert(args.num_splits >= 1);
+ // If there's tanh softcapping, we do tanh(scores * softmax_scale / softcap_val) * softcap_val.
+ // Right after this, we multiply by log2(e) before applying exp2.
+ // To reduce the number of instructions, we instead pre-multiply softmax_scale / softcap_val
+ // (assigning it to params.softcap_val) and pre-multiply softcap_val * log2(e)
+ // (assigning it to params.softmax_scale_log2).
+ return {args.ptr_Q, args.shape_Q, args.stride_Q, shape_Q_packed, stride_Q_packed,
+ args.ptr_K, args.shape_K, args.stride_K, args.ptr_V, args.stride_V,
+ args.ptr_K_new, args.shape_K_new, args.stride_K_new, args.ptr_V_new, args.stride_V_new,
+ args.ptr_rotary_cos, args.shape_rotary, args.stride_rotary_cos,
+ args.ptr_rotary_sin, args.stride_rotary_sin, args.is_rotary_interleaved,
+ args.ptr_pagetable, args.shape_pagetable, args.stride_pagetable,
+ cutlass::FastDivmod(int(get<0>(args.shape_K))),
+ cutlass::FastDivmod(cute::ceil_div(get<2>(args.shape_Q), get<2>(args.shape_K))),
+ tma_load_Q, tma_load_K, tma_load_V, tma_load_K_new, tma_load_V_new,
+ !Has_softcap ? float(args.softmax_scale * M_LOG2E) : float(args.softcap_val * M_LOG2E),
+ args.ptr_q_descale, args.ptr_k_descale, args.ptr_v_descale,
+ args.stride_q_descale, args.stride_k_descale, args.stride_v_descale,
+ !Has_softcap ? 0.f : args.softmax_scale / args.softcap_val,
+ args.window_size_left, args.window_size_right, args.sink_token_length,
+ !Split ? 1 : args.num_splits,
+ args.kv_batch_idx,
+ args.cu_seqlens_q, args.cu_seqlens_k, args.cu_seqlens_k_new,
+ args.seqused_q, args.seqused_k, args.leftpad_k};
+ }
+
+ /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best performance
+ CUTLASS_DEVICE
+ static void prefetch_tma_descriptors(Params const& params) {
+ if constexpr (Use_TMA_Q) {
+ cute::prefetch_tma_descriptor(params.tma_load_Q.get_tma_descriptor());
+ }
+ if constexpr (Use_TMA_KV) {
+ cute::prefetch_tma_descriptor(params.tma_load_K.get_tma_descriptor());
+ cute::prefetch_tma_descriptor(params.tma_load_V.get_tma_descriptor());
+ }
+ if constexpr (AppendKV) {
+ cute::prefetch_tma_descriptor(params.tma_load_K_new.get_tma_descriptor());
+ cute::prefetch_tma_descriptor(params.tma_load_V_new.get_tma_descriptor());
+ }
+ }
+
+ CUTLASS_DEVICE
+ cute::tuple<int, int> get_n_block_min_max(Params const& params, SeqlenInfo_t const& seqlen_info,
+ int m_block, int bidb, int split_idx=0, int num_splits=1) {
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ int const seqlen_k = seqlen_info.seqlen_k;
+ int const seqlen_q = seqlen_info.seqlen_q;
+ int n_block_max = cute::ceil_div(seqlen_k, kBlockN);
+ if constexpr (Is_causal || Is_local) {
+ int m_idx_max = (m_block + 1) * kBlockM;
+ // TODO: check off-by-1 error
+ if (PackGQA) { m_idx_max = params.qhead_per_khead_divmod.divide(m_idx_max - 1) + 1 ; }
+ n_block_max = std::min(n_block_max,
+ cute::ceil_div(m_idx_max + seqlen_k - seqlen_q + params.window_size_right, kBlockN));
+ }
+ int n_block_min = 0;
+ if constexpr (Is_local) {
+ int m_idx_min = m_block * kBlockM;
+ if (PackGQA) { m_idx_min = params.qhead_per_khead_divmod.divide(m_idx_min); }
+ n_block_min = std::max(int(0), (m_idx_min + seqlen_k - seqlen_q - params.window_size_left) / kBlockN);
+ }
+ // if (threadIdx.x == 128) { printf("Inside, bid.x = %d, bid.y = %d, bid.z = %d, split_idx = %d, n_block_min: %d, n_block_max: %d\n", blockIdx.x, blockIdx.y, blockIdx.z, split_idx, n_block_min, n_block_max); }
+ if constexpr (Split) {
+ int num_n_blocks_per_split = n_block_max <= n_block_min ? 0 : cute::ceil_div(n_block_max - n_block_min, num_splits);
+ n_block_min = n_block_min + split_idx * num_n_blocks_per_split;
+ n_block_max = std::min(n_block_min + num_n_blocks_per_split, n_block_max);
+ }
+ // if (threadIdx.x == 128) { printf("After split, inside, bid.y = %d, bid.z = %d, split_idx = %d, n_block_min: %d, n_block_max: %d\n", blockIdx.y, blockIdx.z, split_idx, n_block_min, n_block_max); }
+ return {n_block_min, n_block_max};
+ }
+
+ template <typename SchedulerPrefetch, typename SharedStorage>
+ CUTLASS_DEVICE void
+ load(Params const& params,
+ MainloopPipelineK pipeline_k,
+ MainloopPipelineV pipeline_v,
+ MainloopPipelineVt pipeline_vt,
+ PipelineState& smem_pipe_write,
+ SharedStorage &shared_storage,
+ SchedulerPrefetch const& scheduler_prefetch,
+ SeqlenInfo_t const& seqlen_info,
+ cute::tuple<int32_t, int32_t, int32_t, int32_t> block_coord,
+ int &work_idx
+ ) {
+
+ auto [m_block, bidh, bidb, split_idx] = block_coord;
+ auto [n_block_min, n_block_max] = get_n_block_min_max(params, seqlen_info, m_block, bidb, split_idx, params.num_splits);
+ // It's possible to have n_block_max <= n_block_min. Loading K can cause illegal memory access.
+ if constexpr (Is_causal || Is_local || Varlen || Split) {
+ if (n_block_max <= n_block_min) {
+ scheduler_prefetch();
+ return;
+ }
+ }
+
+ Tensor sQ = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), SmemLayoutQ{});
+ Tensor sK = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), SmemLayoutK{});
+ Tensor sK_pi = as_position_independent_swizzle_tensor(sK);
+ // as_position_independent_swizzle_tensor makes address calculation easier when we do LDSM & STSM to transpose.
+ // But it requires smem_vt and smem_v to be aligned to e.g 512 bytes.
+ Tensor sVt = [&] {
+ if constexpr (!Transpose_V) {
+ return make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutVt{});
+ } else {
+ return cute::as_position_independent_swizzle_tensor(make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_vt.data()), SmemLayoutVt{}));
+ }
+ }();
+ // Only used if Transpose_V
+ Tensor sV = cute::as_position_independent_swizzle_tensor(make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutVtMma{}));
+ // Only used if we're using cp.async to load V
+ Tensor sVcpasync = [&] {
+ if constexpr (!Transpose_V) {
+ return cute::as_position_independent_swizzle_tensor(make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutVCpAsync{}));
+ } else {
+ return cute::as_position_independent_swizzle_tensor(make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_vt.data()), SmemLayoutVCpAsync{}));
+ }
+ }();
+
+ int const thread_idx = threadIdx.x % NumProducerThreads;
+ int const bidh_kv = !PackGQA ? params.qhead_per_khead_divmod.divide(bidh) : bidh;
+ int const bidb_kv = params.kv_batch_idx == nullptr ? bidb : params.kv_batch_idx[bidb];
+
+ // Prepare the TMA loads
+ uint32_t block_rank_in_cluster = cute::block_rank_in_cluster();
+ constexpr uint32_t cluster_shape_x = get<0>(ClusterShape());
+ uint2 cluster_local_block_id = {block_rank_in_cluster % cluster_shape_x, block_rank_in_cluster / cluster_shape_x};
+
+ bool const is_varlen_q = Varlen && params.cu_seqlens_q;
+ bool const is_varlen_k = Varlen && params.cu_seqlens_k;
+ Tensor mQ = params.tma_load_Q.get_tma_tensor(params.shape_Q)(_, _, bidh, !is_varlen_q ? bidb : 0);
+ Tensor mK_TMA = params.tma_load_K.get_tma_tensor(params.shape_K)(_, _, bidh_kv, !is_varlen_k ? bidb_kv : 0);
+ Tensor mVt_TMA = params.tma_load_V.get_tma_tensor(select<1, 0, 2, 3>(params.shape_K))(_, _, bidh_kv, !is_varlen_k ? bidb_kv : 0);
+
+ Tensor gQ = local_tile(domain_offset(make_coord(seqlen_info.offset_q, _0{}), mQ), select<0, 2>(TileShape_MNK{}), make_coord(m_block, _0{})); // (M, K)
+ // if (cute::thread0()) { printf("Varlen = %d, params.leftpad_k = %p, leftpad_k = %d\n", Varlen, params.leftpad_k, leftpad_k); }
+ Tensor gK_TMA = local_tile(domain_offset(make_coord(seqlen_info.offset_k, _0{}), mK_TMA), select<1, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (N, K, _)
+ Tensor gVt_TMA = local_tile(domain_offset(make_coord(_0{}, seqlen_info.offset_k), mVt_TMA), select<2, 1>(TileShape_MNK{}), make_coord(_0{}, _)); // (K, N, _)
+
+ auto block_tma_Q = params.tma_load_Q.get_slice(_0{});
+ Tensor tQgQ = group_modes<0, 3>(block_tma_Q.partition_S(gQ)); // (TMA)
+ Tensor tQsQ = group_modes<0, 3>(block_tma_Q.partition_D(sQ)); // (TMA)
+ // tma_partition doesn't handle position_independent_swizzle_tensor correctly, so we need to do it manually
+ auto block_tma_K = params.tma_load_K.get_slice(cluster_local_block_id.x);
+ Tensor tKgK_TMA = group_modes<0, 3>(block_tma_K.partition_S(gK_TMA)); // (TMA, k)
+ Tensor tKsK_TMA = group_modes<0, 3>(block_tma_K.partition_D(sK)); // (TMA, PIPE)
+ auto block_tma_V = params.tma_load_V.get_slice(cluster_local_block_id.x);
+ Tensor tVgVt_TMA = group_modes<0, 3>(block_tma_V.partition_S(gVt_TMA)); // (TMA, k)
+ Tensor tVsVt_TMA = group_modes<0, 3>(block_tma_V.partition_D(sVt)); // (TMA, PIPE)
+
+ using PagedKVManager_t = PagedKVManager<get<1>(TileShape_MNK{}), get<2>(TileShape_MNK{}), NumProducerThreads, Element, Transpose_V || !IntraWGOverlap /*KV_Same_Iter*/>;
+ PagedKVManager_t paged_kv_manager(
+ params.ptr_pagetable, params.shape_pagetable, params.stride_pagetable,
+ params.ptr_K, params.shape_K, params.stride_K,
+ params.ptr_V, params.stride_V,
+ params.page_size_divmod, bidb_kv, bidh_kv, thread_idx, seqlen_info.seqlen_k, seqlen_info.leftpad_k
+ );
+
+ // Set up for transposing V, only used if Transpose_V
+ S2RTiledCopyVt s2r_tiled_copy_vt;
+ R2STiledCopyV r2s_tiled_copy_v;
+ auto s2r_thr_copy_vt = s2r_tiled_copy_vt.get_thread_slice(thread_idx);
+ auto r2s_thr_copy_v = r2s_tiled_copy_v.get_thread_slice(thread_idx);
+ // flat_divide(sVt, LDSM_divide_shape{}): (64, 8, kHeadDim / 64, kBlockN / 8, kStages)
+ Tensor tTranssVt_ = s2r_thr_copy_vt.partition_S(flat_divide(sVt, LDSM_divide_shape{})); // ((16, 1), 1, 1, kHeadDim / 64, kBlockN / 32, kStages)
+ // flat_divide(sV, STSM_divide_shape{}): (8, 16, kHeadDim / 8, (4, kBlockN / 64), kStages)
+ Tensor tTranssV_ = r2s_thr_copy_v.partition_D(flat_divide(sV, STSM_divide_shape{})); // ((16, 1), 1, 1, kHeadDim / 64, (2, kBlockN / 64), kStages)
+ CUTE_STATIC_ASSERT_V(rank(tTranssVt_) == rank(tTranssV_));
+ CUTE_STATIC_ASSERT_V(size<0>(tTranssVt_) == size<0>(tTranssV_));
+ CUTE_STATIC_ASSERT_V(size<1>(tTranssVt_) == size<1>(tTranssV_));
+ CUTE_STATIC_ASSERT_V(size<2>(tTranssVt_) == size<2>(tTranssV_));
+ CUTE_STATIC_ASSERT_V(size<3>(tTranssVt_) == size<3>(tTranssV_));
+ CUTE_STATIC_ASSERT_V(size<4>(tTranssVt_) == size<4>(tTranssV_));
+ // Faster to have 2 LDSM.T, byte permute, STSM for better ILP
+ static constexpr int Transpose_ILP = (size<2>(tTranssVt_) * size<3>(tTranssVt_)) % 2 == 0 ? 2 : 1;
+ Tensor tTranssVt = logical_divide(group_modes<1, rank(tTranssVt_) - 1>(tTranssVt_), Shape<Underscore, Int<Transpose_ILP>>{}); // ((16, 1), (2, kHeadDim / 64 * kBlockN / 32 / 2), kStages)
+ Tensor tTranssV = logical_divide(group_modes<1, rank(tTranssV_) - 1>(tTranssV_), Shape<Underscore, Int<Transpose_ILP>>{}); // ((16, 1), (2, kHeadDim / 64 * kBlockN / 32 / 2), kStages)
+ auto transpose_V = [&](int stage) {
+ if constexpr (Transpose_V) {
+ #pragma unroll
+ for (int i = 0; i < size<1, 1>(tTranssVt); ++i) {
+ Tensor tTransrV = make_fragment_like(tTranssV(_, make_coord(_, _0{}), _0{}));
+ static_assert(size<0>(tTransrV) == 16);
+ Tensor tTransrV_64 = recast<uint2>(tTransrV);
+ cute::copy(s2r_tiled_copy_vt, tTranssVt(_, make_coord(_, i), stage), tTransrV);
+ #pragma unroll
+ for (int j = 0; j < size(tTransrV_64); ++j) {
+ uint32_t upper = tTransrV_64[j].x;
+ uint32_t lower = tTransrV_64[j].y;
+ tTransrV_64[j].x = __byte_perm(upper, lower, 0x6420);
+ tTransrV_64[j].y = __byte_perm(upper, lower, 0x7531);
+ }
+ cute::copy(r2s_tiled_copy_v, tTransrV, tTranssV(_, make_coord(_, i), stage));
+ }
+ }
+ };
+
+ uint16_t mcast_mask_kv = 0;
+ if constexpr (cute::is_same_v<GmemTiledCopyKV, SM90_TMA_LOAD_MULTICAST>) {
+ auto block_layout = Layout<ClusterShape>{}; // (m,n) -> block_id
+ for (int m = 0; m < size<0>(block_layout); ++m) {
+ mcast_mask_kv |= (uint16_t(1) << block_layout(m, cluster_local_block_id.y, _0{}));
+ }
+ }
+
+ auto load_K = [&] (int const n_block, auto const& smem_pipe_write, auto need_seqlenk_masking_type) {
+ pipeline_k.producer_acquire(smem_pipe_write);
+ if constexpr (!PagedKV) {
+ copy(params.tma_load_K.with(*pipeline_k.producer_get_barrier(smem_pipe_write), mcast_mask_kv, TMA::CacheHintSm90::EVICT_LAST),
+ tKgK_TMA(_, n_block), tKsK_TMA(_, smem_pipe_write.index()));
+ } else {
+ constexpr bool Seqlenk_mask = decltype(need_seqlenk_masking_type)::value;
+ paged_kv_manager.template load_K<Seqlenk_mask>(n_block, sK_pi(_, _, smem_pipe_write.index()));
+ pipeline_k.producer_commit(smem_pipe_write, cutlass::arch::cpasync_barrier_arrive);
+ }
+ };
+
+ auto load_V = [&] (int const n_block, auto const& smem_pipe_write, auto need_seqlenk_masking_type) {
+ auto pipeline_v_load = cute::conditional_return<!Transpose_V>(pipeline_v, pipeline_vt);
+ pipeline_v_load.producer_acquire(smem_pipe_write);
+ if constexpr (!PagedKV) {
+ copy(params.tma_load_V.with(*pipeline_v_load.producer_get_barrier(smem_pipe_write), mcast_mask_kv, TMA::CacheHintSm90::EVICT_LAST),
+ tVgVt_TMA(_, n_block), tVsVt_TMA(_, smem_pipe_write.index()));
+ } else {
+ constexpr bool Seqlenk_mask = decltype(need_seqlenk_masking_type)::value;
+ paged_kv_manager.template load_V<Seqlenk_mask>(n_block, sVcpasync(_, _, smem_pipe_write.index()));
+ pipeline_v_load.producer_commit(smem_pipe_write, cutlass::arch::cpasync_barrier_arrive);
+ }
+ };
+
+ auto copy_Vt_to_V = [&] (auto const& smem_pipe_write) {
+ // Instead of maintaining smem_pipe_read as a separate variable, we can just use smem_pipe_write,
+ // and exploit the invariance that smem_pipe_write.phase() == smem_pipe_read.phase() ^ 1.
+ // This saves 1 or 2 registers.
+ PipelineState smem_pipe_read{smem_pipe_write.index(), smem_pipe_write.phase() ^ 1, smem_pipe_write.count()};
+ pipeline_vt.consumer_wait(smem_pipe_read);
+ pipeline_v.producer_acquire(smem_pipe_write);
+ transpose_V(smem_pipe_write.index());
+ // SMEM fence to make sure V is transposed before math
+ cutlass::arch::fence_view_async_shared();
+ pipeline_v.producer_commit(smem_pipe_write);
+ // Very important: PipelineTmaAsync::consumer_release assumes that the warpgroup is synchronized
+ // before calling. Without this we get race conditions.
+ cutlass::arch::NamedBarrier::sync(cutlass::NumThreadsPerWarpGroup, static_cast<uint32_t>(FwdNamedBarriers::ProducerWG) /*id*/);
+ pipeline_vt.consumer_release(smem_pipe_read);
+ };
+
+ int n_block = n_block_max - 1;
+
+ int warp_idx_in_warpgroup = __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0);
+ // If this is true, we're guaranteed that only the first warp will execute this function
+ static constexpr bool SingleProducerWarp = NumProducerThreads == cutlass::NumThreadsPerWarp;
+ bool should_load_KV = !Use_TMA_KV || ((SingleProducerWarp || warp_idx_in_warpgroup == 0) && cute::elect_one_sync());
+
+ if (should_load_KV) {
+ if constexpr (PagedKV) {
+ paged_kv_manager.template load_page_table<true /*Seqlenk_mask*/, true /*First_iter*/>(n_block);
+ }
+ if constexpr (Transpose_V) { load_V(n_block, smem_pipe_write, cute::true_type{} /*Seqlenk_mask*/); }
+ // if (thread_idx == 0) { printf("Producer: main load, before load_K, index = %d\n", smem_pipe_write.index());}
+ load_K(n_block, smem_pipe_write, cute::true_type{} /*Seqlenk_mask*/);
+ // if (thread_idx == 0) { printf("Producer: main load, after load K, index = %d\n", smem_pipe_write.index());}
+ }
+
+ if constexpr (Use_TMA_Q) {
+ // Wait for the MMA warpgroups to signal that smem_q is ready
+ if (SingleProducerWarp || warp_idx_in_warpgroup == 0) {
+ cutlass::arch::NamedBarrier::sync(NumMmaThreads + cutlass::NumThreadsPerWarp, static_cast<uint32_t>(FwdNamedBarriers::QueryEmpty) /*id*/);
+ }
+
+ if ((SingleProducerWarp || warp_idx_in_warpgroup == 0) && cute::elect_one_sync()) {
+ shared_storage.pipelines.barrier_Q.arrive_and_expect_tx(TmaTransactionBytesQ);
+ copy(params.tma_load_Q.with(reinterpret_cast<typename cutlass::arch::ClusterTransactionBarrier::ValueType&>(shared_storage.pipelines.barrier_Q), 0 /*mcast_mask*/, !Split ? TMA::CacheHintSm90::EVICT_FIRST : TMA::CacheHintSm90::EVICT_LAST),
+ tQgQ, tQsQ);
+ }
+ } else { // Load Q with cp.async
+ cutlass::arch::NamedBarrier::sync(NumMmaThreads + NumProducerThreads, static_cast<uint32_t>(FwdNamedBarriers::QueryEmpty) /*id*/);
+ Tensor mQ = make_tensor(make_gmem_ptr(params.ptr_Q + seqlen_info.offset_q * get<0>(params.stride_Q)), params.shape_Q_packed, params.stride_Q_packed)(_, _, bidh, !is_varlen_q ? bidb : 0);
+ Tensor sQ_pi = cute::as_position_independent_swizzle_tensor(sQ);
+ using PackGQAt = flash::PackGQAManager<get<0>(TileShape_MNK{}), get<2>(TileShape_MNK{}), NumProducerThreads, Element>;
+ PackGQAt::load_Q(mQ, sQ_pi, params.qhead_per_khead_divmod, thread_idx, seqlen_info.seqlen_q, m_block);
+ auto &barrier_Q = shared_storage.pipelines.barrier_Q;
+ cutlass::arch::cpasync_barrier_arrive(reinterpret_cast<uint64_t*>(&barrier_Q));
+ barrier_Q.arrive();
+ }
+
+ // Wait for the MMA WGs to signal that smem_v are ready and V can be copied from gmem
+ // Need ClusterBarrier, not just NamedBarrier. Otherwise we might have CTA 0 finishing the
+ // TMA store on O first, call TMA multicast load on V, before CTA 1 can finishing TMA store on O.
+ // if (thread_idx == 0) { printf("Producer: main load, before barrier_O, work_idx = %d\n", work_idx);}
+ shared_storage.pipelines.barrier_O.wait((work_idx + 1) % 2);
+ // if (thread_idx == 0) { printf("Producer: main load, after barrier_O\n");}
+
+ if constexpr (!Transpose_V && !IntraWGOverlap) {
+ if (should_load_KV) { load_V(n_block, smem_pipe_write, cute::true_type{} /*Seqlenk_mask*/); }
+ }
+ int n_block_prev = n_block;
+ --n_block;
+ #pragma unroll (!Transpose_V && Use_TMA_KV ? 2 : 1)
+ for (; n_block >= n_block_min; --n_block) {
+ PipelineState smem_pipe_write_v = smem_pipe_write; // copy the state, write_v is always 1 step behind
+ ++smem_pipe_write;
+ if (should_load_KV) {
+ if constexpr (PagedKV) {
+ paged_kv_manager.template load_page_table<false /*Seqlenk_mask*/>(n_block);
+ }
+ if constexpr (Transpose_V) { load_V(n_block, smem_pipe_write, cute::false_type{} /*Seqlenk_mask*/); }
+ load_K(n_block, smem_pipe_write, cute::false_type{} /*Seqlenk_mask*/);
+ if constexpr (!Transpose_V) {
+ if constexpr (IntraWGOverlap) {
+ load_V(n_block_prev, smem_pipe_write_v, cute::true_type{} /*Seqlenk_mask*/);
+ } else {
+ load_V(n_block, smem_pipe_write, cute::false_type{} /*Seqlenk_mask*/);
+ }
+ }
+ }
+ n_block_prev = n_block;
+ if constexpr (Transpose_V) { copy_Vt_to_V(smem_pipe_write_v); }
+ }
+ // if constexpr (Is_local) {
+ // Disable sink token code for now
+ if constexpr (false && Is_local) {
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ int n_block_sink_max = cute::ceil_div(params.sink_token_length, kBlockN);
+ #pragma unroll 1
+ for (n_block = std::min(n_block, n_block_sink_max - 1); n_block >= 0; --n_block) {
+ PipelineState smem_pipe_write_v = smem_pipe_write; // copy the state, write_v is always 1 step behind
+ ++smem_pipe_write;
+ if (should_load_KV) {
+ if constexpr (PagedKV) {
+ paged_kv_manager.template load_page_table<false /*Seqlenk_mask*/>(n_block);
+ }
+ if constexpr (Transpose_V) { load_V(n_block, smem_pipe_write, cute::false_type{} /*Seqlenk_mask*/); }
+ load_K(n_block, smem_pipe_write, cute::false_type{} /*Seqlenk_mask*/);
+ if constexpr (!Transpose_V) {
+ if constexpr (IntraWGOverlap) {
+ load_V(n_block_prev, smem_pipe_write_v, cute::true_type{} /*Seqlenk_mask*/);
+ } else {
+ load_V(n_block, smem_pipe_write, cute::false_type{} /*Seqlenk_mask*/);
+ }
+ }
+ }
+ n_block_prev = n_block;
+ if constexpr (Transpose_V) { copy_Vt_to_V(smem_pipe_write_v); }
+ }
+ }
+ scheduler_prefetch();
+ if constexpr (!Transpose_V && IntraWGOverlap) {
+ if (should_load_KV) { load_V(n_block_prev, smem_pipe_write, cute::true_type{} /*Seqlenk_mask*/); }
+ }
+ if constexpr (Transpose_V) { copy_Vt_to_V(smem_pipe_write); }
+ ++smem_pipe_write;
+ // At the end, all threads have the correct smem_pipe_write.
+ ++work_idx;
+ }
+
+ template <typename SharedStorage>
+ CUTLASS_DEVICE void
+ load_tail(MainloopPipelineK pipeline_k, MainloopPipelineV pipeline_v, MainloopPipelineVt pipeline_vt,
+ PipelineState& smem_pipe_write, SharedStorage &shared_storage, int const work_idx) {
+ // If we don't wait for barrier_O here, when using Cluster, CTA0 might exit early and CTA1 will
+ // try to arrive on barrier_O of CTA0, causing "unspecified launch failure".
+ shared_storage.pipelines.barrier_O.wait((work_idx + 1) % 2);
+ int warp_idx_in_warpgroup = __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0);
+ // Issue the epilogue waits
+ // TODO: check if this should be called by 1 thread or more
+ if (warp_idx_in_warpgroup == 0 && cute::elect_one_sync()) {
+ /* This helps avoid early exit of blocks in Cluster
+ * Waits for all stages to either be released (all Consumer UNLOCKs), or if the stage was never used
+ * then would just be acquired since the phase was still inverted from make_producer_start_state
+ */
+ pipeline_k.producer_tail(smem_pipe_write);
+ pipeline_v.producer_tail(smem_pipe_write);
+ if constexpr (Transpose_V) { pipeline_vt.producer_tail(smem_pipe_write); }
+ }
+ }
+
+ CUTLASS_DEVICE void
+ warp_scheduler_barrier_sync() {
+ if constexpr (UseSchedulerBarrier) {
+ cutlass::arch::NamedBarrier::sync(2 * cutlass::NumThreadsPerWarpGroup, static_cast<uint32_t>(FwdNamedBarriers::WarpSchedulerWG1) - 1 + flash::canonical_warp_group_idx_nosync() /*id*/);
+ }
+ }
+
+ CUTLASS_DEVICE void
+ warp_scheduler_barrier_arrive() {
+ if constexpr (UseSchedulerBarrier) {
+ static_assert(NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3);
+ int const cur_WG = flash::canonical_warp_group_idx_nosync() - 1;
+ int const next_WG = NumMmaWarpGroups == 2
+ ? 1 - cur_WG
+ : (cur_WG < NumMmaWarpGroups - 1 ? cur_WG + 1 : 0);
+ cutlass::arch::NamedBarrier::arrive(2 * cutlass::NumThreadsPerWarpGroup, static_cast<uint32_t>(FwdNamedBarriers::WarpSchedulerWG1) + next_WG /*id*/);
+ }
+ }
+
+ CUTLASS_DEVICE void
+ mma_init() {
+ // Tell producers that smem_q is ready
+ cutlass::arch::NamedBarrier::arrive(NumMmaThreads + (Use_TMA_Q ? cutlass::NumThreadsPerWarp : NumProducerThreads), static_cast<uint32_t>(FwdNamedBarriers::QueryEmpty) /*id*/);
+ if constexpr (UseSchedulerBarrier) {
+ // We have NamedBarrier for up to 3 WGs
+ static_assert(NumMmaWarpGroups == 2 || NumMmaWarpGroups == 3);
+ // WG1 needs the very first signal to start
+ if (flash::canonical_warp_group_idx_nosync() == 1) {
+ cutlass::arch::NamedBarrier::arrive(2 * cutlass::NumThreadsPerWarpGroup, static_cast<uint32_t>(FwdNamedBarriers::WarpSchedulerWG1) /*id*/);
+ }
+ }
+ }
+
+ template <typename SharedStorage, typename FrgTensorO, typename Softmax>
+ CUTLASS_DEVICE bool
+ mma(Params const& params,
+ MainloopPipelineK pipeline_k,
+ MainloopPipelineV pipeline_v,
+ PipelineState& smem_pipe_read,
+ FrgTensorO& tOrO,
+ Softmax& softmax,
+ int const thread_idx,
+ int &work_idx,
+ SeqlenInfo_t const& seqlen_info,
+ cute::tuple<int32_t, int32_t, int32_t, int32_t> block_coord,
+ SharedStorage& shared_storage
+ ) {
+ static_assert(is_rmem<FrgTensorO>::value, "O tensor must be rmem resident.");
+ static constexpr int kBlockM = get<0>(TileShape_MNK{});
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+
+ // can't use auto [m_block, ...] = block_coord since structured binding cannot be captured in lambda
+ int const m_block = get<0>(block_coord);
+ int const bidh = get<1>(block_coord);
+ int const bidb = get<2>(block_coord);
+ int const split_idx = get<3>(block_coord);
+ int const bidh_kv = !PackGQA ? params.qhead_per_khead_divmod.divide(bidh) : bidh;
+ auto [n_block_min, n_block_max] = get_n_block_min_max(params, seqlen_info, m_block, bidb, split_idx, params.num_splits);
+ // It's possible to have n_block_max <= n_block_min. We don't want to load Q or change any barrier
+ if constexpr (Is_causal || Is_local || Varlen || Split) {
+ if (n_block_max <= n_block_min) { return false; }
+ }
+
+ Tensor sQ = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), SmemLayoutQ{});
+ Tensor sK = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), SmemLayoutK{});
+ Tensor sV = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutVtMma{});
+ Tensor sP = [&] {
+ if constexpr (Mma1_is_RS) {
+ // We might not have smem_p if !Mma1_is_RS1, just use smem_q as a placeholder since we don't use it
+ return make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_q.data()), SmemLayoutP{});
+ } else {
+ return make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_p.data()), SmemLayoutP{});
+ }
+ }();
+
+ if constexpr (!Mma0_is_RS) {
+ static_assert(stride<0>(typename TiledMma0::ALayout{}) == 0 and
+ stride<0>(typename TiledMma0::BLayout{}) == 0 and
+ size<0>(typename TiledMma0::ALayout{}) == cutlass::NumThreadsPerWarpGroup and
+ size<0>(typename TiledMma0::BLayout{}) == cutlass::NumThreadsPerWarpGroup,
+ "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup");
+ }
+ constexpr int MmaWarpGroups = size(TiledMma0{}) / cutlass::NumThreadsPerWarpGroup;
+ Layout warp_group_thread_layout = make_layout(make_shape(Int<MmaWarpGroups>{}),
+ make_stride(Int<cutlass::NumThreadsPerWarpGroup>{}));
+
+ int warp_group_idx = __shfl_sync(0xFFFFFFFF, thread_idx / cutlass::NumThreadsPerWarpGroup, 0);
+ TiledMma0 tiled_mma0;
+ TiledMma1 tiled_mma1;
+ auto wg_mma0 = tiled_mma0.get_slice(warp_group_thread_layout(warp_group_idx));
+ auto wg_mma1 = tiled_mma1.get_slice(warp_group_thread_layout(warp_group_idx));
+
+ auto smem_tiled_copy_P = make_tiled_copy_C(SmemCopyAtomP{}, tiled_mma0);
+ auto smem_thr_copy_P = smem_tiled_copy_P.get_thread_slice(thread_idx);
+
+ // Allocate "fragments/descriptors"
+ Tensor tSrQ = wg_mma0.partition_fragment_A(sQ);
+ Tensor tSrK = wg_mma0.partition_fragment_B(sK);
+ Tensor tOrV = wg_mma1.partition_fragment_B(sV);
+ Tensor tOsP = wg_mma1.partition_fragment_A(sP);
+ Tensor tPsP = smem_thr_copy_P.partition_D(cute::as_position_independent_swizzle_tensor(sP));
+
+ auto consumer_wait = [](auto& pipeline, auto& smem_pipe_read) {
+ auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read);
+ pipeline.consumer_wait(smem_pipe_read, barrier_token);
+ };
+
+ // Need to initialize tOrO in the case of RescaleOBeforeGemm where we will scale tOrO even in the 1st iter
+ clear(tOrO);
+ // tiled_mma1.accumulate_ = GMMA::ScaleOut::Zero;
+
+ int const seqlen_q = seqlen_info.seqlen_q;
+ int const seqlen_k = seqlen_info.seqlen_k;
+ int n_block = n_block_max - 1;
+
+ flash::Mask<kBlockM, kBlockN, PackGQA, TiledMma0> mask(
+ thread_idx, seqlen_q, seqlen_k, params.window_size_left, params.window_size_right, params.sink_token_length,
+ params.qhead_per_khead_divmod
+ );
+
+ float softcap_val = params.softcap_val;
+ if constexpr (Has_softcap && Is_FP8) {
+ float const q_descale = params.ptr_q_descale == nullptr ? 1.0f : params.ptr_q_descale[bidb * get<0>(params.stride_q_descale) + bidh_kv * get<1>(params.stride_q_descale)];
+ float const k_descale = params.ptr_k_descale == nullptr ? 1.0f : params.ptr_k_descale[bidb * get<0>(params.stride_k_descale) + bidh_kv * get<1>(params.stride_k_descale)];
+ softcap_val *= q_descale * k_descale;
+ }
+ // Softcapping needs to happen before masking since if we apply after masking, softcapping can turn
+ // -inf to e.g. -50.0, which can affect the attention softmax.
+ auto scoremod_premask_fn = [&](auto& tSrS) {
+ if constexpr (Has_softcap) { flash::apply_softcap(tSrS, softcap_val); }
+ };
+
+ auto &barrier_Q = shared_storage.pipelines.barrier_Q;
+ if constexpr (!AppendKV) {
+ barrier_Q.wait(work_idx % 2);
+ } else {
+ if (get<1>(params.shape_rotary) > 0) { // Apply rotary to Q
+ int const offset_rotary = seqlen_info.seqlen_k_og + seqlen_info.leftpad_k;
+ using Rotary_t = Rotary<kBlockM, kHeadDim, NumMmaThreads, Element, !(Is_causal || Is_local) /*FixedPosition*/>;
+ Rotary_t rotary(params.ptr_rotary_cos, params.shape_rotary, params.stride_rotary_cos,
+ params.ptr_rotary_sin, params.stride_rotary_sin,
+ params.is_rotary_interleaved, thread_idx, seqlen_q, offset_rotary);
+ Tensor sQ_pi = cute::as_position_independent_swizzle_tensor(sQ);
+ int const qhead_per_khead = !PackGQA ? 1 : params.qhead_per_khead_divmod.divisor;
+ if (params.is_rotary_interleaved) {
+ auto [tRrCos, tRrSin] = cute::conditional_return<!PackGQA>(
+ rotary.template load_cos_sin<true /*kInterleaved*/>(m_block),
+ rotary.template load_cos_sin_packgqa<true /*kInterleaved*/>(m_block, params.qhead_per_khead_divmod)
+ );
+ barrier_Q.wait(work_idx % 2);
+ rotary.apply_Q_interleaved(sQ_pi, tRrCos, tRrSin, m_block, qhead_per_khead);
+ } else {
+ auto [tRrCosCont, tRrSinCont] = cute::conditional_return<!PackGQA>(
+ rotary.template load_cos_sin<false /*kInterleaved*/>(m_block),
+ rotary.template load_cos_sin_packgqa<false /*kInterleaved*/>(m_block, params.qhead_per_khead_divmod)
+ );
+ barrier_Q.wait(work_idx % 2);
+ rotary.apply_Q_contiguous(sQ_pi, tRrCosCont, tRrSinCont, m_block, qhead_per_khead);
+ }
+ // SMEM fence to make sure the rotated Q is visible to GMMA
+ cutlass::arch::fence_view_async_shared();
+ cutlass::arch::NamedBarrier::sync(NumMmaThreads, static_cast<uint32_t>(FwdNamedBarriers::QueryRotated) /*id*/);
+ } else {
+ barrier_Q.wait(work_idx % 2);
+ }
+ }
+
+ if constexpr (Mma0_is_RS) {
+ using SmemCopyAtomQ = Copy_Atom<cute::SM75_U32x4_LDSM_N, Element>;
+ auto smem_tiled_copy_Q = make_tiled_copy_A(SmemCopyAtomQ{}, tiled_mma0);
+ auto smem_thr_copy_Q = smem_tiled_copy_Q.get_thread_slice(thread_idx);
+ Tensor tSrQ_copy_view = smem_thr_copy_Q.retile_D(tSrQ);
+ Tensor tSsQ_copy_view = smem_thr_copy_Q.partition_S(cute::as_position_independent_swizzle_tensor(sQ));
+ cute::copy(smem_tiled_copy_Q, tSsQ_copy_view, tSrQ_copy_view);
+ }
+
+ // TODO: check the case where n_block_max <= n_block_min but there are sink tokens
+ if constexpr (IntraWGOverlap) {
+ Tensor tSrS = partition_fragment_C(tiled_mma0, select<0, 1>(TileShape_MNK{}));
+ consumer_wait(pipeline_k, smem_pipe_read);
+ flash::gemm</*zero_init=*/true, /*wg_wait=*/-1>(tiled_mma0, tSrQ, tSrK(_, _, _, smem_pipe_read.index()), tSrS);
+ warpgroup_wait<0>();
+ pipeline_k.consumer_release(smem_pipe_read);
+ scoremod_premask_fn(tSrS);
+ mask.template apply<true /*Seqlenk_mask*/, Is_causal, Is_local>(tSrS, m_block, n_block);
+
+ Tensor scores_scale = softmax.template max_get_scale</*Is_first=*/true, /*Check_inf=*/true>(tSrS);
+ softmax.template online_softmax</*Is_first=*/true, /*Check_inf=*/true>(tSrS);
+ if constexpr (Is_FP8 && !V_colmajor) { flash::permute_Cregs_fp8(tSrS); }
+ Tensor tOrP_acc = make_tensor(tSrS.data(), flash::convert_layout_acc_Aregs<TiledMma1>(tSrS.layout()));
+ Tensor tOrP = make_tensor_like<Element>(tOrP_acc);
+ convert_type_out(tOrP_acc, tOrP);
+ if constexpr (Is_FP8 && V_colmajor) { flash::permute_Aregs_fp8(tOrP); }
+ if constexpr (!Mma1_is_RS) {
+ cute::copy(smem_tiled_copy_P, smem_thr_copy_P.retile_S(tOrP), tPsP);
+ cutlass::arch::fence_view_async_shared();
+ __syncwarp(); // Only need syncwarp since each warp is using its own P values for Mma1
+ }
+ --n_block;
+
+ // Each step does gemm0 for iter n_block, gemm1 for iter n_block + 1, and softmax for iter n_block.
+ auto fwd_step = [&](int const n_block, auto mask_fn, auto check_inf_type) {
+ static constexpr bool Check_inf = decltype(check_inf_type)::value;
+ PipelineState smem_pipe_read_v(smem_pipe_read.index(), smem_pipe_read.phase(), smem_pipe_read.count());
+ ++smem_pipe_read;
+ Tensor tSrS = partition_fragment_C(tiled_mma0, select<0, 1>(TileShape_MNK{}));
+ if (!UseSchedulerBarrier || warp_group_idx == 0) { consumer_wait(pipeline_k, smem_pipe_read); }
+ warp_scheduler_barrier_sync();
+ flash::gemm</*zero_init=*/true, /*wg_wait=*/-1>(tiled_mma0, tSrQ, tSrK(_, _, _, smem_pipe_read.index()), tSrS);
+ if constexpr (RescaleOBeforeGemm) { softmax.rescale_o(tOrO, scores_scale); }
+ if (!UseSchedulerBarrier || warp_group_idx == 0) { consumer_wait(pipeline_v, smem_pipe_read_v); }
+ flash::gemm</*zero_init=*/false, /*wg_wait=*/-1>(tiled_mma1, cute::conditional_return<Mma1_is_RS>(tOrP, tOsP), tOrV(_, _, _, smem_pipe_read_v.index()), tOrO);
+ warp_scheduler_barrier_arrive();
+ warpgroup_wait<1>();
+ pipeline_k.consumer_release(smem_pipe_read); // release K
+ scoremod_premask_fn(tSrS);
+ mask_fn(tSrS, n_block);
+ cute::copy(softmax.template max_get_scale</*Is_first=*/false, Check_inf>(tSrS), scores_scale);
+ softmax.template online_softmax</*Is_first=*/false, Check_inf>(tSrS);
+ warpgroup_wait<0>();
+ pipeline_v.consumer_release(smem_pipe_read_v); // release V
+ if constexpr (Is_FP8 && !V_colmajor) { flash::permute_Cregs_fp8(tSrS); }
+ convert_type_out(make_tensor(tSrS.data(), tOrP.layout()), tOrP);
+ if constexpr (Is_FP8 && V_colmajor) { flash::permute_Aregs_fp8(tOrP); }
+ if constexpr (!Mma1_is_RS) { cute::copy(smem_tiled_copy_P, smem_thr_copy_P.retile_S(tOrP), tPsP); }
+ if constexpr (!RescaleOBeforeGemm) { softmax.rescale_o(tOrO, scores_scale); }
+ if constexpr (!Mma1_is_RS) {
+ cutlass::arch::fence_view_async_shared();
+ __syncwarp();
+ }
+ };
+
+ if constexpr (Is_causal || Is_local) { // Separate iterations with causal or local masking
+ auto mask_fn = [&](auto& tSrS, int n_block) { mask.template apply<false /*Seqlenk_mask*/, Is_causal, Is_local>(tSrS, m_block, n_block); };
+ int const m_idx_min = !PackGQA ? m_block * kBlockM : params.qhead_per_khead_divmod.divide(m_block * kBlockM);
+ int const n_block_min_causal_local_mask =
+ std::max(n_block_min, (m_idx_min + seqlen_k - seqlen_q + params.window_size_right) / kBlockN);
+ #pragma unroll 1
+ for (; n_block >= n_block_min_causal_local_mask; --n_block) {
+ fwd_step(n_block, mask_fn, cute::true_type{} /*check_inf*/);
+ }
+ }
+
+ int const m_idx_max = !PackGQA ? (m_block + 1) * kBlockM : params.qhead_per_khead_divmod.divide((m_block + 1) * kBlockM - 1) + 1;
+ int const n_block_min_before_local_mask = !Is_local
+ ? n_block_min
+ : std::max(n_block_min,
+ cute::ceil_div(m_idx_max + seqlen_k - seqlen_q - params.window_size_left, kBlockN));
+ auto no_mask_fn = [](auto& tSrS, int n_block) { };
+ #pragma unroll 1
+ for (; n_block >= n_block_min_before_local_mask; --n_block) {
+ fwd_step(n_block, no_mask_fn, cute::false_type{} /*check_inf*/);
+ }
+ // Separate masking iterations on the left for local attention
+ if constexpr (Is_local) {
+ auto local_mask_fn = [&](auto& tSrS, int n_block) { mask.template apply<false /*Seqlenk_mask*/, false /*Causal_mask*/, Is_local>(tSrS, m_block, n_block); };
+ #pragma unroll 1
+ for (; n_block >= n_block_min; --n_block) {
+ fwd_step(n_block, local_mask_fn, cute::bool_constant<Is_local>{} /*check_inf*/);
+ }
+ // Disable sink token code for now
+ // int n_block_sink_max = cute::ceil_div(params.sink_token_length, kBlockN);
+ // #pragma unroll 1
+ // for (n_block = std::min(n_block, n_block_sink_max - 1); n_block >= 0; --n_block) {
+ // fwd_step(n_block, local_mask_fn, cute::bool_constant<Is_local>{} /*check_inf*/);
+ // }
+ }
+ // Tell producers that smem_q is ready
+ cutlass::arch::NamedBarrier::arrive(NumMmaThreads + (Use_TMA_Q ? cutlass::NumThreadsPerWarp : NumProducerThreads), static_cast<uint32_t>(FwdNamedBarriers::QueryEmpty) /*id*/);
+ if constexpr (RescaleOBeforeGemm) { softmax.rescale_o(tOrO, scores_scale); }
+ consumer_wait(pipeline_v, smem_pipe_read);
+ flash::gemm</*zero_init=*/false, /*wg_wait=*/-1>(tiled_mma1, cute::conditional_return<Mma1_is_RS>(tOrP, tOsP), tOrV(_, _, _, smem_pipe_read.index()), tOrO);
+ float const v_descale = !Is_FP8 || params.ptr_v_descale == nullptr ? 1.0f : params.ptr_v_descale[bidb * get<0>(params.stride_v_descale) + bidh_kv * get<1>(params.stride_v_descale)];
+ cute::copy(softmax.finalize(v_descale), scores_scale);
+ warpgroup_wait<0>();
+ pipeline_v.consumer_release(smem_pipe_read); // release V, otherwise producers will hang
+ softmax.rescale_o(tOrO, scores_scale);
+ if constexpr (Is_FP8 && !V_colmajor) { flash::permute_output_fp8(tOrO); }
+ ++smem_pipe_read;
+
+ } else { // No intra-WG overlap
+
+ warp_scheduler_barrier_sync();
+
+ auto fwd_step = [&](int const n_block, auto mask_fn, auto is_first_iter_type, auto check_inf_type) {
+ static constexpr bool Is_first_iter = decltype(is_first_iter_type)::value;
+ static constexpr bool Check_inf = decltype(check_inf_type)::value;
+ Tensor tSrS = partition_fragment_C(tiled_mma0, select<0, 1>(TileShape_MNK{}));
+ consumer_wait(pipeline_k, smem_pipe_read);
+ flash::gemm</*zero_init=*/true, /*wg_wait=*/-1>(tiled_mma0, tSrQ, tSrK(_, _, _, smem_pipe_read.index()), tSrS);
+ warp_scheduler_barrier_arrive();
+ warpgroup_wait<0>();
+ pipeline_k.consumer_release(smem_pipe_read); // release K
+ scoremod_premask_fn(tSrS);
+ mask_fn(tSrS, n_block);
+ Tensor scores_scale = softmax.template max_get_scale</*Is_first=*/Is_first_iter, Check_inf>(tSrS);
+ softmax.template online_softmax</*Is_first=*/Is_first_iter, Check_inf>(tSrS);
+ if constexpr (Is_FP8 && !V_colmajor) { flash::permute_Cregs_fp8(tSrS); }
+ Tensor tOrP_acc = make_tensor(tSrS.data(), flash::convert_layout_acc_Aregs<TiledMma1>(tSrS.layout()));
+ Tensor tOrP = make_tensor_like<Element>(tOrP_acc);
+ convert_type_out(tOrP_acc, tOrP);
+ if constexpr (Is_FP8 && V_colmajor) { flash::permute_Aregs_fp8(tOrP); }
+ if constexpr (!Is_first_iter) { softmax.rescale_o(tOrO, scores_scale); }
+ consumer_wait(pipeline_v, smem_pipe_read);
+ warp_scheduler_barrier_sync();
+ flash::gemm</*zero_init=*/false, /*wg_wait=*/0>(tiled_mma1, tOrP, tOrV(_, _, _, smem_pipe_read.index()), tOrO);
+ pipeline_v.consumer_release(smem_pipe_read); // release V
+ ++smem_pipe_read;
+ };
+
+ auto first_iter_mask_fn = [&](auto& tSrS, int n_block) { mask.template apply<true /*Seqlenk_mask*/, Is_causal, Is_local>(tSrS, m_block, n_block); };
+ fwd_step(n_block, first_iter_mask_fn, cute::true_type{} /*is_first_iter*/, cute::true_type{} /*check_inf*/);
+ --n_block;
+ if constexpr (Is_causal || Is_local) { // Separate iterations with causal or local masking
+ auto mask_fn = [&](auto& tSrS, int n_block) { mask.template apply<false /*Seqlenk_mask*/, Is_causal, Is_local>(tSrS, m_block, n_block); };
+ int const m_idx_min = !PackGQA ? m_block * kBlockM : params.qhead_per_khead_divmod.divide(m_block * kBlockM);
+ int const n_block_min_causal_local_mask =
+ std::max(n_block_min, (m_idx_min + seqlen_k - seqlen_q + params.window_size_right) / kBlockN);
+ #pragma unroll 1
+ for (; n_block >= n_block_min_causal_local_mask; --n_block) {
+ fwd_step(n_block, mask_fn, cute::false_type{} /*is_first_iter*/, cute::true_type{} /*check_inf*/);
+ }
+ }
+ int const m_idx_max = !PackGQA ? (m_block + 1) * kBlockM : params.qhead_per_khead_divmod.divide((m_block + 1) * kBlockM - 1) + 1;
+ int const n_block_min_before_local_mask = !Is_local
+ ? n_block_min
+ : std::max(n_block_min,
+ cute::ceil_div(m_idx_max + seqlen_k - seqlen_q - params.window_size_left, kBlockN));
+ auto no_mask_fn = [](auto& tSrS, int n_block) { };
+ #pragma unroll 1
+ for (; n_block >= n_block_min_before_local_mask; --n_block) {
+ fwd_step(n_block, no_mask_fn, cute::false_type{} /*is_first_iter*/, cute::false_type{} /*check_inf*/);
+ }
+ // Separate masking iterations on the left for local attention
+ if constexpr (Is_local) {
+ auto local_mask_fn = [&](auto& tSrS, int n_block) { mask.template apply<false /*Seqlenk_mask*/, false /*Causal_mask*/, Is_local>(tSrS, m_block, n_block); };
+ #pragma unroll 1
+ for (; n_block >= n_block_min; --n_block) {
+ fwd_step(n_block, local_mask_fn, cute::false_type{} /*is_first_iter*/, cute::bool_constant<Is_local>{} /*check_inf*/);
+ }
+ // Disable sink token code for now
+ // int n_block_sink_max = cute::ceil_div(params.sink_token_length, kBlockN);
+ // #pragma unroll 1
+ // for (n_block = std::min(n_block, n_block_sink_max - 1); n_block >= 0; --n_block) {
+ // fwd_step(n_block, local_mask_fn, cute::false_type{} /*is_first_iter*/, cute::bool_constant<Is_local>{} /*check_inf*/);
+ // }
+ }
+ warp_scheduler_barrier_arrive();
+ // Tell producers that smem_q is ready
+ cutlass::arch::NamedBarrier::arrive(NumMmaThreads + (Use_TMA_Q ? cutlass::NumThreadsPerWarp : NumProducerThreads), static_cast<uint32_t>(FwdNamedBarriers::QueryEmpty) /*id*/);
+ float const v_descale = !Is_FP8 || params.ptr_v_descale == nullptr ? 1.0f : params.ptr_v_descale[bidb * get<0>(params.stride_v_descale) + bidh_kv * get<1>(params.stride_v_descale)];
+ Tensor scores_scale = softmax.finalize(v_descale);
+ softmax.rescale_o(tOrO, scores_scale);
+ if constexpr (Is_FP8 && !V_colmajor) { flash::permute_output_fp8(tOrO); }
+ }
+ ++work_idx;
+ return true;
+ }
+
+ CUTLASS_DEVICE
+ cute::tuple<int, int> get_n_block_k_new_min_max(Params const& params, SeqlenInfo_t const& seqlen_info,
+ int m_block, int bidb, int split_idx=0, int num_splits=1) {
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ auto [n_block_min, n_block_max] = get_n_block_min_max(params, seqlen_info, m_block, bidb, split_idx, num_splits);
+ int const idx_k_new_min = std::max(n_block_min * kBlockN - seqlen_info.seqlen_k_og, 0);
+ int const idx_k_new_max = std::min(n_block_max * kBlockN - seqlen_info.seqlen_k_og, seqlen_info.seqlen_k_new);
+ int const n_block_new_min = idx_k_new_min / kBlockN;
+ int const n_block_new_max = idx_k_new_max > idx_k_new_min ? cute::ceil_div(idx_k_new_max, kBlockN) : n_block_new_min;
+ // if (threadIdx.x == 128 && m_block == 0) { printf("bidb = %d, seqlen_k_new = %d, seqlen_k_og = %d, n_block_min = %d, n_block_max = %d, idx_k_new_min = %d, idx_k_new_max = %d, n_block_new_min = %d, n_block_new_max = %d\n", bidb, seqlen_k_new, seqlen_k_og, n_block_min, n_block_max, idx_k_new_min, idx_k_new_max, n_block_new_min, n_block_new_max);}
+ return {n_block_new_min, n_block_new_max};
+ }
+
+ template <typename SharedStorage>
+ CUTLASS_DEVICE bool
+ load_kv_new(Params const& params,
+ MainloopPipelineKVNew pipeline_k_new,
+ MainloopPipelineKVNew pipeline_v_new,
+ PipelineState& smem_pipe_write,
+ SharedStorage &shared_storage,
+ SeqlenInfo_t const& seqlen_info,
+ cute::tuple<int32_t, int32_t, int32_t, int32_t> block_coord,
+ int const work_idx
+ ) {
+
+ auto [m_block, bidh, bidb, split_idx] = block_coord;
+ auto [n_block_new_min, n_block_new_max] = get_n_block_k_new_min_max(params, seqlen_info, m_block, bidb, split_idx, params.num_splits);
+ if (n_block_new_max <= n_block_new_min) { return false; }
+
+ Tensor sK = make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), SmemLayoutK{});
+ Tensor sVt = [&] {
+ if constexpr (!Transpose_V) {
+ return make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutVt{});
+ } else {
+ return make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_vt.data()), SmemLayoutVt{});
+ }
+ }();
+
+ // int const thread_idx = threadIdx.x % NumProducerThreads;
+ int const bidh_kv = !PackGQA ? params.qhead_per_khead_divmod.divide(bidh) : bidh;
+
+ // Prepare the TMA loads
+ uint32_t block_rank_in_cluster = cute::block_rank_in_cluster();
+ constexpr uint32_t cluster_shape_x = get<0>(ClusterShape());
+ uint2 cluster_local_block_id = {block_rank_in_cluster % cluster_shape_x, block_rank_in_cluster / cluster_shape_x};
+
+ bool const is_varlen_k_new = Varlen && params.cu_seqlens_k_new;
+ Tensor mKnew_TMA = params.tma_load_K_new.get_tma_tensor(params.shape_K_new)(_, _, bidh_kv, !is_varlen_k_new ? bidb : 0);
+ Tensor mVnewt_TMA = params.tma_load_V_new.get_tma_tensor(select<1, 0, 2, 3>(params.shape_K_new))(_, _, bidh_kv, !is_varlen_k_new ? bidb : 0);
+
+ Tensor gKnew_TMA = local_tile(domain_offset(make_coord(seqlen_info.offset_k_new, _0{}), mKnew_TMA), select<1, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (N, K, _)
+ Tensor gVnewt_TMA = local_tile(domain_offset(make_coord(_0{}, seqlen_info.offset_k_new), mVnewt_TMA), select<2, 1>(TileShape_MNK{}), make_coord(_0{}, _)); // (K, N, _)
+
+ auto block_tma_K_new = params.tma_load_K_new.get_slice(cluster_local_block_id.x);
+ Tensor tKgKnew_TMA = group_modes<0, 3>(block_tma_K_new.partition_S(gKnew_TMA)); // (TMA, k)
+ Tensor tKsK_TMA = group_modes<0, 3>(block_tma_K_new.partition_D(sK)); // (TMA, PIPE)
+ auto block_tma_V_new = params.tma_load_V_new.get_slice(cluster_local_block_id.x);
+ Tensor tVgVnewt_TMA = group_modes<0, 3>(block_tma_V_new.partition_S(gVnewt_TMA)); // (TMA, k)
+ Tensor tVsVt_TMA = group_modes<0, 3>(block_tma_V_new.partition_D(sVt)); // (TMA, PIPE)
+
+ uint16_t mcast_mask_kv = 0;
+ if constexpr (cute::is_same_v<GmemTiledCopyKV, SM90_TMA_LOAD_MULTICAST>) {
+ auto block_layout = Layout<ClusterShape>{}; // (m,n) -> block_id
+ for (int m = 0; m < size<0>(block_layout); ++m) {
+ mcast_mask_kv |= (uint16_t(1) << block_layout(m, cluster_local_block_id.y, _0{}));
+ }
+ }
+
+ auto load_K_new = [&] (int const n_block, auto const& smem_pipe_write) {
+ pipeline_k_new.producer_acquire(smem_pipe_write);
+ copy(params.tma_load_K_new.with(*pipeline_k_new.producer_get_barrier(smem_pipe_write), mcast_mask_kv, TMA::CacheHintSm90::EVICT_FIRST),
+ tKgKnew_TMA(_, n_block), tKsK_TMA(_, smem_pipe_write.index()));
+ };
+
+ auto load_V_new = [&] (int const n_block, auto const& smem_pipe_write) {
+ pipeline_v_new.producer_acquire(smem_pipe_write);
+ copy(params.tma_load_V_new.with(*pipeline_v_new.producer_get_barrier(smem_pipe_write), mcast_mask_kv, TMA::CacheHintSm90::EVICT_FIRST),
+ tVgVnewt_TMA(_, n_block), tVsVt_TMA(_, smem_pipe_write.index()));
+ };
+
+ int warp_idx_in_warpgroup = __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0);
+ // If this is true, we're guaranteed that only the first warp will execute this function
+ static constexpr bool SingleProducerWarp = NumProducerThreads == cutlass::NumThreadsPerWarp;
+ bool should_load_KV = (SingleProducerWarp || warp_idx_in_warpgroup == 0) && cute::elect_one_sync();
+
+ int n_block = n_block_new_max - 1;
+ // Need to wait for barrier_O even before load_K_new since the pipelines for AppendKV
+ // and the main attention are not the same. We want to make sure the consumers
+ // have finished reading all smem_k and smem_v for the previous iteration.
+ shared_storage.pipelines.barrier_O.wait((work_idx + 1) % 2);
+ if (should_load_KV) { load_K_new(n_block, smem_pipe_write); }
+ // if (thread_idx == 0) { printf("Producer: Done loading K, n_block = %d, n_block_new_min = %d\n", n_block, n_block_new_min); }
+ if (should_load_KV) { load_V_new(n_block, smem_pipe_write); }
+ // if (thread_idx == 0) { printf("Producer: Done loading V, n_block = %d, n_block_new_min = %d\n", n_block, n_block_new_min); }
+ ++smem_pipe_write;
+ --n_block;
+ // if (thread_idx == 0) { printf("Producer: before for loop\n"); }
+ #pragma unroll 1
+ for (; n_block >= n_block_new_min; --n_block) {
+ if (should_load_KV) {
+ load_K_new(n_block, smem_pipe_write);
+ // if (thread_idx == 0) { printf("Producer: Done loading K, n_block = %d, n_block_new_min = %d\n", n_block, n_block_new_min); }
+ load_V_new(n_block, smem_pipe_write);
+ // if (thread_idx == 0) { printf("Producer: Done loading V, n_block = %d, n_block_new_min = %d\n", n_block, n_block_new_min); }
+ }
+ ++smem_pipe_write;
+ }
+ // if (thread_idx == 0) { printf("Producer: after for loop\n"); }
+ // At the end, all threads have the correct smem_pipe_write.
+ return true;
+ }
+
+ template <typename SharedStorage>
+ CUTLASS_DEVICE bool
+ store_kv_new(Params const& params,
+ MainloopPipelineKVNew pipeline_k_new,
+ MainloopPipelineKVNew pipeline_v_new,
+ PipelineState& smem_pipe_read,
+ int const thread_idx,
+ SharedStorage &shared_storage,
+ SeqlenInfo_t const& seqlen_info,
+ cute::tuple<int32_t, int32_t, int32_t, int32_t> block_coord
+ ) {
+ auto [m_block, bidh, bidb, split_idx] = block_coord;
+ auto [n_block_new_min, n_block_new_max] = get_n_block_k_new_min_max(params, seqlen_info, m_block, bidb, split_idx, params.num_splits);
+ if (n_block_new_max <= n_block_new_min) { return false; }
+
+ // as_position_independent_swizzle_tensor makes address calculation easier
+ Tensor sK = cute::as_position_independent_swizzle_tensor(make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_k.data()), SmemLayoutK{}));
+ // We want to use SmemLayoutVCpAsync to have shape (kBlockN, kHeadDim) instead of (kHeadDim, kBlockN)
+ Tensor sV = [&] {
+ if constexpr (!Transpose_V) {
+ return cute::as_position_independent_swizzle_tensor(make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_v.data()), SmemLayoutVCpAsync{}));
+ } else {
+ return cute::as_position_independent_swizzle_tensor(make_tensor(make_smem_ptr(shared_storage.tensors.mainloop.smem_vt.data()), SmemLayoutVCpAsync{}));
+ }
+ }();
+
+ int const bidh_kv = !PackGQA ? params.qhead_per_khead_divmod.divide(bidh) : bidh;
+ int const bidb_kv = params.kv_batch_idx == nullptr ? bidb : params.kv_batch_idx[bidb];
+
+ bool const is_varlen_k = Varlen && params.cu_seqlens_k;
+ Tensor mK = make_tensor(make_gmem_ptr(params.ptr_K), params.shape_K, params.stride_K)(_, _, bidh_kv, !is_varlen_k ? bidb_kv : 0);
+ Tensor mV = make_tensor(make_gmem_ptr(params.ptr_V), params.shape_K, params.stride_V)(_, _, bidh_kv, !is_varlen_k ? bidb_kv : 0);
+
+ int const offset_k = seqlen_info.offset_k + seqlen_info.seqlen_k_og;
+ Tensor gK = local_tile(domain_offset(make_coord(offset_k, _0{}), mK), select<1, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (N, K, _)
+ Tensor gV = local_tile(domain_offset(make_coord(offset_k, _0{}), mV), select<1, 2>(TileShape_MNK{}), make_coord(_, _0{})); // (N, K, _)
+
+ static constexpr int kBlockN = get<1>(TileShape_MNK{});
+ static constexpr int kHeadDim = get<2>(TileShape_MNK{});
+ int const offset_rotary = seqlen_info.seqlen_k_og + seqlen_info.leftpad_k;
+ int const seqlen_k_new = seqlen_info.seqlen_k_new;
+ using Rotary_t = Rotary<kBlockN, kHeadDim, NumMmaThreads, Element>;
+ Rotary_t rotary(params.ptr_rotary_cos, params.shape_rotary, params.stride_rotary_cos,
+ params.ptr_rotary_sin, params.stride_rotary_sin,
+ params.is_rotary_interleaved, thread_idx, seqlen_k_new, offset_rotary);
+
+ using PagedKVManager_t = PagedKVManager<get<1>(TileShape_MNK{}), get<2>(TileShape_MNK{}), NumMmaThreads, Element, true /*KV_Same_Iter*/, 2 /*LoadsPerRow_LB*/>;
+ PagedKVManager_t paged_kv_manager(
+ params.ptr_pagetable, params.shape_pagetable, params.stride_pagetable,
+ params.ptr_K, params.shape_K, params.stride_K,
+ params.ptr_V, params.stride_V,
+ params.page_size_divmod, bidb_kv, bidh_kv, thread_idx, seqlen_k_new, offset_k
+ // passing offset_k instead of leftpad_k will move the PageTable pointer to the right position
+ );
+
+ if constexpr (UseSchedulerBarrier) {
+ // WG1 already got the very first signal from mma_init(), but we'll be using the same NamedBarrier.
+ // So we'll need to "cancel it out" here and then re-signal it at the end.
+ if (flash::canonical_warp_group_idx_nosync() == 1) {
+ cutlass::arch::NamedBarrier::sync(2 * cutlass::NumThreadsPerWarpGroup, static_cast<uint32_t>(FwdNamedBarriers::WarpSchedulerWG1) /*id*/);
+ }
+ }
+
+ static_assert(std::is_same_v<GmemLayoutAtom, typename Rotary_t::LayoutAtom>);
+ static_assert(!PagedKV || std::is_same_v<GmemLayoutAtom, typename PagedKVManager_t::GmemLayoutAtomKVCpAsync>);
+ GmemTiledCopyAppendKV gmem_tiled_copy_kv;
+ auto gmem_thr_copy_kv = gmem_tiled_copy_kv.get_thread_slice(thread_idx);
+ Tensor tKsK = gmem_thr_copy_kv.partition_S(sK); // ((Atom,AtomNum),ATOM_M,ATOM_N)
+ Tensor tKgK = gmem_thr_copy_kv.partition_D(gK);
+ Tensor tVsV = gmem_thr_copy_kv.partition_S(sV); // ((Atom,AtomNum),ATOM_M,ATOM_N)
+ Tensor tVgV = gmem_thr_copy_kv.partition_D(gV);
+ Tensor cK = cute::make_identity_tensor(select<1, 2>(TileShape_MNK{})); // (BLK_N,BLK_K) -> (blk_n,blk_k)
+ Tensor tKcK = gmem_thr_copy_kv.partition_D(cK);
+ Tensor tKpK = make_tensor<bool>(make_shape(size<2>(tKsK)));
+ #pragma unroll
+ for (int k = 0; k < size(tKpK); ++k) { tKpK(k) = get<1>(tKcK(_0{}, _0{}, k)) < get<1>(params.shape_K); }
+
+ auto store_K = [&] (int const n_block, auto const& smem_pipe_read) {
+ int const n_limit = std::min(seqlen_k_new - n_block * kBlockN, kBlockN);
+ if (get<1>(params.shape_rotary) <= 0) {
+ pipeline_k_new.consumer_wait(smem_pipe_read);
+ Tensor tKsK_cur = tKsK(_, _, _, smem_pipe_read.index());
+ if constexpr (!PagedKV) {
+ Tensor tKgK_cur = tKgK(_, _, _, n_block);
+ // Clear_OOB_K must be false since we don't want to write zeros to gmem
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/false>(
+ gmem_tiled_copy_kv, tKsK_cur, tKgK_cur, tKcK, tKpK, std::min(seqlen_k_new - n_block * kBlockN, kBlockN)
+ );
+ } else {
+ paged_kv_manager.store_K(n_block, tKsK_cur);
+ }
+ } else {
+ Tensor gK_cur = gK(_, _, n_block);
+ auto tPrKPtr = cute::conditional_return<PagedKV>(paged_kv_manager.compute_K_ptr(), nullptr);
+ if (params.is_rotary_interleaved) {
+ auto [tRrCos, tRrSin] = rotary.template load_cos_sin<true /*kInterleaved*/>(n_block);
+ pipeline_k_new.consumer_wait(smem_pipe_read);
+ rotary.template apply_K_interleaved<PagedKV>(sK(_, _, smem_pipe_read.index()), gK_cur, tKpK, tRrCos, tRrSin, tPrKPtr, n_block);
+ } else {
+ auto [tRrCosCont, tRrSinCont] = rotary.template load_cos_sin<false /*kInterleaved*/>(n_block);
+ pipeline_k_new.consumer_wait(smem_pipe_read);
+ rotary.template apply_K_contiguous<PagedKV>(sK(_, _, smem_pipe_read.index()), gK_cur, tKpK, tRrCosCont, tRrSinCont, tPrKPtr, n_block, get<1>(params.shape_K));
+ }
+ }
+ // Without this sync I'm getting race condition when seqlen_k is large
+ cutlass::arch::fence_view_async_shared();
+ // Very important: PipelineTmaAsync::consumer_release assumes that the warpgroup is synchronized
+ // before calling.
+ cutlass::arch::NamedBarrier::sync(cutlass::NumThreadsPerWarpGroup, static_cast<uint32_t>(FwdNamedBarriers::WarpSchedulerWG1) - 1 + flash::canonical_warp_group_idx_nosync() /*id*/);
+ pipeline_k_new.consumer_release(smem_pipe_read);
+ // if (thread_idx == 0) { print_tensor(tKpK); printf("\n"); printf("seqlen_limit = %d\n", seqlen_k_new - n_block * kBlockN);}
+ };
+
+ auto store_V = [&] (int const n_block, auto const& smem_pipe_read) {
+ pipeline_v_new.consumer_wait(smem_pipe_read);
+ int const n_limit = std::min(seqlen_k_new - n_block * kBlockN, kBlockN);
+ Tensor tVsV_cur = tVsV(_, _, _, smem_pipe_read.index());
+ if constexpr (!PagedKV) {
+ Tensor tVgV_cur = tVgV(_, _, _, n_block);
+ // Clear_OOB_K must be false since we don't want to write zeros to gmem
+ flash::copy</*Is_even_MN=*/false, /*Is_even_K=*/false, /*Clear_OOB_MN=*/false, /*Clear_OOB_K=*/false>(
+ gmem_tiled_copy_kv, tVsV_cur, tVgV_cur, tKcK, tKpK, n_limit);
+ } else {
+ paged_kv_manager.store_V(n_block, tVsV_cur);
+ }
+ cutlass::arch::fence_view_async_shared();
+ cutlass::arch::NamedBarrier::sync(cutlass::NumThreadsPerWarpGroup, static_cast<uint32_t>(FwdNamedBarriers::WarpSchedulerWG1) - 1 + flash::canonical_warp_group_idx_nosync() /*id*/);
+ pipeline_v_new.consumer_release(smem_pipe_read);
+ };
+
+ #pragma unroll 1
+ for (int n_block = n_block_new_max - 1; n_block >= n_block_new_min; --n_block) {
+ if constexpr (PagedKV) { paged_kv_manager.template load_page_table<true /*Seqlenk_mask*/>(n_block); }
+ store_K(n_block, smem_pipe_read);
+ // if (thread_idx == 0) { printf("Done storing K, n_block = %d, n_block_new_min = %d\n", n_block, n_block_new_min); }
+ store_V(n_block, smem_pipe_read);
+ // if (thread_idx == 0) { printf("Done storing V, n_block = %d, n_block_new_min = %d\n", n_block, n_block_new_min); }
+ ++smem_pipe_read;
+ }
+ // if (thread_idx == 0) { printf("After for loop\n"); }
+
+ // Re-signaling the NamedBarrier that we "canceled out"
+ if constexpr (UseSchedulerBarrier) {
+ if (flash::canonical_warp_group_idx_nosync() == 1) {
+ cutlass::arch::NamedBarrier::arrive(2 * cutlass::NumThreadsPerWarpGroup, static_cast<uint32_t>(FwdNamedBarriers::WarpSchedulerWG1) /*id*/);
+ }
+ }
+
+ return true;
+
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/mask.h b/src/solarwm/backends/minimax_h3/hopper_src/mask.h
new file mode 100644
index 0000000..02d0462
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/mask.h
@@ -0,0 +1,157 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cute/tensor.hpp>
+
+#include "cutlass/fast_math.h" // For cutlass::FastDivmod
+
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <int kBlockM, int kBlockN, bool PackGQA, typename TiledMma, bool SwapAB=false>
+struct Mask {
+
+ static_assert(!(PackGQA && SwapAB), "Cannot be both PackGQA and SwapAB");
+
+ int const thread_idx;
+ int const seqlen_q, seqlen_k;
+ int const window_size_left, window_size_right, sink_token_length;
+ cutlass::FastDivmod const qhead_per_khead_divmod;
+
+ CUTLASS_DEVICE
+ Mask(const int thread_idx, const int seqlen_q, const int seqlen_k,
+ const int window_size_left, const int window_size_right, const int sink_token_length,
+ cutlass::FastDivmod const &qhead_per_khead_divmod)
+ : thread_idx(thread_idx)
+ , seqlen_q(seqlen_q)
+ , seqlen_k(seqlen_k)
+ , window_size_left(window_size_left)
+ , window_size_right(window_size_right)
+ , sink_token_length(sink_token_length)
+ , qhead_per_khead_divmod(qhead_per_khead_divmod)
+ {
+ };
+
+ template <bool Seqlenk_mask=false, bool Causal_mask=false, bool Local_mask=false,
+ typename Engine, typename Layout>
+ CUTLASS_DEVICE
+ void apply(Tensor<Engine, Layout> &tSrS, const int m_block, const int n_block) const {
+ static_assert(!(Causal_mask && Local_mask), "Cannot be both causal and local");
+ static_assert(Layout::rank == 3, "Only support 3D Tensor");
+ if (!Seqlenk_mask && !Causal_mask && !Local_mask) { return; }
+
+ auto thread_mma = TiledMma{}.get_thread_slice(thread_idx);
+ auto thread0_mma = TiledMma{}.get_thread_slice(_0{});
+
+ static constexpr int Row = !SwapAB ? 0 : 1, Col = !SwapAB ? 1 : 0;
+
+ Tensor cS = cute::make_identity_tensor(Shape<Int<!SwapAB ? kBlockM : kBlockN>, Int<!SwapAB ? kBlockN : kBlockM>>{});
+ Tensor tScS = thread_mma.partition_C(cS);
+ Tensor tSrS_rowcol = make_tensor(tSrS.data(), flash::convert_layout_acc_rowcol</*Transposed=*/SwapAB>(tSrS.layout()));
+ Tensor tScS_rowcol = make_tensor(tScS.data(), flash::convert_layout_acc_rowcol</*Transposed=*/SwapAB>(tScS.layout()));
+ Tensor t0ScS = thread0_mma.partition_C(cS);
+ Tensor t0ScS_rowcol = make_tensor(t0ScS.data(), flash::convert_layout_acc_rowcol</*Transposed=*/SwapAB>(t0ScS.layout()));
+ // We want to use the col indices of thread0 to compare, since that is known at compile time.
+ // So we subtract the limit by the first col index of this thread (get<Col>(tScS_rowcol(_0{}, _0{})))
+ int const thread_col_offset = get<Col>(tScS_rowcol(_0{}, _0{}));
+ int const seqlenk_col_limit = seqlen_k - n_block * kBlockN - thread_col_offset;
+ if constexpr (!Causal_mask && !Local_mask) {
+ if constexpr (Seqlenk_mask) { // Just masking based on col
+ #pragma unroll
+ for (int n = 0; n < size<1>(tSrS_rowcol); ++n) {
+ if (int(get<Col>(t0ScS_rowcol(_0{}, n))) >= seqlenk_col_limit) {
+ #pragma unroll
+ for (int m = 0; m < size<0>(tSrS_rowcol); ++m) { tSrS_rowcol(m, n) = -INFINITY; }
+ }
+ }
+ }
+ } else { // mask based on both row and col
+ if constexpr (!SwapAB) {
+ // If PackGQA, we split the work of compute divmod among threads in the same row
+ static constexpr int kMmaThreadsPerRow = size<0, 0>(typename TiledMma::AtomLayoutC_TV{});
+ static_assert(cutlass::NumThreadsPerWarp % kMmaThreadsPerRow == 0);
+ static_assert(!PackGQA || CUTE_STATIC_V(size<0>(tSrS_rowcol)) <= kMmaThreadsPerRow);
+ int mma_m_idx;
+ // Might get OOB but it's ok since we'll check it later
+ if constexpr (PackGQA) {
+ mma_m_idx = qhead_per_khead_divmod.divide(m_block * kBlockM + get<Row>(tScS_rowcol(thread_idx % kMmaThreadsPerRow, _0{})));
+ }
+ int const causal_row_offset = 1 + seqlen_k - n_block * kBlockN - seqlen_q - thread_col_offset;
+ if constexpr (Causal_mask) {
+ #pragma unroll
+ for (int m = 0; m < size<0>(tSrS_rowcol); ++m) {
+ int const row_idx = !PackGQA
+ ? get<Row>(tScS_rowcol(m, _0{})) + m_block * kBlockM
+ : __shfl_sync(0xffffffff, mma_m_idx, m % kMmaThreadsPerRow, kMmaThreadsPerRow);
+ int const col_limit_right = !Seqlenk_mask
+ ? row_idx + causal_row_offset
+ : __viaddmin_s32(row_idx, causal_row_offset, seqlenk_col_limit);
+ #pragma unroll
+ for (int n = 0; n < size<1>(tSrS_rowcol); ++n) {
+ if (int(get<Col>(t0ScS_rowcol(_0{}, n))) >= col_limit_right) { tSrS_rowcol(m, n) = -INFINITY; }
+ }
+ }
+ } else {
+ int const local_row_offset_right = causal_row_offset + window_size_right;
+ int const local_row_offset_left = causal_row_offset - 1 - window_size_left;
+ int const col_limit_sink = sink_token_length - n_block * kBlockN;
+ #pragma unroll
+ for (int m = 0; m < size<0>(tSrS_rowcol); ++m) {
+ int const row_idx = !PackGQA
+ ? get<Row>(tScS_rowcol(m, _0{})) + m_block * kBlockM
+ : __shfl_sync(0xffffffff, mma_m_idx, m % kMmaThreadsPerRow, kMmaThreadsPerRow);
+ int const col_limit_right = !Seqlenk_mask
+ ? row_idx + local_row_offset_right
+ : __viaddmin_s32(row_idx, local_row_offset_right, seqlenk_col_limit);
+ int const col_limit_left = row_idx + local_row_offset_left;
+ #pragma unroll
+ for (int n = 0; n < size<1>(tSrS_rowcol); ++n) {
+ int const col_idx = int(get<Col>(t0ScS_rowcol(m, n)));
+ if (col_idx >= col_limit_right || (col_idx < col_limit_left && col_idx >= col_limit_sink)) { tSrS_rowcol(m, n) = -INFINITY; }
+ }
+ }
+ }
+ } else {
+ int const thread_row_offset = get<Row>(tScS_rowcol(_0{}, _0{}));
+ int const causal_row_offset = seqlenk_col_limit - seqlen_q + m_block * kBlockM + thread_row_offset;
+ if constexpr (Causal_mask) {
+ #pragma unroll
+ for (int n = 0; n < size<1>(tSrS_rowcol); ++n) {
+ int const col0 = int(get<Col>(t0ScS_rowcol(_0{}, n)));
+ // If col0 is beyond the column limit, we want to mask out the entire column, by setting
+ // row limit to be kBlockM.
+ int const row_limit_top = col0 >= seqlenk_col_limit ? kBlockM : col0 - causal_row_offset;
+ #pragma unroll
+ for (int m = 0; m < size<0>(tSrS_rowcol); ++m) {
+ if (int(get<Row>(t0ScS_rowcol(m, _0{}))) < row_limit_top) { tSrS_rowcol(m, n) = -INFINITY; }
+ }
+ }
+ } else {
+ int const col_limit_sink = sink_token_length - n_block * kBlockN - thread_col_offset;
+ #pragma unroll
+ for (int n = 0; n < size<1>(tSrS_rowcol); ++n) {
+ int const col0 = int(get<Col>(t0ScS_rowcol(_0{}, n)));
+ // If col0 is beyond the column limit, we want to mask out the entire column, by setting
+ // row limit to be kBlockM.
+ int const row_limit_top = col0 >= seqlenk_col_limit ? kBlockM : col0 - causal_row_offset - window_size_right;
+ int const row_limit_bot = col0 < col_limit_sink ? kBlockM : col0 - causal_row_offset + window_size_left;
+ #pragma unroll
+ for (int m = 0; m < size<0>(tSrS_rowcol); ++m) {
+ int const row_idx = int(get<Row>(t0ScS_rowcol(m, _0{})));
+ if (row_idx < row_limit_top || row_idx > row_limit_bot) { tSrS_rowcol(m, n) = -INFINITY; }
+ }
+ }
+ }
+ }
+ }
+ };
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/named_barrier.hpp b/src/solarwm/backends/minimax_h3/hopper_src/named_barrier.hpp
new file mode 100644
index 0000000..f77ea77
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/named_barrier.hpp
@@ -0,0 +1,76 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cutlass/arch/barrier.h"
+
+namespace flash {
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// cutlass::arch::NamedBarrier::sync/arrive are only enabled Sm90 even though they work
+// for Sm80 as well. We reimplement them here, enabled for both Sm90 and Sm80.
+
+CUTLASS_DEVICE
+static void named_barrier_sync(uint32_t num_threads, uint32_t barrier_id_) {
+ static constexpr uint32_t ReservedNamedBarrierCount = static_cast<uint32_t>(cutlass::arch::ReservedNamedBarriers::FirstUserBarrier);
+ uint32_t barrier_id = barrier_id_ + ReservedNamedBarrierCount;
+ asm volatile("bar.sync %0, %1;" : : "r"(barrier_id), "r"(num_threads));
+ cutlass::arch::synclog_emit_named_barrier_arrive_and_wait(__LINE__, num_threads, barrier_id);
+}
+
+CUTLASS_DEVICE
+static void named_barrier_sync(uint32_t num_threads, cutlass::arch::ReservedNamedBarriers reserved_named_barriers) {
+ uint32_t barrier_id = static_cast<uint32_t>(reserved_named_barriers);
+ asm volatile("bar.sync %0, %1;" : : "r"(barrier_id), "r"(num_threads));
+ cutlass::arch::synclog_emit_named_barrier_arrive_and_wait(__LINE__, num_threads, barrier_id);
+}
+
+CUTLASS_DEVICE
+static void named_barrier_arrive(uint32_t num_threads, uint32_t barrier_id_) {
+ static constexpr uint32_t ReservedNamedBarrierCount = static_cast<uint32_t>(cutlass::arch::ReservedNamedBarriers::FirstUserBarrier);
+ uint32_t barrier_id = barrier_id_ + ReservedNamedBarrierCount;
+ cutlass::arch::synclog_emit_named_barrier_arrive(__LINE__, num_threads, barrier_id);
+ asm volatile("bar.arrive %0, %1;" : : "r"(barrier_id), "r"(num_threads));
+}
+
+CUTLASS_DEVICE
+static void named_barrier_arrive(uint32_t num_threads, cutlass::arch::ReservedNamedBarriers reserved_named_barriers) {
+ uint32_t barrier_id = static_cast<uint32_t>(reserved_named_barriers);
+ cutlass::arch::synclog_emit_named_barrier_arrive(__LINE__, num_threads, barrier_id);
+ asm volatile("bar.arrive %0, %1;" : : "r"(barrier_id), "r"(num_threads));
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// Enumerates the reserved named barriers to avoid potential conflicts
+
+enum class FwdNamedBarriers {
+ QueryEmpty = 0,
+ ProducerWG = 1,
+ TileCountSmemEmpty = 2,
+ TileCountSmemFull = 3,
+ WarpSchedulerWG1 = 4,
+ WarpSchedulerWG2 = 5,
+ WarpSchedulerWG3 = 6,
+ AppendKV = 7,
+ QueryRotated = 8,
+};
+
+enum class BwdNamedBarriers {
+ KVEmpty = 0,
+ PdS = 1,
+ // This needs to match FwdNamedBarriers::TileCountSmemEmpty since TileScheduler uses it
+ TileCountSmemEmpty = 2,
+ TileCountSmemFull = 3,
+ dQEmptyWG1 = 4,
+ dQEmptyWG2 = 5,
+ dQEmptyWG3 = 6,
+ dQFullWG1 = 7,
+ dQFullWG2 = 8,
+ dQFullWG3 = 9,
+};
+
+} // flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/pack_gqa.h b/src/solarwm/backends/minimax_h3/hopper_src/pack_gqa.h
new file mode 100644
index 0000000..160bf43
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/pack_gqa.h
@@ -0,0 +1,255 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cute/tensor.hpp>
+
+#include "cutlass/fast_math.h" // For cutlass::FastDivmod
+
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <int kBlockM, int kHeadDim, int NumThreads, typename Element>
+struct PackGQAManager {
+ // We use CpAsync for Q, since TMA doesn't work there
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(Element);
+ static constexpr int kGmemElemsPerStore = kGmemElemsPerLoad;
+ static_assert(kHeadDim % kGmemElemsPerLoad == 0, "Headdim must be a multiple of kGmemElemsPerLoad");
+ // We want each "row" to have 64 elements (128 bytes, i.e. 1 cache line). E.g. if hdim=128, we want each
+ // thread to have 4 loads in the M direction and 2 vectorized load in the K direction.
+ // In the case of PackGQA, this reduces the number of times we need to call divmod.
+ static constexpr int kBytePerRow = kHeadDim * sizeof(Element);
+ static constexpr int kBlockKGmem = (kBytePerRow % 128 == 0 ? 128 : (kBytePerRow % 64 == 0 ? 64 : 32)) / sizeof(Element);
+ static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad;
+ static_assert(NumThreads % kGmemThreadsPerRow == 0, "NumThreads must be a multiple of kGmemThreadsPerRow");
+ // We assume threads loading the same row are in the same warp. This is for an optimization in PagedKV where
+ // these threads share the same page table entry and share the work of computing pointers to paged K and paged V.
+ static_assert(cutlass::NumThreadsPerWarp % kGmemThreadsPerRow == 0, "kGmemThreadsPerRow must divide NumThreadsPerWarp");
+ using GmemCopyAtomCpAsync = cute::Copy_Atom<SM80_CP_ASYNC_CACHEGLOBAL_ZFILL<uint128_t>, Element>;
+ using GmemLayoutAtom = Layout<Shape <Int<NumThreads / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+ using GmemTiledCopyQCpAsync = decltype(
+ make_tiled_copy(GmemCopyAtomCpAsync{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per load
+
+ // Was trying to have each WG loading Q to the rows in sQ that only that WG needs so that we only need
+ // to sync within each WG, but didn't seem to be any faster.
+ // using GmemLayoutAtomWG = Layout<Shape <Int<128 / kGmemThreadsPerRow>, Int<NumThreads / 128>, Int<kGmemThreadsPerRow> >,
+ // Stride<Int<kGmemThreadsPerRow>, _128, _1>>;
+ // using GmemTiledCopyQCpAsyncWG = decltype(
+ // make_tiled_copy(GmemCopyAtomCpAsync{},
+ // GmemLayoutAtomNew{},
+ // Layout<Shape<_1, _1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per load
+
+ using GmemTiledCopyO = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ GmemLayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerStore>>>{})); // Val layout, 8 or 16 vals per store
+
+ template <int NumThreadsPerRow=kGmemThreadsPerRow, typename Engine, typename Layout, typename TensorC>
+ CUTLASS_DEVICE
+ static auto
+ compute_ptr(Tensor<Engine, Layout> &tensor, TensorC const &tRows,
+ cutlass::FastDivmod const &qhead_per_khead_divmod, int const thread_idx, int const m_block) {
+ // tensor of shape ((qhead_per_khead, seqlen_q))
+ static constexpr int NumPtrPerThread = cute::ceil_div(CUTE_STATIC_V(cute::size(tRows)), NumThreadsPerRow);
+ using TensorType = typename Engine::value_type;
+ Tensor tPrPtr = make_tensor<TensorType const*>(Shape<Int<NumPtrPerThread>>{});
+ #pragma unroll
+ for (int i = 0; i < NumPtrPerThread; ++i) {
+ int const row = i * NumThreads + get<0>(tRows(thread_idx % NumThreadsPerRow));
+ int const idx = m_block * kBlockM + row;
+ int m_idx, h_idx;
+ m_idx = qhead_per_khead_divmod.divmod(h_idx, idx);
+ tPrPtr[i] = &tensor(make_coord(make_coord(h_idx, m_idx)));
+ }
+ return tPrPtr;
+ }
+
+
+ template <typename TensormQ, typename TensorsQ>
+ CUTLASS_DEVICE
+ static void
+ load_Q(TensormQ const &mQ, // ((qhead_per_khead, seqlen_q), headdim)
+ TensorsQ &sQ, // (kBlockM, kHeadDim)
+ cutlass::FastDivmod const &qhead_per_khead_divmod,
+ int const thread_idx, int const seqlen_q, int const m_block
+ )
+ {
+ GmemTiledCopyQCpAsync gmem_tiled_copy_Q_cp_async;
+ // GmemTiledCopyQCpAsyncNew gmem_tiled_copy_Q_cp_async;
+ auto gmem_thr_copy_Q_cp_async = gmem_tiled_copy_Q_cp_async.get_thread_slice(thread_idx);
+ Tensor cQ = make_identity_tensor(make_shape(size<0>(sQ), size<1>(sQ))); // (BLK_M,BLK_K) -> (blk_m,blk_k)
+ Tensor tQcQ = gmem_thr_copy_Q_cp_async.partition_S(cQ); // (ACPY,ACPY_M,ACPY_K) -> (blk_m,blk_k)
+ Tensor tQsQ = gmem_thr_copy_Q_cp_async.partition_D(sQ); // (ACPY,ACPY_M,ACPY_K) -> (blk_m,blk_k)
+ // Tensor tQcQ_ = gmem_thr_copy_Q_cp_async.partition_S(cute::flat_divide(cQ, _64{})); // (ACPY,ACPY_M,ACPY_K) -> (blk_m,blk_k)
+ // Tensor tQsQ_ = gmem_thr_copy_Q_cp_async.partition_D(cute::flat_divide(sQ, _64{})); // (ACPY,ACPY_M,ACPY_K) -> (blk_m,blk_k)
+ // Tensor tQcQ = group_modes<1, rank(tQcQ_) - 1>(tQcQ_);
+ // Tensor tQsQ = group_modes<1, rank(tQsQ_) - 1>(tQsQ_);
+ Tensor tQpQ = make_tensor<bool>(make_shape(size<2>(tQsQ)));
+ #pragma unroll
+ for (int k = 0; k < size(tQpQ); ++k) { tQpQ(k) = get<1>(tQcQ(_0{}, _0{}, k)) < size<1>(mQ); }
+
+ // Similar to loading K and V when PagedKV, it's expensive to compute the pointers for Q.
+ // We split the work among threads loading the same row of Q, then __shfl_sync the pointers.
+ Tensor mQ_0 = mQ(_, _0{});
+ Tensor tQcQ_row = tQcQ(_0{}, _, _0{});
+ Tensor tPrQPtr = compute_ptr(mQ_0, tQcQ_row, qhead_per_khead_divmod, thread_idx, m_block);
+ int const qhead_per_khead = qhead_per_khead_divmod.divisor;
+ #pragma unroll
+ for (int m = 0; m < size<1>(tQsQ); ++m) {
+ int idx = m_block * kBlockM + get<0>(tQcQ(_0{}, m, _0{}));
+ Element const* q_ptr = reinterpret_cast<Element const*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrQPtr(m / kGmemThreadsPerRow)), m % kGmemThreadsPerRow, kGmemThreadsPerRow));
+ if (idx < seqlen_q * qhead_per_khead) {
+ // if (thread_idx == 0) { printf("m: %d, m_idx: %d, h_idx: %d, q_ptr = %p, q_ptr_og = %p\n", m, m_idx, h_idx, q_ptr, &mQ_copy(0, make_coord(h_idx, m_idx), 0));}
+ Tensor mQ_cur = make_tensor(make_gmem_ptr(q_ptr), Shape<Int<kHeadDim>>{});
+ Tensor mQ_cur_copy = cute::tiled_divide(mQ_cur, Shape<Int<kGmemElemsPerLoad>>{});
+ #pragma unroll
+ for (int k = 0; k < size<2>(tQsQ); ++k) {
+ int ki = get<1>(tQcQ(_0{}, _0{}, k)) / kGmemElemsPerLoad;
+ // the "tiled_copy.with(tQpQ(k))"" will fill in zero for columns where tQpQ(k) is false
+ // TODO: check this
+ cute::copy(gmem_tiled_copy_Q_cp_async.with(tQpQ(k)), mQ_cur_copy(_, ki), tQsQ(_, m, k));
+ }
+ } // Don't need to fill in 0s for sQ since we're not gonna write the output to gmem for those rows
+ }
+ };
+
+ template <typename TensormLSE, typename TensorsLSE, typename TiledMma>
+ CUTLASS_DEVICE
+ static void
+ store_LSE(TensormLSE &mLSE, // ((qhead_per_khead, seqlen_q))
+ TensorsLSE const &tLSErLSE, // (kBlockM) split across threads according to tiled_mma
+ TiledMma tiled_mma,
+ cutlass::FastDivmod const &qhead_per_khead_divmod,
+ int const thread_idx, int const seqlen_o, int const m_block
+ )
+ {
+ Tensor caccO = cute::make_identity_tensor(Shape<Int<kBlockM>, Int<kHeadDim>>{});
+ auto thread_mma = tiled_mma.get_thread_slice(thread_idx);
+ Tensor taccOcO = thread_mma.partition_C(caccO); // (MMA,MMA_M,MMA_K)
+ Tensor taccOcO_row = make_tensor(taccOcO.data(), flash::convert_layout_acc_rowcol(taccOcO.layout()))(_, _0{});
+ CUTE_STATIC_ASSERT_V(size(tLSErLSE) == size(taccOcO_row)); // MMA_M
+
+ // If PackGQA, we split the work of compute divmod among threads in the same row
+ static constexpr int kMmaThreadsPerRow = size<0, 0>(typename TiledMma::AtomLayoutC_TV{});
+ static_assert(cutlass::NumThreadsPerWarp % kMmaThreadsPerRow == 0);
+ static_assert(CUTE_STATIC_V(size(tLSErLSE)) <= kMmaThreadsPerRow);
+ static_assert(CUTE_STATIC_V(size(taccOcO_row)) <= kMmaThreadsPerRow);
+
+ Tensor tPrLSEPtr = compute_ptr<kMmaThreadsPerRow>(mLSE, taccOcO_row, qhead_per_khead_divmod, thread_idx, m_block);
+ static_assert(CUTE_STATIC_V(size(tPrLSEPtr)) == 1);
+ int const qhead_per_khead = qhead_per_khead_divmod.divisor;
+ #pragma unroll
+ for (int mi = 0; mi < size(tLSErLSE); ++mi) {
+ int const row = m_block * kBlockM + get<0>(taccOcO_row(mi));
+ float* ptr_LSE_cur = reinterpret_cast<float*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrLSEPtr[0]), mi % kMmaThreadsPerRow, kMmaThreadsPerRow));
+ if (get<1>(taccOcO_row(_0{})) == 0 && row < seqlen_o * qhead_per_khead) {
+ *ptr_LSE_cur = tLSErLSE(mi);
+ }
+ }
+ };
+
+ template <typename TensormO, typename TensorrO>
+ CUTLASS_DEVICE
+ static void
+ store_O(TensormO &mO, // ((qhead_per_khead, seqlen_o), headdim)
+ TensorrO const &tOrO, // (kBlockM, kHeadDim) split across threads according to gmem_tiled_copy_O
+ cutlass::FastDivmod const &qhead_per_khead_divmod,
+ int const thread_idx, int const seqlen_o, int const m_block
+ )
+ {
+ GmemTiledCopyO gmem_tiled_copy_O;
+ auto gmem_thr_copy_O = gmem_tiled_copy_O.get_thread_slice(thread_idx);
+ Tensor cO = make_identity_tensor(Shape<Int<kBlockM>, Int<kHeadDim>>{}); // (BLK_M,BLK_K) -> (blk_m,blk_k)
+ Tensor tOcO = gmem_thr_copy_O.partition_D(cO); // (ACPY,ACPY_M,ACPY_K) -> (blk_m,blk_k)
+ Tensor tOpO = make_tensor<bool>(make_shape(size<2>(tOcO)));
+ #pragma unroll
+ for (int k = 0; k < size(tOpO); ++k) { tOpO(k) = get<1>(tOcO(_0{}, _0{}, k)) < size<1>(mO); }
+
+ // Similar to loading K and V when PagedKV, it's expensive to compute the pointers for O.
+ // We split the work among threads loading the same row of O, then __shfl_sync the pointers.
+ Tensor mO_0 = mO(_, _0{});
+ Tensor tOcO_row = tOcO(_0{}, _, _0{});
+ Tensor tPrOPtr = compute_ptr(mO_0, tOcO_row, qhead_per_khead_divmod, thread_idx, m_block);
+ int const qhead_per_khead = qhead_per_khead_divmod.divisor;
+ #pragma unroll
+ for (int m = 0; m < size<1>(tOrO); ++m) {
+ int idx = m_block * kBlockM + get<0>(tOcO(_0{}, m, _0{}));
+ Element* o_ptr = reinterpret_cast<Element*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrOPtr(m / kGmemThreadsPerRow)), m % kGmemThreadsPerRow, kGmemThreadsPerRow));
+ if (idx < seqlen_o * qhead_per_khead) {
+ Tensor mO_cur = make_tensor(make_gmem_ptr(o_ptr), Shape<Int<kHeadDim>>{});
+ Tensor mO_cur_copy = cute::tiled_divide(mO_cur, Shape<Int<kGmemElemsPerStore>>{});
+ #pragma unroll
+ for (int k = 0; k < size<2>(tOrO); ++k) {
+ int ki = get<1>(tOcO(_0{}, _0{}, k)) / kGmemElemsPerStore;
+ if (tOpO(k)) {
+ cute::copy(gmem_tiled_copy_O, tOrO(_, m, k), mO_cur_copy(_, ki));
+ }
+ }
+ }
+ }
+ };
+
+ template <typename TensormO, typename TensorrO, typename TiledMma>
+ CUTLASS_DEVICE
+ static void
+ store_O_direct(TensormO &mO, // ((qhead_per_khead, seqlen_o), headdim)
+ TensorrO const &tOrO, // (kBlockM, kHeadDim) split across threads according to tiled_mma
+ TiledMma tiled_mma,
+ cutlass::FastDivmod const &qhead_per_khead_divmod,
+ int const thread_idx, int const seqlen_o, int const m_block
+ )
+ {
+ static constexpr int kGmemElemsPerStoreDirect = 2;
+ cute::Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element> gmem_copy_direct;
+ // Reshape acc from ((2, 2, V), MMA_M, MMA_N) to (nrow=(2, MMA_M), ncol=(2, V, MMA_N))
+ Tensor tOrO_rowcol = make_tensor(tOrO.data(), flash::convert_layout_acc_rowcol(tOrO.layout()));
+ Tensor tOrO_copy = cute::tiled_divide(tOrO_rowcol, Shape<_1, Int<kGmemElemsPerStoreDirect>>{});
+
+ Tensor caccO = cute::make_identity_tensor(Shape<Int<kBlockM>, Int<kHeadDim>>{});
+ auto thread_mma = tiled_mma.get_thread_slice(thread_idx);
+ Tensor taccOcO = thread_mma.partition_C(caccO); // (MMA,MMA_M,MMA_K)
+ Tensor taccOcO_rowcol = make_tensor(taccOcO.data(), flash::convert_layout_acc_rowcol(taccOcO.layout()));
+ Tensor taccOcO_row = taccOcO_rowcol(_, _0{});
+ Tensor taccOcO_col = taccOcO_rowcol(_0{}, _);
+
+ // If PackGQA, we split the work of compute divmod among threads in the same row
+ static constexpr int kMmaThreadsPerRow = size<0, 0>(typename TiledMma::AtomLayoutC_TV{});
+ static_assert(cutlass::NumThreadsPerWarp % kMmaThreadsPerRow == 0);
+ static_assert(CUTE_STATIC_V(size(taccOcO_row)) <= kMmaThreadsPerRow);
+
+ // Similar to loading K and V when PagedKV, it's expensive to compute the pointers for O.
+ // We split the work among threads loading the same row of O, then __shfl_sync the pointers.
+ Tensor mO_0 = mO(_, _0{});
+ Tensor tPrOPtr = compute_ptr<kMmaThreadsPerRow>(mO_0, taccOcO_row, qhead_per_khead_divmod, thread_idx, m_block);
+ static_assert(CUTE_STATIC_V(size(tPrOPtr)) == 1);
+
+ int const qhead_per_khead = qhead_per_khead_divmod.divisor;
+ #pragma unroll
+ for (int m = 0; m < size<1>(tOrO_copy); ++m) {
+ int row = m_block * kBlockM + get<0>(taccOcO_row(m));
+ Element* o_ptr = reinterpret_cast<Element*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrOPtr[0]), m % kMmaThreadsPerRow, kMmaThreadsPerRow));
+ if (row < seqlen_o * qhead_per_khead) {
+ Tensor mO_cur = make_tensor(make_gmem_ptr(o_ptr), Shape<Int<kHeadDim>>{});
+ Tensor mO_cur_copy = cute::tiled_divide(mO_cur, Shape<Int<kGmemElemsPerStoreDirect>>{});
+ #pragma unroll
+ for (int k = 0; k < size<2>(tOrO_copy); ++k) {
+ int col = get<1>(taccOcO_col(k * kGmemElemsPerStoreDirect));
+ if (col < size<1>(mO)) {
+ cute::copy(gmem_copy_direct, tOrO_copy(_, m, k), mO_cur_copy(_, col / kGmemElemsPerStoreDirect));
+ }
+ }
+ }
+ }
+ };
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/paged_kv.h b/src/solarwm/backends/minimax_h3/hopper_src/paged_kv.h
new file mode 100644
index 0000000..0f710e5
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/paged_kv.h
@@ -0,0 +1,301 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cute/tensor.hpp>
+
+#include "cutlass/fast_math.h" // For cutlass::FastDivmod
+
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+template <int kBlockN, int kHeadDim, int NumThreads, typename Element, bool KV_Same_Iter=false, int LoadsPerRow_LB=1>
+struct PagedKVManager {
+ // If KV_Same_Iter=false, then we do load_page_table(0), load_K(0), load_page_table(1), load_K(1), load_V(0),
+ // load_page_table(2), load_K(2), load_V(1), etc.
+ // So we need to compute the V pointers for the previous iteration.
+
+ // LoadsPerRow_LB is the lower bound on number of loads per row in the K direction. This is useful for
+ // rotary where we want each thread to have at least 2 loads per row.
+
+ // We use CpAsync for K and V if PagedKV, since TMA doesn't work there
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(Element);
+ static_assert(kHeadDim % kGmemElemsPerLoad == 0, "Headdim must be a multiple of kGmemElemsPerLoad");
+ // We want each "row" to have 64 elements (128 bytes, i.e. 1 cache line). E.g. if hdim=128, we want each
+ // thread to have 4 loads in the M direction and 2 vectorized load in the K direction.
+ // In the case of PackGQA, this reduces the number of times we need to call divmod.
+ static_assert(kHeadDim % LoadsPerRow_LB == 0, "Headdim must be a multiple of LoadsPerRow_LB");
+ static constexpr int kBytePerRow = kHeadDim / LoadsPerRow_LB * sizeof(Element);
+ static constexpr int kBlockKGmem = (kBytePerRow % 128 == 0 ? 128 : (kBytePerRow % 64 == 0 ? 64 : 32)) / sizeof(Element);
+ static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad;
+ static_assert(NumThreads % kGmemThreadsPerRow == 0, "NumThreads must be a multiple of kGmemThreadsPerRow");
+ // We assume threads loading the same row are in the same warp. This is for an optimization in PagedKV where
+ // these threads share the same page table entry and share the work of computing pointers to paged K and paged V.
+ static_assert(cutlass::NumThreadsPerWarp % kGmemThreadsPerRow == 0, "kGmemThreadsPerRow must divide NumThreadsPerWarp");
+ using GmemCopyAtomCpAsync = cute::Copy_Atom<SM80_CP_ASYNC_CACHEGLOBAL_ZFILL<uint128_t>, Element>;
+ using GmemLayoutAtomKVCpAsync = Layout<Shape <Int<NumThreads / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+ using GmemTiledCopyKVCpAsync = decltype(
+ make_tiled_copy(GmemCopyAtomCpAsync{},
+ GmemLayoutAtomKVCpAsync{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per load
+ using GmemTiledCopyKVStore = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ GmemLayoutAtomKVCpAsync{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per load
+
+ using ShapeKV = cute::Shape<int32_t, int32_t, int32_t, int32_t>; // (seqlen, d, head, batch)
+ using StrideKV = cute::Stride<int64_t, _1, int64_t, int64_t>;
+ using ShapePageTable = cute::Shape<int32_t, int32_t>; // (batch, max_num_pages_per_seq)
+ using StridePageTable = cute::Stride<int64_t, _1>;
+
+ using TensorPageTable = decltype(make_tensor(make_gmem_ptr(static_cast<int const*>(nullptr)), ShapePageTable{}, StridePageTable{})(int(0), _));
+ using TensorKV = decltype(make_tensor(make_gmem_ptr(static_cast<Element*>(nullptr)), ShapeKV{}, StrideKV{})(_, _, int(0), _));
+ using GmemThrCopyKVCpAsync = decltype(GmemTiledCopyKVCpAsync{}.get_thread_slice(int(0)));
+ using TensortKcK = decltype(GmemTiledCopyKVCpAsync{}.get_thread_slice(int(0)).partition_D(cute::make_identity_tensor(Shape<Int<kBlockN>, Int<kHeadDim>>{})));
+ using TensortKpK = decltype(make_tensor<bool>(make_shape(size<1>(TensortKcK{}), size<2>(TensortKcK{})), Stride<_0, _1>{}));
+
+ // For PagedKV, it's expensive the calculate the pointers to K and V for each page table entry,
+ // since those require int64_t arithmetic. We optimize by having threads split this work.
+ // Typically there are 8 threads loading per row (e.g. hdim 64 and 128), and there are 11 rows
+ // that each thread needs to load for the case of hdim 128 and kBlockN = 176.
+ // So each of those 8 threads will calculate the K_ptr and V_ptr for 11 / 8 = 2 rows.
+ // We then use __shfl_sync to broadcast the pointers to the other threads in the warp.
+ static constexpr int kPageEntryPerThread = cute::ceil_div(size<1>(TensortKcK{}), kGmemThreadsPerRow);
+ using TensorPageOffset = decltype(make_tensor<cute::tuple<int, int>>(Shape<Int<kPageEntryPerThread>>{}));
+ using TensorKVPtr = decltype(make_tensor<Element*>(Shape<Int<kPageEntryPerThread>>{}));
+
+ GmemTiledCopyKVCpAsync gmem_tiled_copy_kv;
+ cutlass::FastDivmod const &page_size_divmod;
+ int const thread_idx;
+ int const seqlen_k;
+ int const leftpad_k;
+ GmemThrCopyKVCpAsync const gmem_thr_copy_kv;
+ TensorPageTable mPageTable;
+ TensorKV mK_paged, mV_paged;
+ TensortKpK tKpK;
+ TensorPageOffset tPrPageOffset;
+ TensorKVPtr tPrVPtr;
+
+
+ CUTLASS_DEVICE
+ PagedKVManager(int const* const ptr_page_table,
+ ShapePageTable const &shape_pagetable, StridePageTable const &stride_pagetable,
+ Element* const ptr_K, ShapeKV const &shape_K, StrideKV const &stride_K,
+ Element* const ptr_V, StrideKV const &stride_V,
+ cutlass::FastDivmod const &page_size_divmod,
+ int const bidb, int const bidh, int const thread_idx, int const seqlen_k, int const leftpad_k
+ )
+ : page_size_divmod(page_size_divmod)
+ , thread_idx(thread_idx)
+ , seqlen_k(seqlen_k)
+ , leftpad_k(leftpad_k)
+ , gmem_thr_copy_kv(gmem_tiled_copy_kv.get_thread_slice(thread_idx))
+
+ {
+ mPageTable = make_tensor(make_gmem_ptr(ptr_page_table), shape_pagetable, stride_pagetable)(bidb, _);
+ mK_paged = make_tensor(make_gmem_ptr(ptr_K), shape_K, stride_K)(_, _, bidh, _);
+ mV_paged = make_tensor(make_gmem_ptr(ptr_V), shape_K, stride_V)(_, _, bidh, _);
+ tKpK = make_tensor<bool>(make_shape(size<1>(TensortKcK{}), size<2>(TensortKcK{})), Stride<_0, _1>{});
+
+ Tensor cK = cute::make_identity_tensor(Shape<Int<kBlockN>, Int<kHeadDim>>{}); // (BLK_N,BLK_K) -> (blk_n,blk_k)
+ Tensor tKcK = gmem_thr_copy_kv.partition_S(cK);
+ #pragma unroll
+ for (int k = 0; k < size<1>(tKpK); ++k) { tKpK(_0{}, k) = get<1>(tKcK(_0{}, _0{}, k)) < get<1>(shape_K); }
+ };
+
+ template <bool Seqlenk_mask=false, bool First_iter=false>
+ CUTLASS_DEVICE
+ void load_page_table(const int n_block) {
+ // The uncoalesced gmem load is intentional. This is so that each thread only loads the page table entries
+ // it needs, and we don't need any sync between warps.
+ // Assuming 8 threads per row, and 176 rows, then the rows from 0 to 175 are loaded by
+ // threads 0, 8, 16, ..., 120, 1, 9, ..., 121, 2, 10, ..., 122, etc.
+ #pragma unroll
+ for (int i = 0; i < kPageEntryPerThread; ++i) {
+ int const row = i * NumThreads + (thread_idx % kGmemThreadsPerRow) * (NumThreads / kGmemThreadsPerRow) + (thread_idx / kGmemThreadsPerRow);
+ int const row_idx = n_block * kBlockN + row;
+ int page_idx, page_offset;
+ page_idx = page_size_divmod.divmod(page_offset, row_idx + leftpad_k);
+ // Add the condition (i + 1) * NumThreads <= kBlockN since that is an upper bound of row
+ // and is known at compile time. It avoids branching when e.g., kBlockN = 176 and i = 0.
+ int const page = ((i + 1) * NumThreads <= kBlockN || row < kBlockN) && (!Seqlenk_mask || row_idx < seqlen_k) ? mPageTable[page_idx] : 0;
+ tPrPageOffset[i] = {page, page_offset};
+ // if (cute::thread0()) { printf("row = %d, page_idx = %d, page_offset = %d, page = %d, leftpad_k = %d, seqlen_k = %d\n", row, page_idx, page_offset, page, leftpad_k, seqlen_k); }
+ }
+ if constexpr (First_iter && !KV_Same_Iter) { compute_V_ptr(); }
+ };
+
+ CUTLASS_DEVICE
+ TensorKVPtr compute_K_ptr() {
+ Tensor tPrKPtr = make_tensor<Element*>(Shape<Int<kPageEntryPerThread>>{});
+ #pragma unroll
+ for (int i = 0; i < kPageEntryPerThread; ++i) {
+ auto [page, page_offset] = tPrPageOffset[i];
+ tPrKPtr[i] = &mK_paged(page_offset, _0{}, page);
+ }
+ return tPrKPtr;
+ };
+
+ CUTLASS_DEVICE
+ void compute_V_ptr() {
+ #pragma unroll
+ for (int i = 0; i < kPageEntryPerThread; ++i) {
+ auto [page, page_offset] = tPrPageOffset[i];
+ tPrVPtr[i] = &mV_paged(page_offset, _0{}, page);
+ }
+ };
+
+ template <bool Seqlenk_mask=false, typename TensorK>
+ CUTLASS_DEVICE
+ void load_K(const int n_block, TensorK &&sK) {
+ // Do we need bound check to make sure the row doesn't go above kBlockN
+ static constexpr bool EvenN = kBlockN % CUTE_STATIC_V(shape<0>(GmemLayoutAtomKVCpAsync{})) == 0;
+
+ Tensor tPrKPtr = compute_K_ptr();
+
+ // Only for index calculation, since all the indices of thread 0 are known at compile time
+ auto gmem_thr0_copy_kv = gmem_tiled_copy_kv.get_thread_slice(_0{});
+ Tensor tKsK = gmem_thr_copy_kv.partition_D(sK);
+ Tensor cK = cute::make_identity_tensor(Shape<Int<kBlockN>, Int<kHeadDim>>{}); // (BLK_N,BLK_K) -> (blk_n,blk_k)
+ // Repeat the partitioning with identity layouts
+ Tensor tKcK = gmem_thr_copy_kv.partition_S(cK);
+ Tensor t0KcK = gmem_thr0_copy_kv.partition_S(cK);
+
+ // We want to use the row indices of thread0 to compare, since that is known at compile time.
+ // So we subtract the limit by the first row index of this thread (get<0>(tKcK(_0{}, _0{}, _0{})))
+ int const seqlenk_row_limit = -int(get<0>(tKcK(_0{}, _0{}, _0{}))) + (EvenN
+ ? seqlen_k - n_block * kBlockN
+ : (!Seqlenk_mask ? kBlockN : std::min(seqlen_k - n_block * kBlockN, kBlockN)));
+ #pragma unroll
+ for (int m = 0; m < size<1>(tKsK); ++m) {
+ bool const should_load = EvenN
+ ? (!Seqlenk_mask || get<0>(t0KcK(_0{}, m, _0{})) < seqlenk_row_limit)
+ : get<0>(t0KcK(_0{}, m, _0{})) < seqlenk_row_limit;
+ Element const* k_ptr = reinterpret_cast<Element const*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrKPtr(m / kGmemThreadsPerRow)), (m % kGmemThreadsPerRow), kGmemThreadsPerRow));
+ Tensor mK_paged_cur = make_tensor(make_gmem_ptr(k_ptr), Shape<Int<kHeadDim>>{});
+ Tensor mK_paged_cur_copy = cute::tiled_divide(mK_paged_cur, Shape<Int<kGmemElemsPerLoad>>{});
+ if (should_load) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(tKsK); ++k) {
+ int const ki = get<1>(tKcK(_0{}, _0{}, k)) / kGmemElemsPerLoad;
+ cute::copy(gmem_tiled_copy_kv.with(tKpK(_0{}, k)), mK_paged_cur_copy(_, ki), tKsK(_, m, k));
+ }
+ } // Don't need to clear out the rest of the smem since we'll mask out the scores anyway
+ }
+ };
+
+ template <bool Seqlenk_mask=false, typename TensorV>
+ CUTLASS_DEVICE
+ void load_V(const int n_block, TensorV &&sV) {
+ // Do we need bound check to make sure the row doesn't go above kBlockN
+ static constexpr bool EvenN = kBlockN % CUTE_STATIC_V(shape<0>(GmemLayoutAtomKVCpAsync{})) == 0;
+
+ if constexpr (KV_Same_Iter) { compute_V_ptr(); }
+ // Only for index calculation, since all the indices of thread 0 are known at compile time
+ auto gmem_thr0_copy_kv = gmem_tiled_copy_kv.get_thread_slice(_0{});
+ Tensor tVsV = gmem_thr_copy_kv.partition_D(sV);
+ Tensor cK = cute::make_identity_tensor(Shape<Int<kBlockN>, Int<kHeadDim>>{}); // (BLK_N,BLK_K) -> (blk_n,blk_k)
+ // Repeat the partitioning with identity layouts
+ Tensor tKcK = gmem_thr_copy_kv.partition_S(cK);
+ Tensor t0KcK = gmem_thr0_copy_kv.partition_S(cK);
+
+ int const seqlenk_row_limit = seqlen_k - n_block * kBlockN - get<0>(tKcK(_0{}, _0{}, _0{}));
+ #pragma unroll
+ for (int m = 0; m < size<1>(tVsV); ++m) {
+ // Faster to rely on the cp.async to clear smem that are out of bound,
+ // rather than calling cute::clear directly.
+ // We have to be careful not to write to smem past `kBlockN` if !EvenN.
+ // If kBlockN doesn't evenly divide the tiled copy, only the last `m` needs to checked
+ if (EvenN || m < size<1>(tVsV) - 1 || get<0>(tKcK(_0{}, m, _0{})) < kBlockN) {
+ bool const should_load = !Seqlenk_mask || get<0>(t0KcK(_0{}, m, _0{})) < seqlenk_row_limit;
+ Element const* v_ptr = reinterpret_cast<Element const*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrVPtr(m / kGmemThreadsPerRow)), m % kGmemThreadsPerRow, kGmemThreadsPerRow));
+ Tensor mV_paged_cur = make_tensor(make_gmem_ptr(v_ptr), Shape<Int<kHeadDim>>{});
+ Tensor mV_paged_cur_copy = cute::tiled_divide(mV_paged_cur, Shape<Int<kGmemElemsPerLoad>>{});
+ #pragma unroll
+ for (int k = 0; k < size<2>(tVsV); ++k) {
+ int const ki = get<1>(tKcK(_0{}, _0{}, k)) / kGmemElemsPerLoad;
+ cute::copy(gmem_tiled_copy_kv.with(tKpK(_0{}, k) && should_load), mV_paged_cur_copy(_, ki), tVsV(_, m, k));
+ }
+ }
+ }
+ if constexpr (!KV_Same_Iter) { compute_V_ptr(); }
+ };
+
+ template <typename TensorK>
+ CUTLASS_DEVICE
+ void store_K(const int n_block, TensorK &&tKrK) {
+ Tensor tPrKPtr = compute_K_ptr();
+ // We're using the same partitioning as GmemTiledCopyKVCpAsync (used for loading)
+ // Only for index calculation, since all the indices of thread 0 are known at compile time
+ auto gmem_thr0_copy_kv = gmem_tiled_copy_kv.get_thread_slice(_0{});
+ Tensor cK = cute::make_identity_tensor(Shape<Int<kBlockN>, Int<kHeadDim>>{}); // (BLK_N,BLK_K) -> (blk_n,blk_k)
+ // Repeat the partitioning with identity layouts
+ Tensor tKcK = gmem_thr_copy_kv.partition_S(cK);
+ Tensor t0KcK = gmem_thr0_copy_kv.partition_S(cK);
+
+ GmemTiledCopyKVStore gmem_tiled_copy_kv_store;
+ // We want to use the row indices of thread0 to compare, since that is known at compile time.
+ // So we subtract the limit by the first row index of this thread (get<0>(tKcK(_0{}, _0{}, _0{})))
+ // int const seqlenk_row_limit = seqlen_k - n_block * kBlockN - get<0>(tKcK(_0{}, _0{}, _0{}));
+ int const seqlenk_row_limit = std::min(seqlen_k - n_block * kBlockN, kBlockN) - get<0>(tKcK(_0{}, _0{}, _0{}));
+ // if (threadIdx.x == 128) { printf("bidx = %d, bidy = %d, bidz = %d, seqlen_k = %d, seqlenk_row_limit = %d\n", blockIdx.x, blockIdx.y, blockIdx.z, seqlen_k, seqlenk_row_limit); }
+ #pragma unroll
+ for (int m = 0; m < size<1>(tKrK); ++m) {
+ bool const should_load = get<0>(t0KcK(_0{}, m, _0{})) < seqlenk_row_limit;
+ Element* k_ptr = reinterpret_cast<Element*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrKPtr(m / kGmemThreadsPerRow)), (m % kGmemThreadsPerRow), kGmemThreadsPerRow));
+ Tensor mK_paged_cur = make_tensor(make_gmem_ptr(k_ptr), Shape<Int<kHeadDim>>{});
+ Tensor mK_paged_cur_copy = cute::tiled_divide(mK_paged_cur, Shape<Int<kGmemElemsPerLoad>>{});
+ if (should_load) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(tKrK); ++k) {
+ int const ki = get<1>(tKcK(_0{}, _0{}, k)) / kGmemElemsPerLoad;
+ if (tKpK(_0{}, k)) {
+ cute::copy(gmem_tiled_copy_kv_store, tKrK(_, m, k), mK_paged_cur_copy(_, ki));
+ }
+ }
+ }
+ }
+ };
+
+ template <typename TensorV>
+ CUTLASS_DEVICE
+ void store_V(const int n_block, TensorV &&tVrV) {
+ if constexpr (KV_Same_Iter) { compute_V_ptr(); }
+ // Only for index calculation, since all the indices of thread 0 are known at compile time
+ auto gmem_thr0_copy_kv = gmem_tiled_copy_kv.get_thread_slice(_0{});
+ Tensor cK = cute::make_identity_tensor(Shape<Int<kBlockN>, Int<kHeadDim>>{}); // (BLK_N,BLK_K) -> (blk_n,blk_k)
+ // Repeat the partitioning with identity layouts
+ Tensor tKcK = gmem_thr_copy_kv.partition_S(cK);
+ Tensor t0KcK = gmem_thr0_copy_kv.partition_S(cK);
+
+ GmemTiledCopyKVStore gmem_tiled_copy_kv_store;
+ int const seqlenk_row_limit = std::min(seqlen_k - n_block * kBlockN, kBlockN) - get<0>(tKcK(_0{}, _0{}, _0{}));
+ #pragma unroll
+ for (int m = 0; m < size<1>(tVrV); ++m) {
+ bool const should_load = get<0>(t0KcK(_0{}, m, _0{})) < seqlenk_row_limit;
+ Element* v_ptr = reinterpret_cast<Element*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrVPtr(m / kGmemThreadsPerRow)), m % kGmemThreadsPerRow, kGmemThreadsPerRow));
+ Tensor mV_paged_cur = make_tensor(make_gmem_ptr(v_ptr), Shape<Int<kHeadDim>>{});
+ Tensor mV_paged_cur_copy = cute::tiled_divide(mV_paged_cur, Shape<Int<kGmemElemsPerLoad>>{});
+ if (should_load) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(tVrV); ++k) {
+ int const ki = get<1>(tKcK(_0{}, _0{}, k)) / kGmemElemsPerLoad;
+ if (tKpK(_0{}, k)) {
+ cute::copy(gmem_tiled_copy_kv_store, tVrV(_, m, k), mV_paged_cur_copy(_, ki));
+ }
+ }
+ }
+ }
+ if constexpr (!KV_Same_Iter) { compute_V_ptr(); }
+ };
+
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/rotary.h b/src/solarwm/backends/minimax_h3/hopper_src/rotary.h
new file mode 100644
index 0000000..5e30456
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/rotary.h
@@ -0,0 +1,489 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cute/tensor.hpp>
+
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Engine1, typename Layout1, typename Engine2, typename Layout2>
+CUTLASS_DEVICE void
+apply_rotary_interleaved(Tensor<Engine1, Layout1> &rK,
+ Tensor<Engine2, Layout2> const &rCos,
+ Tensor<Engine2, Layout2> const &rSin) {
+ CUTE_STATIC_ASSERT_V(rank(rK) == _1{});
+ CUTE_STATIC_ASSERT_V(rank(rCos) == _1{});
+ CUTE_STATIC_ASSERT_V(rank(rSin) == _1{});
+ CUTE_STATIC_ASSERT_V(size<0>(rCos) == size<0>(rSin));
+ static_assert(decltype(size<0>(rK))::value == decltype(size<0>(rCos))::value * 2);
+ static_assert(decltype(size<0>(rCos))::value % 2 == 0); // Since we do fast conversion from fp16/bf16 to fp32
+ Tensor K_fp32 = make_tensor_like<float>(rK);
+ convert_type_out(rK, K_fp32);
+ Tensor cos_fp32 = make_tensor_like<float>(rCos);
+ convert_type_out(rCos, cos_fp32);
+ Tensor sin_fp32 = make_tensor_like<float>(rSin);
+ convert_type_out(rSin, sin_fp32);
+ #pragma unroll
+ for (int i = 0; i < size<0>(K_fp32) / 2; ++i) {
+ float real = K_fp32[2 * i] * cos_fp32[i] - K_fp32[2 * i + 1] * sin_fp32[i];
+ float imag = K_fp32[2 * i] * sin_fp32[i] + K_fp32[2 * i + 1] * cos_fp32[i];
+ K_fp32[2 * i] = real;
+ K_fp32[2 * i + 1] = imag;
+ }
+ convert_type_out(K_fp32, rK);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Engine1, typename Layout1, typename Engine2, typename Layout2>
+CUTLASS_DEVICE void
+apply_rotary_contiguous(Tensor<Engine1, Layout1> &rK_left,
+ Tensor<Engine1, Layout1> &rK_right,
+ Tensor<Engine2, Layout2> const &rCos,
+ Tensor<Engine2, Layout2> const &rSin) {
+ CUTE_STATIC_ASSERT_V(rank(rK_left) == _1{});
+ CUTE_STATIC_ASSERT_V(rank(rK_right) == _1{});
+ CUTE_STATIC_ASSERT_V(rank(rCos) == _1{});
+ CUTE_STATIC_ASSERT_V(rank(rSin) == _1{});
+ CUTE_STATIC_ASSERT_V(size<0>(rK_left) == size<0>(rK_right));
+ CUTE_STATIC_ASSERT_V(size<0>(rK_left) == size<0>(rCos));
+ CUTE_STATIC_ASSERT_V(size<0>(rCos) == size<0>(rSin));
+ static_assert(decltype(size<0>(rCos))::value % 2 == 0); // Since we do fast conversion from fp16/bf16 to fp32
+ Tensor K_left_fp32 = make_tensor_like<float>(rK_left);
+ convert_type_out(rK_left, K_left_fp32);
+ Tensor K_right_fp32 = make_tensor_like<float>(rK_right);
+ convert_type_out(rK_right, K_right_fp32);
+ Tensor cos_fp32 = make_tensor_like<float>(rCos);
+ convert_type_out(rCos, cos_fp32);
+ Tensor sin_fp32 = make_tensor_like<float>(rSin);
+ convert_type_out(rSin, sin_fp32);
+ #pragma unroll
+ for (int i = 0; i < size<0>(K_left_fp32); ++i) {
+ float real = K_left_fp32[i] * cos_fp32[i] - K_right_fp32[i] * sin_fp32[i];
+ float imag = K_left_fp32[i] * sin_fp32[i] + K_right_fp32[i] * cos_fp32[i];
+ K_left_fp32[i] = real;
+ K_right_fp32[i] = imag;
+ }
+ convert_type_out(K_left_fp32, rK_left);
+ convert_type_out(K_right_fp32, rK_right);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <int kBlockMN, int kHeadDim, int NumThreads, typename Element, bool FixedPosition=false>
+struct Rotary {
+
+ static constexpr int kGmemElemsPerLoad = sizeof(cute::uint128_t) / sizeof(Element);
+ static_assert(kHeadDim % kGmemElemsPerLoad == 0, "Headdim must be a multiple of kGmemElemsPerLoad");
+ // We want each "row" to have 64 elements (128 bytes, i.e. 1 cache line). E.g. if hdim=128, we want each
+ // thread to have 4 loads in the M direction and 2 vectorized load in the K direction.
+ // We want each thread to have at least 2 loads in the K direction since in the case of non-interleaved
+ // rotary (combining elements at indices 0 and rotary_dim/2, 1 and rotary_dim/2+1, etc), each thread will
+ // load twice from the same row.
+ static constexpr int kBytePerHalfRow = kHeadDim / 2 * sizeof(Element);
+ static constexpr int kBlockKGmem = (kBytePerHalfRow % 128 == 0 ? 128 : (kBytePerHalfRow % 64 == 0 ? 64 : 32)) / sizeof(Element);
+ static constexpr int kGmemThreadsPerRow = kBlockKGmem / kGmemElemsPerLoad;
+ static_assert(NumThreads % kGmemThreadsPerRow == 0, "NumThreads must be a multiple of kGmemThreadsPerRow");
+ // We assume threads loading the same row are in the same warp.
+ static_assert(cutlass::NumThreadsPerWarp % kGmemThreadsPerRow == 0, "kGmemThreadsPerRow must divide NumThreadsPerWarp");
+
+ using LayoutAtom = Layout<Shape <Int<NumThreads / kGmemThreadsPerRow>, Int<kGmemThreadsPerRow>>,
+ Stride<Int<kGmemThreadsPerRow>, _1>>;
+ using TiledCopyQK = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ LayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per store
+ using GmemTiledCopyRotary = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<64>, Element>{},
+ LayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad / 2>>>{})); // Val layout, 4 or 8 vals per store
+ using GmemTiledCopyRotaryCont = decltype(
+ make_tiled_copy(Copy_Atom<AutoVectorizingCopyWithAssumedAlignment<128>, Element>{},
+ LayoutAtom{},
+ Layout<Shape<_1, Int<kGmemElemsPerLoad>>>{})); // Val layout, 8 or 16 vals per store
+
+ using ShapeRotary = cute::Shape<int32_t, int32_t>; // (seqlen_ro, rotary_dim // 2)
+ using StrideRotary = cute::Stride<int64_t, _1>;
+
+ using GmemThrCopyRotary = decltype(GmemTiledCopyRotary{}.get_thread_slice(int(0)));
+ using GmemThrCopyRotaryCont = decltype(GmemTiledCopyRotaryCont{}.get_thread_slice(int(0)));
+ using TensortRcR = decltype(GmemTiledCopyRotary{}.get_thread_slice(int(0)).partition_D(cute::make_identity_tensor(Shape<Int<kBlockMN>, Int<kHeadDim / 2>>{})));
+ using TensortRpR = decltype(make_tensor<bool>(make_shape(size<2>(TensortRcR{}))));
+ using TensortRcRCont = decltype(GmemTiledCopyRotaryCont{}.get_thread_slice(int(0)).partition_D(cute::make_identity_tensor(Shape<Int<kBlockMN>, Int<kHeadDim / 2>>{})));
+ using TensortRpRCont = decltype(make_tensor<bool>(make_shape(size<2>(TensortRcRCont{}))));
+ using TensormR = decltype(make_tensor(
+ make_gmem_ptr((Element const*)nullptr),
+ ShapeRotary{},
+ make_stride(cute::conditional_return<FixedPosition>(_0{}, int64_t(0)), _1{})));
+ using TensortRgR = decltype(
+ GmemTiledCopyRotary{}.get_thread_slice(int(0)).partition_S(make_tensor(
+ make_gmem_ptr((Element const*)nullptr),
+ make_shape(Int<kBlockMN>{}, Int<kHeadDim / 2>{}, int(0)),
+ make_stride(cute::conditional_return<FixedPosition>(_0{}, int64_t(0)), _1{}, cute::conditional_return<FixedPosition>(_0{}, int64_t(0))))));
+ using TensortRgRCont = decltype(
+ GmemTiledCopyRotaryCont{}.get_thread_slice(int(0)).partition_S(make_tensor(
+ make_gmem_ptr((Element const*)nullptr),
+ make_shape(Int<kBlockMN>{}, Int<kHeadDim / 2>{}, int(0)),
+ make_stride(cute::conditional_return<FixedPosition>(_0{}, int64_t(0)), _1{}, cute::conditional_return<FixedPosition>(_0{}, int64_t(0))))));
+
+ GmemTiledCopyRotary gmem_tiled_copy_rotary;
+ GmemTiledCopyRotaryCont gmem_tiled_copy_rotary_cont;
+ bool const is_rotary_interleaved;
+ int const rotary_dim;
+ int const thread_idx;
+ int const max_seqlen;
+ GmemThrCopyRotary const gmem_thr_copy_rotary;
+ GmemThrCopyRotaryCont const gmem_thr_copy_rotary_cont;
+ TensortRpR tRpR;
+ TensortRpRCont tRpRCont;
+ TensormR mCos, mSin;
+ TensortRgR tRgCos, tRgSin;
+ TensortRgRCont tRgCosCont, tRgSinCont;
+
+ CUTLASS_DEVICE
+ Rotary(Element const* const ptr_rotary_cos, ShapeRotary const &shape_rotary, StrideRotary const &stride_rotary_cos_,
+ Element const* const ptr_rotary_sin, StrideRotary const &stride_rotary_sin_,
+ bool const is_rotary_interleaved, int const thread_idx, int const max_seqlen, int const start_idx)
+ : is_rotary_interleaved(is_rotary_interleaved)
+ , rotary_dim(get<1>(shape_rotary) * 2)
+ , thread_idx(thread_idx)
+ , max_seqlen(max_seqlen)
+ , gmem_thr_copy_rotary(gmem_tiled_copy_rotary.get_thread_slice(thread_idx))
+ , gmem_thr_copy_rotary_cont(gmem_tiled_copy_rotary_cont.get_thread_slice(thread_idx))
+
+ {
+ auto stride_rotary_cos = make_stride(cute::conditional_return<!FixedPosition>(get<0>(stride_rotary_cos_), _0{}), get<1>(stride_rotary_cos_));
+ auto stride_rotary_sin = make_stride(cute::conditional_return<!FixedPosition>(get<0>(stride_rotary_sin_), _0{}), get<1>(stride_rotary_sin_));
+ mCos = make_tensor(make_gmem_ptr(ptr_rotary_cos + start_idx * get<0>(stride_rotary_cos_)), shape_rotary, stride_rotary_cos);
+ mSin = make_tensor(make_gmem_ptr(ptr_rotary_sin + start_idx * get<0>(stride_rotary_sin_)), shape_rotary, stride_rotary_sin);
+ Tensor gCos = local_tile(mCos, Shape<Int<kBlockMN>, Int<kHeadDim / 2>>{}, make_coord(_, _0{})); // (MN, K / 2, _)
+ Tensor gSin = local_tile(mSin, Shape<Int<kBlockMN>, Int<kHeadDim / 2>>{}, make_coord(_, _0{})); // (MN, K / 2, _)
+ tRgCos = gmem_thr_copy_rotary.partition_S(gCos);
+ tRgSin = gmem_thr_copy_rotary.partition_S(gSin);
+ tRgCosCont = gmem_thr_copy_rotary_cont.partition_S(gCos);
+ tRgSinCont = gmem_thr_copy_rotary_cont.partition_S(gSin);
+ Tensor cR = cute::make_identity_tensor(Shape<Int<kBlockMN>, Int<kHeadDim / 2>>{}); // (BLK_N,BLK_K / 2)
+ Tensor tRcR = gmem_thr_copy_rotary.partition_D(cR);
+ tRpR = make_tensor<bool>(make_shape(size<2>(tRcR)));
+ #pragma unroll
+ for (int k = 0; k < size(tRpR); ++k) { tRpR(k) = get<1>(tRcR(_0{}, _0{}, k)) < get<1>(shape_rotary); }
+ Tensor tRcRCont = gmem_thr_copy_rotary_cont.partition_D(cR);
+ tRpRCont = make_tensor<bool>(make_shape(size<2>(tRcRCont)));
+ #pragma unroll
+ for (int k = 0; k < size(tRpRCont); ++k) { tRpRCont(k) = get<1>(tRcRCont(_0{}, _0{}, k)) < get<1>(shape_rotary); }
+ };
+
+ template <bool kInterleaved=true>
+ CUTLASS_DEVICE
+ auto load_cos_sin(int const block) {
+ using GmemTiledCopyRo = std::conditional_t<kInterleaved, GmemTiledCopyRotary, GmemTiledCopyRotaryCont>;
+ auto gmem_thr_copy_ro = cute::conditional_return<kInterleaved>(gmem_thr_copy_rotary, gmem_thr_copy_rotary_cont);
+ Tensor tRpRCur = cute::conditional_return<kInterleaved>(tRpR, tRpRCont);
+ Tensor tRgCosCur = cute::conditional_return<kInterleaved>(tRgCos, tRgCosCont)(_, _, _, block);
+ Tensor tRgSinCur = cute::conditional_return<kInterleaved>(tRgSin, tRgSinCont)(_, _, _, block);
+ // make_tensor_like, not make_fragment_like. If the row_stride is _0{} we want to keep it that way
+ Tensor tRrCos = make_tensor_like(tRgCosCur);
+ Tensor tRrSin = make_tensor_like(tRgSinCur);
+ Tensor cR = cute::make_identity_tensor(Shape<Int<kBlockMN>, Int<kHeadDim / 2>>{}); // (BLK_N,BLK_K / 2)
+ Tensor tRcR = gmem_thr_copy_ro.partition_D(cR);
+ // If FixedPosition, only copy the first row as we only need the cos/sin for position cache_seqlens
+ #pragma unroll
+ for (int m = 0; m < (!FixedPosition ? size<1>(tRrCos) : 1); ++m) {
+ if (get<0>(tRcR(_0{}, m, _0{})) < std::min(max_seqlen - block * kBlockMN, kBlockMN)) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(tRrCos); ++k) {
+ if (tRpRCur(k)) {
+ cute::copy(GmemTiledCopyRo{}, tRgCosCur(_, m, k), tRrCos(_, m, k));
+ cute::copy(GmemTiledCopyRo{}, tRgSinCur(_, m, k), tRrSin(_, m, k));
+ }
+ }
+ }
+ }
+ return cute::make_tuple(tRrCos, tRrSin);;
+ }
+
+ template <bool kInterleaved=true>
+ CUTLASS_DEVICE
+ auto load_cos_sin_packgqa(int const block, cutlass::FastDivmod const &qhead_per_khead_divmod) {
+ static constexpr int kGmemElemsPerLoadCur = kInterleaved ? kGmemElemsPerLoad / 2 : kGmemElemsPerLoad;
+ using GmemTiledCopyRo = std::conditional_t<kInterleaved, GmemTiledCopyRotary, GmemTiledCopyRotaryCont>;
+ auto gmem_thr_copy_ro = cute::conditional_return<kInterleaved>(gmem_thr_copy_rotary, gmem_thr_copy_rotary_cont);
+ Tensor tRpRCur = cute::conditional_return<kInterleaved>(tRpR, tRpRCont);
+ // make_tensor_like, not make_fragment_like. If the row_stride is _0{} we want to keep it that way
+ Tensor tRrCos = make_tensor_like(cute::conditional_return<kInterleaved>(tRgCos, tRgCosCont)(_, _, _, _0{}));
+ Tensor tRrSin = make_tensor_like(cute::conditional_return<kInterleaved>(tRgSin, tRgSinCont)(_, _, _, _0{}));
+ int const qhead_per_khead = qhead_per_khead_divmod.divisor;
+ Tensor cR = cute::make_identity_tensor(Shape<Int<kBlockMN>, Int<kHeadDim / 2>>{}); // (BLK_N,BLK_K / 2)
+ Tensor tRcR = gmem_thr_copy_ro.partition_D(cR);
+
+ // The main bottleneck here is actually instruction cache misses.
+
+ // Similar to PagedKV, it's expensive to compute the pointers.
+ // We split the work among threads loading the same row, then __shfl_sync the pointers.
+ static constexpr int NumPtrPerThread = cute::ceil_div(CUTE_STATIC_V(cute::size<1>(tRrCos)), kGmemThreadsPerRow);
+ Tensor tPrCosPtr = make_tensor<Element const*>(Shape<Int<NumPtrPerThread>>{});
+ Tensor tPrSinPtr = make_tensor<Element const*>(Shape<Int<NumPtrPerThread>>{});
+ #pragma unroll
+ for (int i = 0; i < NumPtrPerThread; ++i) {
+ int const row = i * NumThreads + get<0>(tRcR(_0{}, thread_idx % kGmemThreadsPerRow, _0{}));
+ int const idx = block * kBlockMN + row;
+ int row_actual = qhead_per_khead_divmod.divide(idx);
+ tPrCosPtr[i] = &mCos(row_actual, _0{});
+ tPrSinPtr[i] = &mSin(row_actual, _0{});
+ }
+
+ #pragma unroll
+ for (int m = 0; m < (!FixedPosition ? size<1>(tRgCos) : 1); ++m) {
+ int const idx = block * kBlockMN + get<0>(tRcR(_0{}, m, _0{}));
+ Element const* cos_ptr = reinterpret_cast<Element const*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrCosPtr(m / kGmemThreadsPerRow)), m % kGmemThreadsPerRow, kGmemThreadsPerRow));
+ Element const* sin_ptr = reinterpret_cast<Element const*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrSinPtr(m / kGmemThreadsPerRow)), m % kGmemThreadsPerRow, kGmemThreadsPerRow));
+ if (idx < max_seqlen * qhead_per_khead) {
+ Tensor mCos_copy = cute::tiled_divide(make_tensor(make_gmem_ptr(cos_ptr), Shape<Int<kHeadDim / 2>>{}),
+ Shape<Int<kGmemElemsPerLoadCur>>{});
+ Tensor mSin_copy = cute::tiled_divide(make_tensor(make_gmem_ptr(sin_ptr), Shape<Int<kHeadDim / 2>>{}),
+ Shape<Int<kGmemElemsPerLoadCur>>{});
+ #pragma unroll
+ for (int k = 0; k < size<2>(tRgCos); ++k) {
+ int const ki = get<1>(tRcR(_0{}, _0{}, k)) / (kGmemElemsPerLoadCur);
+ if (tRpRCur(k)) {
+ cute::copy(GmemTiledCopyRo{}, mCos_copy(_, ki), tRrCos(_, m, k));
+ cute::copy(GmemTiledCopyRo{}, mSin_copy(_, ki), tRrSin(_, m, k));
+ }
+ }
+ }
+ }
+ return cute::make_tuple(tRrCos, tRrSin);
+ }
+
+ template <typename TensorsQ, typename TensortRrR>
+ CUTLASS_DEVICE
+ void
+ apply_Q_interleaved(TensorsQ &sQ, // (kBlockM, kHeadDim)
+ TensortRrR const &tRrCos, // (kBlockM, kHeadDim / 2) split according to GmemThrCopyRotary
+ TensortRrR const &tRrSin, // (kBlockM, kHeadDim / 2) split according to GmemThrCopyRotary
+ int const m_block, int const qhead_per_khead=1)
+ {
+ TiledCopyQK tiled_copy_q;
+ auto gmem_thr_copy_q = tiled_copy_q.get_thread_slice(thread_idx);
+ Tensor tQsQ = gmem_thr_copy_q.partition_S(sQ);
+ Tensor tQcQ = gmem_thr_copy_q.partition_S(cute::make_identity_tensor(Shape<Int<kBlockMN>, Int<kHeadDim>>{}));
+
+ CUTE_STATIC_ASSERT_V(rank(tQsQ) == _3{});
+ CUTE_STATIC_ASSERT_V(rank(tRrCos) == _3{});
+ CUTE_STATIC_ASSERT_V(rank(tRrSin) == _3{});
+ CUTE_STATIC_ASSERT_V(size<1>(tQsQ) == size<1>(tRrCos));
+ CUTE_STATIC_ASSERT_V(size<2>(tQsQ) == size<2>(tRrCos));
+ CUTE_STATIC_ASSERT_V(size<1>(tQsQ) == size<1>(tRrSin));
+ CUTE_STATIC_ASSERT_V(size<2>(tQsQ) == size<2>(tRrSin));
+ CUTE_STATIC_ASSERT_V(size<0>(tRrCos) == size<0>(tRrSin));
+ static_assert(decltype(size<0>(tQsQ))::value == decltype(size<0>(tRrCos))::value * 2);
+ static_assert(decltype(size<0>(tRrCos))::value % 2 == 0); // Since we do fast conversion from fp16/bf16 to fp32
+
+ #pragma unroll
+ for (int m = 0; m < size<1>(tQsQ); ++m) {
+ if (get<0>(tQcQ(_0{}, m, _0{})) < std::min(max_seqlen * qhead_per_khead - m_block * kBlockMN, kBlockMN)) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(tQsQ); ++k) {
+ if (tRpR(k)) {
+ Tensor rQ = make_fragment_like(tQsQ(_, m, k));
+ cute::copy(tiled_copy_q, tQsQ(_, m, k), rQ);
+ apply_rotary_interleaved(rQ, tRrCos(_, m, k), tRrSin(_, m, k));
+ cute::copy(tiled_copy_q, rQ, tQsQ(_, m, k));
+ }
+ }
+ }
+ }
+ };
+
+ template <typename TensorsQ, typename TensortRrR>
+ CUTLASS_DEVICE
+ void
+ apply_Q_contiguous(TensorsQ &sQ, // (kBlockM, kHeadDim)
+ TensortRrR const &tRrCosCont, // (kBlockM, kHeadDim / 2) split according to GmemThrCopyRotaryCont
+ TensortRrR const &tRrSinCont, // (kBlockM, kHeadDim / 2) split according to GmemThrCopyRotaryCont
+ int const m_block, int const qhead_per_khead=1)
+ {
+ TiledCopyQK tiled_copy_q;
+ auto gmem_thr_copy_q = tiled_copy_q.get_thread_slice(thread_idx);
+ Tensor sQ_copy = cute::tiled_divide(sQ, Shape<_1, Int<kGmemElemsPerLoad>>{});
+ Tensor tQcQ = gmem_thr_copy_q.partition_S(cute::make_identity_tensor(Shape<Int<kBlockMN>, Int<kHeadDim / 2>>{}));
+
+ CUTE_STATIC_ASSERT_V(rank(tQcQ) == _3{});
+ CUTE_STATIC_ASSERT_V(rank(tRrCosCont) == _3{});
+ CUTE_STATIC_ASSERT_V(rank(tRrSinCont) == _3{});
+ CUTE_STATIC_ASSERT_V(size<1>(tQcQ) == size<1>(tRrCosCont));
+ CUTE_STATIC_ASSERT_V(size<2>(tQcQ) == size<2>(tRrCosCont));
+ CUTE_STATIC_ASSERT_V(size<1>(tQcQ) == size<1>(tRrSinCont));
+ CUTE_STATIC_ASSERT_V(size<2>(tQcQ) == size<2>(tRrSinCont));
+ CUTE_STATIC_ASSERT_V(size<0>(tRrCosCont) == size<0>(tRrSinCont));
+ CUTE_STATIC_ASSERT_V(size<0>(tQcQ) == size<0>(tRrCosCont));
+ static_assert(decltype(size<0>(tRrCosCont))::value % 2 == 0); // Since we do fast conversion from fp16/bf16 to fp32
+
+ #pragma unroll
+ for (int m = 0; m < size<1>(tQcQ); ++m) {
+ int const row = get<0>(tQcQ(_0{}, m, _0{}));
+ if (row < std::min(max_seqlen * qhead_per_khead - m_block * kBlockMN, kBlockMN)) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(tQcQ); ++k) {
+ int const col = get<1>(tQcQ(_0{}, _0{}, k));
+ if (col < rotary_dim / 2) {
+ int const col_idx_left = col / kGmemElemsPerLoad;
+ int const col_idx_right = col / kGmemElemsPerLoad + rotary_dim / (2 * kGmemElemsPerLoad);
+ Tensor rQ_left = make_fragment_like(sQ_copy(_, row, col_idx_left));
+ cute::copy(tiled_copy_q, sQ_copy(_, row, col_idx_left), rQ_left);
+ Tensor rQ_right = make_fragment_like(rQ_left);
+ cute::copy(tiled_copy_q, sQ_copy(_, row, col_idx_right), rQ_right);
+ apply_rotary_contiguous(rQ_left, rQ_right, tRrCosCont(_, m, k), tRrSinCont(_, m, k));
+ cute::copy(tiled_copy_q, rQ_left, sQ_copy(_, row, col_idx_left));
+ cute::copy(tiled_copy_q, rQ_right, sQ_copy(_, row, col_idx_right));
+ }
+ }
+ }
+ }
+ };
+
+ template <bool PagedKV=false, typename TensorsK, typename TensorgK, typename TensorpK, typename TensortRrR, typename TensorKPtr>
+ CUTLASS_DEVICE
+ void
+ apply_K_interleaved(TensorsK const &sK, // (kBlockN, kHeadDim)
+ TensorgK &gK, // (kBlockN, kHeadDim)
+ TensorpK const &tKpK, // (kBlockN, kHeadDim) split according to ThrCopyKV
+ TensortRrR const &tRrCos, // (kBlockN, kHeadDim/2) split according to GmemThrCopyRotary
+ TensortRrR const &tRrSin, // (kBlockN, kHeadDim/2) split according to GmemThrCopyRotary
+ TensorKPtr const &tPrKPtr,
+ int const n_block)
+ {
+ TiledCopyQK tiled_copy_k;
+ auto gmem_thr_copy_q = tiled_copy_k.get_thread_slice(thread_idx);
+ Tensor tKsK = gmem_thr_copy_q.partition_S(sK);
+ Tensor tKgK = gmem_thr_copy_q.partition_S(gK);
+ Tensor tKcK = gmem_thr_copy_q.partition_S(cute::make_identity_tensor(Shape<Int<kBlockMN>, Int<kHeadDim>>{}));
+
+ CUTE_STATIC_ASSERT_V(rank(tKsK) == _3{});
+ CUTE_STATIC_ASSERT_V(rank(tRrCos) == _3{});
+ CUTE_STATIC_ASSERT_V(rank(tRrSin) == _3{});
+ CUTE_STATIC_ASSERT_V(size<1>(tKsK) == size<1>(tRrCos));
+ CUTE_STATIC_ASSERT_V(size<2>(tKsK) == size<2>(tRrCos));
+ CUTE_STATIC_ASSERT_V(size<1>(tKsK) == size<1>(tRrSin));
+ CUTE_STATIC_ASSERT_V(size<2>(tKsK) == size<2>(tRrSin));
+ CUTE_STATIC_ASSERT_V(size<0>(tRrCos) == size<0>(tRrSin));
+ static_assert(decltype(size<0>(tKsK))::value == decltype(size<0>(tRrCos))::value * 2);
+ static_assert(decltype(size<0>(tRrCos))::value % 2 == 0); // Since we do fast conversion from fp16/bf16 to fp32
+ if constexpr (PagedKV) {
+ static_assert(decltype(size(tPrKPtr))::value == cute::ceil_div(size<1>(tKcK), kGmemThreadsPerRow));
+ }
+
+ #pragma unroll
+ for (int m = 0; m < size<1>(tKsK); ++m) {
+ int const row = get<0>(tKcK(_0{}, m, _0{}));
+ auto mK_cur_copy = [&] {
+ if constexpr (PagedKV) {
+ Element* k_ptr = reinterpret_cast<Element*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrKPtr(m / kGmemThreadsPerRow)), (m % kGmemThreadsPerRow), kGmemThreadsPerRow));
+ Tensor mK_cur = make_tensor(make_gmem_ptr(k_ptr), Shape<Int<kHeadDim>>{});
+ return cute::tiled_divide(mK_cur, Shape<Int<kGmemElemsPerLoad>>{});
+ } else {
+ return nullptr;
+ }
+ }();
+ if (row < std::min(max_seqlen - n_block * kBlockMN, kBlockMN)) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(tKsK); ++k) {
+ if (tKpK(k)) {
+ Tensor rK = make_fragment_like(tKsK(_, m, k));
+ cute::copy(tiled_copy_k, tKsK(_, m, k), rK);
+ if (tRpR(k)) { apply_rotary_interleaved(rK, tRrCos(_, m, k), tRrSin(_, m, k)); }
+ if constexpr (!PagedKV) {
+ cute::copy(tiled_copy_k, rK, tKgK(_, m, k));
+ } else {
+ int const ki = get<1>(tKcK(_0{}, _0{}, k)) / kGmemElemsPerLoad;
+ cute::copy(tiled_copy_k, rK, mK_cur_copy(_, ki));
+ }
+ }
+ }
+ }
+ }
+ };
+
+ template <bool PagedKV=false, typename TensorsK, typename TensorgK, typename TensorpK, typename TensortRrR, typename TensorKPtr>
+ CUTLASS_DEVICE
+ void
+ apply_K_contiguous(TensorsK const &sK, // (kBlockN, kHeadDim)
+ TensorgK &gK, // (kBlockN, kHeadDim)
+ TensorpK const &tKpK, // (kBlockN, kHeadDim) split according to ThrCopyKV
+ TensortRrR const &tRrCosCont, // (kBlockN, kHeadDim/2) split according to GmemThrCopyRotaryCont
+ TensortRrR const &tRrSinCont, // (kBlockN, kHeadDim/2) split according to GmemThrCopyRotaryCont
+ TensorKPtr const &tPrKPtr,
+ int const n_block, int const max_k)
+ {
+ TiledCopyQK tiled_copy_k;
+ auto gmem_thr_copy_q = tiled_copy_k.get_thread_slice(thread_idx);
+ Tensor sK_copy = cute::tiled_divide(sK, Shape<_1, Int<kGmemElemsPerLoad>>{});
+ Tensor gK_copy = cute::tiled_divide(gK, Shape<_1, Int<kGmemElemsPerLoad>>{});
+ Tensor tKcK = gmem_thr_copy_q.partition_S(cute::make_identity_tensor(Shape<Int<kBlockMN>, Int<kHeadDim / 2>>{}));
+
+ CUTE_STATIC_ASSERT_V(rank(tKcK) == _3{});
+ CUTE_STATIC_ASSERT_V(rank(tRrCosCont) == _3{});
+ CUTE_STATIC_ASSERT_V(rank(tRrSinCont) == _3{});
+ CUTE_STATIC_ASSERT_V(size<1>(tKcK) == size<1>(tRrCosCont));
+ CUTE_STATIC_ASSERT_V(size<2>(tKcK) == size<2>(tRrCosCont));
+ CUTE_STATIC_ASSERT_V(size<1>(tKcK) == size<1>(tRrSinCont));
+ CUTE_STATIC_ASSERT_V(size<2>(tKcK) == size<2>(tRrSinCont));
+ CUTE_STATIC_ASSERT_V(size<0>(tRrCosCont) == size<0>(tRrSinCont));
+ CUTE_STATIC_ASSERT_V(size<0>(tKcK) == size<0>(tRrCosCont));
+ static_assert(decltype(size<0>(tRrCosCont))::value % 2 == 0); // Since we do fast conversion from fp16/bf16 to fp32
+ if constexpr (PagedKV) {
+ static_assert(decltype(size(tPrKPtr))::value == cute::ceil_div(size<1>(tKcK), kGmemThreadsPerRow));
+ }
+
+ const int ro_dim_vec = rotary_dim / kGmemElemsPerLoad;
+ const int non_ro_dim_vec = (max_k - rotary_dim) / kGmemElemsPerLoad;
+ #pragma unroll
+ for (int m = 0; m < size<1>(tKcK); ++m) {
+ int const row = get<0>(tKcK(_0{}, m, _0{}));
+ Tensor gK_cur_copy = [&] {
+ if constexpr (PagedKV) {
+ Element* k_ptr = reinterpret_cast<Element*>(__shfl_sync(0xffffffff, reinterpret_cast<uint64_t>(tPrKPtr(m / kGmemThreadsPerRow)), (m % kGmemThreadsPerRow), kGmemThreadsPerRow));
+ Tensor mK_cur = make_tensor(make_gmem_ptr(k_ptr), Shape<Int<kHeadDim>>{});
+ return cute::tiled_divide(mK_cur, Shape<Int<kGmemElemsPerLoad>>{});
+ } else {
+ return gK_copy(_, row, _);
+ }
+ }();
+ if (row < std::min(max_seqlen - n_block * kBlockMN, kBlockMN)) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(tKcK); ++k) {
+ if (tKpK(k)) {
+ int const col = get<1>(tKcK(_0{}, _0{}, k));
+ bool rotate = col < rotary_dim / 2;
+ int const col_idx_left = rotate ? col / kGmemElemsPerLoad : (col + rotary_dim / 2) / kGmemElemsPerLoad;
+ int const col_idx_right = col_idx_left + (rotate ? ro_dim_vec / 2 : non_ro_dim_vec / 2);
+ Tensor rK_left = make_fragment_like(sK_copy(_, row, col_idx_left));
+ cute::copy(tiled_copy_k, sK_copy(_, row, col_idx_left), rK_left);
+ Tensor rK_right = make_fragment_like(rK_left);
+ cute::copy(tiled_copy_k, sK_copy(_, row, col_idx_right), rK_right);
+ if (rotate) {
+ apply_rotary_contiguous(rK_left, rK_right, tRrCosCont(_, m, k), tRrSinCont(_, m, k));
+ }
+ cute::copy(tiled_copy_k, rK_left, gK_cur_copy(_, col_idx_left));
+ if (col_idx_right * kGmemElemsPerLoad < max_k) {
+ cute::copy(tiled_copy_k, rK_right, gK_cur_copy(_, col_idx_right));
+ }
+ }
+ }
+ }
+ }
+ };
+
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/seqlen.h b/src/solarwm/backends/minimax_h3/hopper_src/seqlen.h
new file mode 100644
index 0000000..21a7471
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/seqlen.h
@@ -0,0 +1,93 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+namespace flash {
+
+// We consolidate all the info related to sequence length here. This is so that we can do all
+// the gmem reads once at the beginning of each tile, rather than having to repeat these reads
+// to compute various things like n_block_min, n_block_max, etc.
+
+template <bool Varlen, int kBlock>
+struct SeqlenInfo {
+
+ int const offset, offset_padded;
+ int const seqlen;
+
+ CUTLASS_DEVICE
+ SeqlenInfo(int const bidb, int const seqlen_static, int const* const cu_seqlens, int const* const seqused)
+ : offset(!Varlen || cu_seqlens == nullptr ? 0 : cu_seqlens[bidb])
+ , offset_padded(!Varlen || cu_seqlens == nullptr ? 0 : (cu_seqlens[bidb] + bidb * kBlock) / kBlock * kBlock)
+ , seqlen(!Varlen
+ ? seqlen_static
+ : (seqused ? seqused[bidb] : (cu_seqlens ? cu_seqlens[bidb + 1] - cu_seqlens[bidb] : seqlen_static)))
+ {
+ }
+
+};
+
+template <bool Varlen, int kBlockM>
+struct SeqlenInfoQK {
+
+ int const offset_q, offset_k, offset_q_padded;
+ int const seqlen_q, seqlen_k;
+
+ CUTLASS_DEVICE
+ SeqlenInfoQK(int const bidb, int const seqlen_q_static, int const seqlen_k_static,
+ int const* const cu_seqlens_q, int const* const cu_seqlens_k,
+ int const* const seqused_q, int const* const seqused_k
+ )
+ : offset_q(!Varlen || cu_seqlens_q == nullptr ? 0 : cu_seqlens_q[bidb])
+ , offset_k(!Varlen || cu_seqlens_k == nullptr ? 0 : cu_seqlens_k[bidb])
+ // If varlen, the layout for dPSum, LSE_log2, and dQaccum is that we pad each sequence in the batch
+ // by an extra kBlockM, so that the write for each sequence doesn't touch the next sequence.
+ // Sequence i starts at cu_seqlens[i] + i * kBlockM and ends at cu_seqlens[i + 1] + i * kBlockM
+ // However, the start must align to multiples of kBlockM.
+ , offset_q_padded(!Varlen || cu_seqlens_q == nullptr ? 0 : (cu_seqlens_q[bidb] + bidb * kBlockM) / kBlockM * kBlockM)
+ , seqlen_q(!Varlen
+ ? seqlen_q_static
+ : (seqused_q ? seqused_q[bidb] : (cu_seqlens_q ? cu_seqlens_q[bidb + 1] - cu_seqlens_q[bidb] : seqlen_q_static)))
+ , seqlen_k(!Varlen
+ ? seqlen_k_static
+ : (seqused_k ? seqused_k[bidb] : (cu_seqlens_k ? cu_seqlens_k[bidb + 1] - cu_seqlens_k[bidb] : seqlen_k_static)))
+ {
+ }
+
+};
+
+template <bool Varlen, bool AppendKV>
+struct SeqlenInfoQKNewK {
+
+ static_assert(!(AppendKV && !Varlen), "AppendKV is only supported with Varlen");
+
+ int const leftpad_k;
+ int const offset_q, offset_k, offset_k_new;
+ int const seqlen_q, seqlen_k_og, seqlen_k_new, seqlen_k;
+
+ CUTLASS_DEVICE
+ SeqlenInfoQKNewK(int const bidb, int const seqlen_q_static, int const seqlen_k_static, int const shape_K_new_0,
+ int const* const cu_seqlens_q, int const* const cu_seqlens_k, int const* const cu_seqlens_k_new,
+ int const* const seqused_q, int const* const seqused_k, int const* const ptr_leftpad_k
+ )
+ : leftpad_k(ptr_leftpad_k ? ptr_leftpad_k[bidb] : 0)
+ , offset_q(!Varlen || cu_seqlens_q == nullptr ? 0 : cu_seqlens_q[bidb])
+ , offset_k(!Varlen ? 0 : (cu_seqlens_k ? cu_seqlens_k[bidb] : 0) + leftpad_k)
+ , offset_k_new(!AppendKV || cu_seqlens_k_new == nullptr ? 0 : cu_seqlens_k_new[bidb])
+ , seqlen_q(!Varlen
+ ? seqlen_q_static
+ : (seqused_q ? seqused_q[bidb] : (cu_seqlens_q ? cu_seqlens_q[bidb + 1] - cu_seqlens_q[bidb] : seqlen_q_static)))
+ , seqlen_k_og(!Varlen
+ ? seqlen_k_static
+ : (seqused_k ? seqused_k[bidb] : (cu_seqlens_k ? cu_seqlens_k[bidb + 1] - cu_seqlens_k[bidb] : seqlen_k_static)) - leftpad_k)
+ , seqlen_k_new(!AppendKV
+ ? 0
+ : (cu_seqlens_k_new ? cu_seqlens_k_new[bidb + 1] - cu_seqlens_k_new[bidb] : shape_K_new_0))
+ , seqlen_k(!AppendKV ? seqlen_k_og : seqlen_k_og + seqlen_k_new)
+ {
+ }
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/sm90_pipeline_no_cluster.hpp b/src/solarwm/backends/minimax_h3/hopper_src/sm90_pipeline_no_cluster.hpp
new file mode 100644
index 0000000..65a3d15
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/sm90_pipeline_no_cluster.hpp
@@ -0,0 +1,99 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include<cutlass/pipeline/sm90_pipeline.hpp>
+
+namespace cutlass {
+
+using namespace cute;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// As of Cutlass v3.6.0, if size(ClusterShape) == 1, PipelineTmaAsync has all threads
+// signaling the barrier during consumer_release. This causes a perf regression in FA3
+// forward pass (especially hdim 128 causal). We instead reimplement the version of
+// PipelineTmaAsync before v3.6.0 where only 1 out of 128 threads signals the barrier.
+//
+// Assumption: params.num_consumers % NumThreadsPerWarpGroup == 0
+template <int Stages_, class Base=cutlass::PipelineTmaAsync<Stages_>>
+class PipelineTmaAsyncNoCluster: public Base {
+public:
+ using FullBarrier = typename Base::FullBarrier;
+ using EmptyBarrier = typename Base::EmptyBarrier;
+ static constexpr uint32_t Stages = Stages_;
+ using PipelineState = typename Base::PipelineState;
+
+ using SharedStorage = typename Base::SharedStorage;
+ using ThreadCategory = typename Base::ThreadCategory;
+ using Params = typename Base::Params;
+
+ static
+ CUTLASS_DEVICE
+ void
+ init_barriers(SharedStorage& storage, Params params) {
+ int warp_idx = canonical_warp_idx_sync();
+ bool is_initializing_warp = (warp_idx == 0);
+ if (is_initializing_warp) {
+ // Barrier FULL and EMPTY init
+ constexpr int producer_arv_cnt = 1;
+ uint32_t const num_consumer_warpgroups_per_cluster = params.num_consumers / NumThreadsPerWarpGroup;
+ uint32_t const multicast_consumer_arrival_count = num_consumer_warpgroups_per_cluster;
+
+ cutlass::arch::detail::initialize_barrier_array_pair_aligned<decltype(storage.full_barrier_), decltype(storage.empty_barrier_), Stages>(
+ storage.full_barrier_, storage.empty_barrier_, producer_arv_cnt, multicast_consumer_arrival_count);
+ }
+ cutlass::arch::fence_barrier_init();
+ }
+
+ template<class ClusterShape, class InitBarriers, class InitMasks>
+ CUTLASS_DEVICE
+ PipelineTmaAsyncNoCluster(SharedStorage& storage, Params params, ClusterShape cluster_shape, InitBarriers = {}, InitMasks = {})
+ : Base(storage, params, make_shape(_1{}, _1{}, _1{}) /*cluster_shape*/, cute::false_type{} /*init_barriers*/, cute::false_type{} /*init_masks*/)
+ , empty_barrier_ptr_(&storage.empty_barrier_[0]) {
+
+ int warp_idx = canonical_warp_idx_sync();
+ int lane_predicate = cute::elect_one_sync();
+
+ static_assert(cute::is_same_v<InitBarriers, cute::true_type> || cute::is_same_v<InitBarriers, cute::false_type>);
+ static_assert(cute::is_same_v<InitMasks, cute::true_type> || cute::is_same_v<InitMasks, cute::false_type>);
+ if constexpr (cute::is_same_v<InitBarriers, cute::true_type>) {
+ init_barriers(storage, params);
+ }
+
+ }
+
+ // Constructor
+ template<class ClusterShape>
+ CUTLASS_DEVICE
+ PipelineTmaAsyncNoCluster(SharedStorage& storage, Params params, ClusterShape cluster_shape)
+ : PipelineTmaAsyncNoCluster(storage, params, cluster_shape, cute::true_type{}, cute::true_type{}) { }
+
+ template<class ClusterShape, class InitBarriers>
+ CUTLASS_DEVICE
+ PipelineTmaAsyncNoCluster(SharedStorage& storage, Params params, ClusterShape cluster_shape, InitBarriers = {})
+ : PipelineTmaAsyncNoCluster(storage, params, cluster_shape, InitBarriers{}, cute::true_type{}) { }
+
+ CUTLASS_DEVICE
+ void consumer_release(PipelineState state) {
+ consumer_release(state.index());
+ }
+
+private:
+ EmptyBarrier* const empty_barrier_ptr_ = nullptr;
+
+ // Consumer signalling Producer of completion
+ // Ensures all blocks in the Same Row and Column get notifed.
+ CUTLASS_DEVICE
+ void consumer_release(uint32_t stage, uint32_t skip = false) {
+ empty_barrier_ptr_[stage].arrive(0 /*dst_blockid_*/, uint32_t(threadIdx.x % cutlass::NumThreadsPerWarpGroup == 0) & (!skip) /*is_signaling_thread*/);
+ }
+
+};
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // end namespace cutlass
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/softmax.h b/src/solarwm/backends/minimax_h3/hopper_src/softmax.h
new file mode 100644
index 0000000..8fcdb6b
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/softmax.h
@@ -0,0 +1,170 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cmath>
+
+#include <cute/tensor.hpp>
+
+#include <cutlass/numeric_types.h>
+
+#include "utils.h"
+
+namespace flash {
+
+using namespace cute;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template<bool zero_init=true, typename Engine0, typename Layout0, typename Engine1, typename Layout1, typename Operator>
+__device__ __forceinline__ void thread_reduce_(Tensor<Engine0, Layout0> const &tensor, Tensor<Engine1, Layout1> &summary, Operator &op) {
+ static_assert(Layout0::rank == 2, "Only support 2D Tensor");
+ static_assert(Layout1::rank == 1, "Only support 1D Tensor");
+ CUTE_STATIC_ASSERT_V(size<0>(summary) == size<0>(tensor));
+ #pragma unroll
+ for (int ni = 0; ni < size<1>(tensor); ni++) {
+ #pragma unroll
+ for (int mi = 0; mi < size<0>(tensor); mi++) {
+ summary(mi) = zero_init && ni == 0 ? tensor(mi, ni) : op(summary(mi), tensor(mi, ni));
+ }
+ }
+}
+
+template<typename Engine0, typename Layout0, typename Engine1, typename Layout1, typename Operator>
+__device__ __forceinline__ void quad_allreduce_(Tensor<Engine0, Layout0> &dst, Tensor<Engine1, Layout1> &src, Operator &op) {
+ CUTE_STATIC_ASSERT_V(size(dst) == size(src));
+ #pragma unroll
+ for (int i = 0; i < size(dst); i++) {
+ dst(i) = Allreduce<4>::run(src(i), op);
+ }
+}
+
+template<bool zero_init=true, typename Engine0, typename Layout0, typename Engine1, typename Layout1, typename Operator>
+__device__ __forceinline__ void reduce_(Tensor<Engine0, Layout0> const& tensor, Tensor<Engine1, Layout1> &summary, Operator &op) {
+ thread_reduce_<zero_init>(tensor, summary, op);
+ quad_allreduce_(summary, summary, op);
+}
+
+template<bool zero_init=true, typename Engine0, typename Layout0, typename Engine1, typename Layout1>
+__device__ __forceinline__ void reduce_max(Tensor<Engine0, Layout0> const& tensor, Tensor<Engine1, Layout1> &max){
+ MaxOp<float> max_op;
+ reduce_<zero_init>(tensor, max, max_op);
+}
+
+template<bool zero_init=true, bool warp_reduce=true, typename Engine0, typename Layout0, typename Engine1, typename Layout1>
+__device__ __forceinline__ void reduce_sum(Tensor<Engine0, Layout0> const& tensor, Tensor<Engine1, Layout1> &sum){
+ SumOp<float> sum_op;
+ thread_reduce_<zero_init>(tensor, sum, sum_op);
+ if constexpr (warp_reduce) { quad_allreduce_(sum, sum, sum_op); }
+}
+
+// Apply the exp to all the elements.
+template <bool Scale_max=true, bool Check_inf=true, int Max_offset=0,
+ typename Engine0, typename Layout0, typename Engine1, typename Layout1>
+__forceinline__ __device__ void scale_apply_exp2(Tensor<Engine0, Layout0> &tensor, Tensor<Engine1, Layout1> const &max, const float scale) {
+ // For FP8, we can subtract max by 8.0 so that the value after exp2 is in the range of [0, 256].
+ // This lets us use more of the FP8 range (instead of just [0, 1]) to reduce underflow.
+ static constexpr float max_offset = float(Max_offset); // We can only template on int, not float
+ static_assert(Layout0::rank == 2, "Only support 2D Tensor");
+ static_assert(Layout1::rank == 1, "Only support 1D Tensor");
+ CUTE_STATIC_ASSERT_V(size<0>(max) == size<0>(tensor));
+ #pragma unroll
+ for (int mi = 0; mi < size<0>(tensor); ++mi) {
+ // If max is -inf, then all elements must have been -inf (possibly due to masking).
+ // We don't want (-inf - (-inf)) since that would give NaN.
+ const float max_scaled = Check_inf
+ ? (max(mi) == -INFINITY ? 0.f : (!Scale_max ? max(mi) : max(mi) * scale) - max_offset)
+ : (!Scale_max ? max(mi) : max(mi) * scale) - max_offset;
+ #pragma unroll
+ for (int ni = 0; ni < size<1>(tensor); ++ni) {
+ // Instead of computing exp(x - max), we compute exp2(x * log_2(e) -
+ // max * log_2(e)). This allows the compiler to use the ffma
+ // instruction instead of fadd and fmul separately.
+ tensor(mi, ni) = exp2f(tensor(mi, ni) * scale - max_scaled);
+ }
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <int kNRows, int Max_offset=0>
+struct Softmax {
+
+ using TensorT = decltype(make_tensor<float>(Shape<Int<kNRows>>{}));
+ TensorT row_max, row_sum;
+ float const softmax_scale_log2;
+
+ CUTLASS_DEVICE Softmax(float const softmax_scale_log2_) : softmax_scale_log2(softmax_scale_log2_) {};
+
+ template<bool Is_first, bool Check_inf=false, typename Tensor0>
+ __forceinline__ __device__ TensorT max_get_scale(Tensor0 &acc_s) {
+ // Reshape acc_s from ((2, 2, V), MMA_M, MMA_N) to (nrow=(2, MMA_M), ncol=(2, V, MMA_N))
+ Tensor scores = make_tensor(acc_s.data(), flash::convert_layout_acc_rowcol(acc_s.layout()));
+ static_assert(CUTE_STATIC_V(size<0>(scores)) == kNRows);
+ TensorT scores_scale;
+ if constexpr (Is_first) {
+ flash::template reduce_max</*zero_init=*/true>(scores, row_max);
+ cute::fill(scores_scale, 1.f);
+ } else {
+ Tensor scores_max_prev = make_fragment_like(row_max);
+ cute::copy(row_max, scores_max_prev);
+ flash::template reduce_max</*zero_init=*/false>(scores, row_max);
+ #pragma unroll
+ for (int mi = 0; mi < size(row_max); ++mi) {
+ float scores_max_cur = !Check_inf
+ ? row_max(mi)
+ : (row_max(mi) == -INFINITY ? 0.0f : row_max(mi));
+ scores_scale(mi) = exp2f((scores_max_prev(mi) - scores_max_cur) * softmax_scale_log2);
+ row_sum(mi) *= scores_scale(mi);
+ }
+ }
+ return scores_scale;
+ };
+
+ template<bool Is_first, bool Check_inf=false, typename Tensor0>
+ __forceinline__ __device__ void online_softmax(Tensor0 &acc_s) {
+ // Reshape acc_s from ((2, 2, V), MMA_M, MMA_N) to (nrow=(2, MMA_M), ncol=(2, V, MMA_N))
+ Tensor scores = make_tensor(acc_s.data(), flash::convert_layout_acc_rowcol(acc_s.layout()));
+ static_assert(CUTE_STATIC_V(size<0>(scores)) == kNRows);
+ flash::template scale_apply_exp2</*Scale_max=*/true, Check_inf, Max_offset>(scores, row_max, softmax_scale_log2);
+ // We don't do the reduce across threads here since we don't need to use the row_sum.
+ // We do that reduce at the end when we need to normalize the softmax.
+ flash::reduce_sum</*zero_init=*/Is_first, /*warp_reduce=*/false>(scores, row_sum);
+ };
+
+ __forceinline__ __device__ TensorT finalize(float const final_scale=1.f) {
+ SumOp<float> sum_op;
+ quad_allreduce_(row_sum, row_sum, sum_op);
+ TensorT scores_scale;
+ #pragma unroll
+ for (int mi = 0; mi < size(row_sum); ++mi) {
+ float sum = row_sum(mi);
+ float inv_sum = (sum == 0.f || sum != sum) ? 0.f : 1.f / sum;
+ scores_scale(mi) = inv_sum * final_scale;
+ // For FP8, we might have scaled the output of exp by 2**8 so we need to divide sum by that amount.
+ if constexpr (Max_offset != 0) {
+ static constexpr float sum_scale = 1.f / float(1 << Max_offset);
+ sum *= sum_scale;
+ }
+ row_sum(mi) = (sum == 0.f || sum != sum) ? -INFINITY : row_max(mi) * (softmax_scale_log2 * float(M_LN2)) + __logf(sum);
+ }
+ return scores_scale;
+ };
+
+ template<typename Tensor1>
+ __forceinline__ __device__ void rescale_o(Tensor1 &acc_o, TensorT const &scores_scale) {
+ // Reshape acc_o from (MMA=4, MMA_M, MMA_K) to (nrow=(2, MMA_M), ncol=(2, MMA_K))
+ Tensor acc_o_rowcol = make_tensor(acc_o.data(), flash::convert_layout_acc_rowcol(acc_o.layout()));
+ static_assert(CUTE_STATIC_V(size<0>(acc_o_rowcol)) == kNRows);
+ #pragma unroll
+ for (int mi = 0; mi < size<0>(acc_o_rowcol); ++mi) {
+ #pragma unroll
+ for (int ni = 0; ni < size<1>(acc_o_rowcol); ++ni) { acc_o_rowcol(mi, ni) *= scores_scale(mi); }
+ }
+ };
+
+};
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/static_switch.h b/src/solarwm/backends/minimax_h3/hopper_src/static_switch.h
new file mode 100644
index 0000000..5e13b5f
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/static_switch.h
@@ -0,0 +1,181 @@
+// Inspired by
+// https://github.com/NVIDIA/DALI/blob/main/include/dali/core/static_switch.h
+// and https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Dispatch.h
+
+#pragma once
+
+/// @param COND - a boolean expression to switch by
+/// @param CONST_NAME - a name given for the constexpr bool variable.
+/// @param ... - code to execute for true and false
+///
+/// Usage:
+/// ```
+/// BOOL_SWITCH(flag, BoolConst, [&] {
+/// some_function<BoolConst>(...);
+/// });
+/// ```
+//
+
+#define BOOL_SWITCH(COND, CONST_NAME, ...) \
+ [&] { \
+ if (COND) { \
+ constexpr static bool CONST_NAME = true; \
+ return __VA_ARGS__(); \
+ } else { \
+ constexpr static bool CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ } \
+ }()
+
+#ifdef FLASHATTENTION_DISABLE_LOCAL
+ #define CAUSAL_LOCAL_SWITCH(CAUSAL_COND, LOCAL_COND, CAUSAL_CONST_NAME, LOCAL_CONST_NAME, ...) \
+ [&] { \
+ constexpr static bool LOCAL_CONST_NAME = false; \
+ if (CAUSAL_COND) { \
+ constexpr static bool CAUSAL_CONST_NAME = true; \
+ return __VA_ARGS__(); \
+ } else { \
+ constexpr static bool CAUSAL_CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ } \
+ }()
+#else
+ #define CAUSAL_LOCAL_SWITCH(CAUSAL_COND, LOCAL_COND, CAUSAL_CONST_NAME, LOCAL_CONST_NAME, ...) \
+ [&] { \
+ if (CAUSAL_COND) { \
+ constexpr static bool CAUSAL_CONST_NAME = true; \
+ constexpr static bool LOCAL_CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ } else if (LOCAL_COND) { \
+ constexpr static bool CAUSAL_CONST_NAME = false; \
+ constexpr static bool LOCAL_CONST_NAME = true; \
+ return __VA_ARGS__(); \
+ } else { \
+ constexpr static bool CAUSAL_CONST_NAME = false; \
+ constexpr static bool LOCAL_CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ } \
+ }()
+#endif
+
+#ifdef FLASHATTENTION_DISABLE_SOFTCAP
+ #define SOFTCAP_SWITCH(COND, CONST_NAME, ...) \
+ [&] { \
+ constexpr static bool CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ }()
+#else
+ #define SOFTCAP_SWITCH BOOL_SWITCH
+#endif
+
+#ifdef FLASHATTENTION_DISABLE_PAGEDKV
+ #define PAGEDKV_SWITCH(COND, CONST_NAME, ...) \
+ [&] { \
+ constexpr static bool CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ }()
+#else
+ #define PAGEDKV_SWITCH BOOL_SWITCH
+#endif
+
+#ifdef FLASHATTENTION_DISABLE_SPLIT
+ #define SPLIT_SWITCH(COND, CONST_NAME, ...) \
+ [&] { \
+ constexpr static bool CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ }()
+#else
+ #define SPLIT_SWITCH BOOL_SWITCH
+#endif
+
+#ifdef FLASHATTENTION_DISABLE_APPENDKV
+ #define APPENDKV_SWITCH(COND, CONST_NAME, ...) \
+ [&] { \
+ constexpr static bool CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ }()
+#else
+ #define APPENDKV_SWITCH BOOL_SWITCH
+#endif
+
+#ifdef FLASHATTENTION_DISABLE_PACKGQA
+ #define PACKGQA_SWITCH(COND, CONST_NAME, ...) \
+ [&] { \
+ constexpr static bool CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ }()
+#else
+ #define PACKGQA_SWITCH BOOL_SWITCH
+#endif
+
+#ifdef FLASHATTENTION_DISABLE_VARLEN
+ #define VARLEN_SWITCH(COND, CONST_NAME, ...) \
+ [&] { \
+ constexpr static bool CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ }()
+#else
+ #define VARLEN_SWITCH BOOL_SWITCH
+#endif
+
+#ifdef FLASHATTENTION_DISABLE_CLUSTER
+ #define CLUSTER_SWITCH(COND, CONST_NAME, ...) \
+ [&] { \
+ constexpr static bool CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ }()
+#else
+ #define CLUSTER_SWITCH BOOL_SWITCH
+#endif
+
+#ifdef FLASHATTENTION_DISABLE_SM8x
+ #define ARCH_SWITCH(ARCH, ARCH_NAME, ...) \
+ [&] { \
+ constexpr static int ARCH_NAME = 90; \
+ return __VA_ARGS__(); \
+ }()
+#else
+ #define ARCH_SWITCH(ARCH, ARCH_NAME, ...) \
+ [&] { \
+ if (ARCH == 86 || ARCH == 89) { \
+ constexpr static int ARCH_NAME = 86; \
+ return __VA_ARGS__(); \
+ } else if (ARCH < 90) { \
+ constexpr static int ARCH_NAME = 80; \
+ return __VA_ARGS__(); \
+ } else { \
+ constexpr static int ARCH_NAME = 90; \
+ return __VA_ARGS__(); \
+ } \
+ }()
+#endif
+
+#ifndef FLASHATTENTION_ENABLE_VCOLMAJOR
+ #define VCOLMAJOR_SWITCH(COND, CONST_NAME, ...) \
+ [&] { \
+ constexpr static bool CONST_NAME = false; \
+ return __VA_ARGS__(); \
+ }()
+#else
+ #define VCOLMAJOR_SWITCH BOOL_SWITCH
+#endif
+
+#define HEADDIM_SWITCH(HEADDIM, ...) \
+ [&] { \
+ if (HEADDIM == 64) { \
+ constexpr static int kHeadSize = 64; \
+ return __VA_ARGS__(); \
+ } else if (HEADDIM == 96) { \
+ constexpr static int kHeadSize = 96; \
+ return __VA_ARGS__(); \
+ } else if (HEADDIM == 128) { \
+ constexpr static int kHeadSize = 128; \
+ return __VA_ARGS__(); \
+ } else if (HEADDIM == 96) { \
+ constexpr static int kHeadSize = 96; \
+ return __VA_ARGS__(); \
+ } else if (HEADDIM == 256) { \
+ constexpr static int kHeadSize = 256; \
+ return __VA_ARGS__(); \
+ } \
+ }()
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/tile_scheduler.hpp b/src/solarwm/backends/minimax_h3/hopper_src/tile_scheduler.hpp
new file mode 100644
index 0000000..0b74d0e
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/tile_scheduler.hpp
@@ -0,0 +1,537 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include "cutlass/fast_math.h"
+#include "cutlass/arch/barrier.h"
+
+#include "named_barrier.hpp"
+
+namespace flash {
+
+///////////////////////////////////////////////////////////////////////////////
+
+// Host side kernel arguments
+struct TileSchedulerArguments {
+ // num_head is num_head_q if not PackGQA, else num_head_k
+ int const num_blocks, num_head, num_batch, num_splits;
+ int const qhead_per_khead;
+ int const seqlen; // Only used if Varlen and cu_seqlens == nullptr and seqused == nullptr
+ int const seqlen_k, headdim, element_size; // Used to calculate L2 swizzling
+ int* const tile_count_semaphore = nullptr;
+ int* const cu_seqlens = nullptr;
+ int* const seqused = nullptr;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+template<bool Varlen=false, bool Split=false, bool PackGQA=false, int kBlock=128>
+class SingleTileScheduler {
+
+public:
+
+ using SharedStorage = int;
+
+ // Device side kernel params
+ struct Params {
+ int const num_blocks, num_head, num_batch, num_splits;
+ int const qhead_per_khead;
+ int const seqlen;
+ cutlass::FastDivmod nsplits_divmod;
+ int* const cu_seqlens;
+ int* const seqused;
+ };
+
+ static Params
+ to_underlying_arguments(TileSchedulerArguments const& args) {
+ return {args.num_blocks, args.num_head, args.num_batch, !Split ? 1 : args.num_splits,
+ args.qhead_per_khead, args.seqlen,
+ cutlass::FastDivmod(!Split ? 1 : args.num_splits),
+ !Varlen ? nullptr : args.cu_seqlens, !Varlen ? nullptr : args.seqused};
+ }
+
+ static dim3
+ get_grid_shape(Params const& params, int num_sm) {
+ return {uint32_t(params.num_blocks), uint32_t((!Split ? 1 : params.num_splits) * params.num_head), uint32_t(params.num_batch)};
+ }
+
+ struct WorkTileInfo {
+ int block_idx = 0;
+ int bidh = 0;
+ int bidb = 0;
+ bool is_valid_tile = false;
+
+ CUTLASS_DEVICE
+ bool
+ is_valid(Params const& params) const {
+ return is_valid_tile;
+ }
+
+ CUTLASS_DEVICE
+ cute::tuple<int32_t, int32_t, int32_t, int32_t>
+ get_block_coord(Params const& params) const {
+ if constexpr (!Split) {
+ return {block_idx, bidh, bidb, 0 /*split_idx*/};
+ } else {
+ int split_idx;
+ int bidh_actual = params.nsplits_divmod.divmod(split_idx, bidh);
+ return {block_idx, bidh_actual, bidb, split_idx};
+ }
+ }
+
+ };
+
+ CUTLASS_DEVICE
+ SingleTileScheduler(SharedStorage* const smem_scheduler) { }
+
+ template<bool IsProducerWarp=false>
+ CUTLASS_DEVICE
+ WorkTileInfo
+ get_initial_work(Params const& params) const {
+ WorkTileInfo work_info {int(blockIdx.x), int(blockIdx.y), int(blockIdx.z), true};
+ if constexpr (Varlen) {
+ int seqlen = params.seqused
+ ? params.seqused[work_info.bidb]
+ : (params.cu_seqlens ? params.cu_seqlens[work_info.bidb + 1] - params.cu_seqlens[work_info.bidb] : params.seqlen);
+ if constexpr (PackGQA) { seqlen *= params.qhead_per_khead; }
+ work_info.is_valid_tile = work_info.block_idx * kBlock < seqlen;
+ }
+ return work_info;
+ }
+
+ CUTLASS_DEVICE
+ void
+ init_consumer() const {}
+
+ CUTLASS_DEVICE
+ void
+ prefetch_next_work(Params const& params, WorkTileInfo& current_work) const {}
+
+ template<bool IsProducerWarp=false>
+ CUTLASS_DEVICE
+ WorkTileInfo
+ get_next_work(Params const& params, WorkTileInfo const& current_work) const {
+ return {-1, -1, -1, false};
+ }
+
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+template<bool Split=false>
+class StaticPersistentTileScheduler {
+
+public:
+
+ using SharedStorage = int;
+
+ // Device side kernel params
+ struct Params {
+ int total_blocks;
+ cutlass::FastDivmod m_block_divmod, head_divmod;
+ cutlass::FastDivmod nsplits_divmod;
+ };
+
+ static Params
+ to_underlying_arguments(TileSchedulerArguments const& args) {
+ return {args.num_blocks * args.num_head * args.num_batch * (!Split ? 1 : args.num_splits),
+ cutlass::FastDivmod(args.num_blocks), cutlass::FastDivmod(args.num_head * (!Split ? 1 : args.num_splits)),
+ cutlass::FastDivmod(!Split ? 1 : args.num_splits)};
+ }
+
+ static dim3
+ get_grid_shape(Params const& params, int num_sm) {
+ return {uint32_t(num_sm)};
+ }
+
+ struct WorkTileInfo {
+ int tile_idx;
+
+ CUTLASS_DEVICE
+ bool
+ is_valid(Params const& params) const {
+ return tile_idx < params.total_blocks;
+ }
+
+ CUTLASS_DEVICE
+ cute::tuple<int32_t, int32_t, int32_t, int32_t>
+ get_block_coord(Params const& params) const {
+ int block, bidh, bidb;
+ bidb = params.head_divmod.divmod(bidh, params.m_block_divmod.divmod(block, tile_idx));
+ int split_idx = 0;
+ if constexpr (Split) {
+ bidh = params.nsplits_divmod.divmod(split_idx, bidh);
+ }
+ return {block, bidh, bidb, split_idx};
+ }
+
+ };
+
+ CUTLASS_DEVICE
+ StaticPersistentTileScheduler(SharedStorage* const smem_scheduler) {};
+
+ template<bool IsProducerWarp=false>
+ CUTLASS_DEVICE
+ WorkTileInfo
+ get_initial_work(Params const& params) const {
+ return {int(blockIdx.x)};
+ }
+
+ CUTLASS_DEVICE
+ void
+ init_consumer() const {}
+
+ CUTLASS_DEVICE
+ void
+ prefetch_next_work(Params const& params, WorkTileInfo& current_work) const {}
+
+ template<bool IsProducerWarp=false>
+ CUTLASS_DEVICE
+ WorkTileInfo
+ get_next_work(Params const& params, WorkTileInfo const& current_work) const {
+ return {current_work.tile_idx + int(gridDim.x)};
+ }
+
+};
+
+template<int NumMmaThreads=2 * cutlass::NumThreadsPerWarpGroup, int NumProducerThreads=cutlass::NumThreadsPerWarp,
+ bool Split=false, bool PackGQA=false, bool WarpSpecialized=true>
+class DynamicPersistentTileScheduler {
+
+ // This scheduler targets the causal (or local) case where each tile takes different
+ // amount of time. We use longest-processing-time-first scheduling:
+ // the longest remaining tile is assigned to the first SM that's free.
+ // SM indicates they are free by incrementing a semaphore.
+ // However, we have to make sure K & V still fit into L2 cache, so we perform scheduling
+ // on "sections" of the head & batch dimension, each section consisting of e.g. 8 heads.
+ // This is the L2 swizzling part. The size of each section is precomputed based on the
+ // size of K & V and the L2 cache size.
+
+ static_assert(WarpSpecialized || NumProducerThreads == NumMmaThreads);
+ static constexpr int NumThreads = WarpSpecialized ? NumMmaThreads + NumProducerThreads : NumMmaThreads;
+
+public:
+ using SharedStorage = int;
+
+protected:
+ SharedStorage* const tile_count_smem;
+
+public:
+
+ // Device side kernel params
+ struct Params {
+ int const total_blocks;
+ cutlass::FastDivmod const m_block_divmod, head_divmod;
+ cutlass::FastDivmod const l2_minor_divmod, l2_major_divmod;
+ cutlass::FastDivmod const l2_minor_residual_divmod;
+ int const num_hb_quotient;
+ int* const tile_count_semaphore;
+ };
+
+ static Params
+ to_underlying_arguments(TileSchedulerArguments const& args) {
+ int const size_one_kv_head = args.seqlen_k * args.headdim * args.element_size * 2;
+ int const size_l2 = 32 * 1024 * 1024; // 32 MB for K & V
+ // Swizzle is the size of each "section". Round swizzle to a power of 2
+ // If not PackGQA already, the size of each section can increase by qhead_per_khead
+ int const swizzle = (1 << cutlass::find_log2(size_l2 / size_one_kv_head)) * (PackGQA ? 1 : args.qhead_per_khead);
+ // If we're in the last section (called residual), we don't want to divide by
+ // swizzle. Instead we want to divide by the remainder.
+ int const num_hb_remainder = (args.num_head * args.num_batch) % swizzle;
+ int const num_split_blocks = args.num_blocks * (!Split ? 1 : args.num_splits);
+ // printf("num_split_blocks = %d, num_head = %d, num_batch = %d, swizzle = %d, PackGQA = %d, qhead_per_khead = %d, num_hb_remainder = %d\n", num_split_blocks, args.num_head, args.num_batch, swizzle, int(PackGQA), args.qhead_per_khead, num_hb_remainder);
+ assert(args.tile_count_semaphore != nullptr);
+ return {num_split_blocks * args.num_head * args.num_batch,
+ cutlass::FastDivmod(args.num_blocks), cutlass::FastDivmod(args.num_head),
+ cutlass::FastDivmod(swizzle), cutlass::FastDivmod(swizzle * num_split_blocks),
+ // don't divide by 0
+ cutlass::FastDivmod(num_hb_remainder > 0 ? num_hb_remainder : 1),
+ (args.num_head * args.num_batch) / swizzle,
+ args.tile_count_semaphore};
+ }
+
+ static dim3
+ get_grid_shape(Params const& params, int num_sm) {
+ return {uint32_t(num_sm)};
+ }
+
+ struct WorkTileInfo {
+ int tile_idx;
+
+ CUTLASS_DEVICE
+ bool
+ is_valid(Params const& params) const {
+ return tile_idx < params.total_blocks;
+ }
+
+ CUTLASS_DEVICE
+ cute::tuple<int32_t, int32_t, int32_t, int32_t>
+ get_block_coord(Params const& params) const {
+ int block, bidh, bidb;
+ int l2_mod, bidhb, bidhb_residual;
+ bidhb = params.l2_major_divmod.divmod(l2_mod, tile_idx);
+ // If we're in the last section (called residual), we don't want to divide by
+ // swizzle. Instead we want to divide by the remainder.
+ if (bidhb < params.num_hb_quotient) {
+ block = params.l2_minor_divmod.divmod(bidhb_residual, l2_mod);
+ } else {
+ block = params.l2_minor_residual_divmod.divmod(bidhb_residual, l2_mod);
+ }
+ bidb = params.head_divmod.divmod(bidh, bidhb * params.l2_minor_divmod.divisor + bidhb_residual);
+ int split_idx = 0;
+ if constexpr (Split) {
+ split_idx = params.m_block_divmod.divmod(block, block);
+ }
+ // Longest-processing-time-first
+ block = params.m_block_divmod.divisor - 1 - block;
+ return {block, bidh, bidb, split_idx};
+ }
+
+ };
+
+ CUTLASS_DEVICE
+ DynamicPersistentTileScheduler(SharedStorage* const smem_scheduler) : tile_count_smem(smem_scheduler) {};
+
+ template<bool IsProducerWarp=false>
+ CUTLASS_DEVICE
+ WorkTileInfo
+ get_initial_work(Params const& params) const {
+ return {int(blockIdx.x)};
+ }
+
+ CUTLASS_DEVICE
+ void
+ init_consumer() const {
+ if (WarpSpecialized || cutlass::canonical_warp_idx_sync() > 0) {
+ flash::named_barrier_arrive(NumThreads, static_cast<uint32_t>(FwdNamedBarriers::TileCountSmemEmpty) /*id*/);
+ }
+ }
+
+ CUTLASS_DEVICE
+ void
+ prefetch_next_work(Params const& params, WorkTileInfo& current_work) const {
+ if (threadIdx.x % NumProducerThreads == 0) {
+ current_work.tile_idx = atomicAdd(params.tile_count_semaphore, 1) + int(gridDim.x);
+ }
+ }
+
+ template<bool IsProducerWarp=false>
+ CUTLASS_DEVICE
+ WorkTileInfo
+ get_next_work(Params const& params, WorkTileInfo const& current_work) const {
+ if constexpr (IsProducerWarp) {
+ // thread 0 already has the right tile_idx, just need to broadcast to the rest of warp 0
+ int new_tile_idx = __shfl_sync(0xffffffff, current_work.tile_idx, 0 /*lane*/);
+ flash::named_barrier_sync(NumThreads, static_cast<uint32_t>(FwdNamedBarriers::TileCountSmemEmpty) /*id*/);
+ if (threadIdx.x % NumProducerThreads == 0) {
+ *tile_count_smem = current_work.tile_idx;
+ }
+ flash::named_barrier_arrive(NumThreads, static_cast<uint32_t>(FwdNamedBarriers::TileCountSmemFull) /*id*/);
+ return {new_tile_idx};
+ } else {
+ flash::named_barrier_sync(NumThreads, static_cast<uint32_t>(FwdNamedBarriers::TileCountSmemFull) /*id*/);
+ int tile_idx = *tile_count_smem;
+ flash::named_barrier_arrive(NumThreads, static_cast<uint32_t>(FwdNamedBarriers::TileCountSmemEmpty) /*id*/);
+ return {tile_idx};
+ }
+ }
+
+};
+
+
+template<int kBlock, int NumMmaThreads=2 * cutlass::NumThreadsPerWarpGroup, int NumProducerThreads=cutlass::NumThreadsPerWarp, bool Split=false, bool PackGQA=false, bool WarpSpecialized=true>
+class VarlenDynamicPersistentTileScheduler {
+
+ static_assert(WarpSpecialized || NumProducerThreads == NumMmaThreads);
+ static constexpr int NumThreads = WarpSpecialized ? NumMmaThreads + NumProducerThreads : NumMmaThreads;
+
+public:
+ using SharedStorage = int4;
+
+protected:
+ SharedStorage* const work_info_smem;
+
+public:
+
+ // Device side kernel params
+ struct Params {
+ int num_head, num_batch;
+ int const qhead_per_khead;
+ int const seqlen;
+ cutlass::FastDivmod nsplits_divmod;
+ int* const tile_count_semaphore;
+ int* const cu_seqlens;
+ int* const seqused;
+ };
+
+ static Params
+ to_underlying_arguments(TileSchedulerArguments const& args) {
+ // If Split, for the purpose of scheduling, we pretend that instead there are
+ // (args.num_splits * args.num_head) number of heads.
+ assert(args.tile_count_semaphore != nullptr);
+ return {args.num_head * (!Split ? 1 : args.num_splits), args.num_batch,
+ args.qhead_per_khead, args.seqlen,
+ cutlass::FastDivmod(!Split ? 1 : args.num_splits),
+ args.tile_count_semaphore, args.cu_seqlens, args.seqused};
+ }
+
+ static dim3
+ get_grid_shape(Params const& params, int num_sm) {
+ return {uint32_t(num_sm)};
+ }
+
+ struct WorkTileInfo {
+ int tile_idx, block, bidh, bidb;
+
+ CUTLASS_DEVICE
+ bool
+ is_valid(Params const& params) const {
+ // if (blockIdx.x >= 0 && (threadIdx.x == 128 || threadIdx.x == 0)) { printf("blockIdx.x = %d, threadIdx.x = %d, checking valid, bidb = %d, params.num_batch = %d\n", blockIdx.x, threadIdx.x, bidb, params.num_batch); }
+ return bidb < params.num_batch;
+ }
+
+ CUTLASS_DEVICE
+ cute::tuple<int32_t, int32_t, int32_t, int32_t>
+ get_block_coord(Params const& params) const {
+ if constexpr (!Split) {
+ return {block, bidh, bidb, 0 /*split_idx*/};
+ } else {
+ int split_idx;
+ int bidh_actual = params.nsplits_divmod.divmod(split_idx, bidh);
+ return {block, bidh_actual, bidb, split_idx};
+ }
+ }
+ };
+
+ CUTLASS_DEVICE
+ VarlenDynamicPersistentTileScheduler(SharedStorage* const smem_scheduler) : work_info_smem(smem_scheduler) {};
+
+ CUTLASS_DEVICE
+ WorkTileInfo
+ tile_idx_to_work_tile(Params const& params, int next_tile_idx, WorkTileInfo const& current_work) const {
+ auto prefix_sum = [](int val) {
+ int lane = threadIdx.x % cutlass::NumThreadsPerWarp;
+ CUTLASS_PRAGMA_UNROLL
+ for (int i = 1; i < cutlass::NumThreadsPerWarp; i <<= 1) {
+ int32_t partial_sum = __shfl_up_sync(0xffffffff, val, i);
+ if (lane >= i) { val += partial_sum; }
+ }
+ return val;
+ };
+
+ auto get_num_m_blocks = [&](int bidb_start) {
+ int lane = threadIdx.x % cutlass::NumThreadsPerWarp;
+ int seqlen;
+ if (params.seqused) {
+ seqlen = lane + bidb_start < params.num_batch ? params.seqused[lane + bidb_start] : 0;
+ } else if (params.cu_seqlens) {
+ int cur_cu_seqlen = lane + bidb_start <= params.num_batch ? params.cu_seqlens[lane + bidb_start] : 0;
+ int next_cu_seqlen = __shfl_down_sync(0xffffffff, cur_cu_seqlen, 1);
+ seqlen = next_cu_seqlen - cur_cu_seqlen;
+ } else {
+ seqlen = params.seqlen;
+ }
+ if constexpr (PackGQA) { seqlen *= params.qhead_per_khead; }
+ return lane + bidb_start < params.num_batch && lane < cutlass::NumThreadsPerWarp - 1
+ ? cute::ceil_div(seqlen, kBlock) : 0;
+ };
+
+ int num_m_blocks = get_num_m_blocks(current_work.bidb); // Different for each lane
+ // Cumulative number of blocks for the next 31 batches
+ int num_m_blocks_cumulative = prefix_sum(num_m_blocks);
+ // Total number of blocks for the next 31 batches
+ int m_blocks_in_group = __shfl_sync(0xffffffff, num_m_blocks_cumulative, cutlass::NumThreadsPerWarp - 1);
+ int group_end_tile = current_work.tile_idx - current_work.block - current_work.bidh * __shfl_sync(0xffffffff, num_m_blocks, 0 /*lane*/) + m_blocks_in_group * params.num_head; // Same for all lanes
+ int bidb = current_work.bidb;
+ // if (blockIdx.x <= 9 && threadIdx.x == 0) {
+ // printf("Before while, blockIdx.x = %d, threadIdx.x = %d, bidb = %d, num_m_blocks = %d, next_tile_idx = %d, group_end_tile = %d, m_blocks_in_group = %d\n", blockIdx.x, threadIdx.x, bidb, num_m_blocks, next_tile_idx, group_end_tile, m_blocks_in_group);
+ // }
+ while (group_end_tile <= next_tile_idx) {
+ bidb += cutlass::NumThreadsPerWarp - 1;
+ if (bidb >= params.num_batch) {
+ // if (blockIdx.x <= 9 && threadIdx.x == 0) {
+ // printf("Returning early, blockIdx.x = %d, threadIdx.x = %d, bidb = %d, num_m_blocks = %d, next_tile_idx = %d, group_end_tile = %d, m_blocks_in_group = %d\n", blockIdx.x, threadIdx.x, bidb, num_m_blocks, next_tile_idx, group_end_tile, m_blocks_in_group);
+ // }
+ return {next_tile_idx, 0, 0, params.num_batch};
+ }
+ num_m_blocks = get_num_m_blocks(bidb);
+ num_m_blocks_cumulative = prefix_sum(num_m_blocks);
+ m_blocks_in_group = __shfl_sync(0xffffffff, num_m_blocks_cumulative, cutlass::NumThreadsPerWarp - 1);
+ group_end_tile += m_blocks_in_group * params.num_head;
+ // if (blockIdx.x <= 9 && threadIdx.x == 0) {
+ // printf("Bottom of while, blockIdx.x = %d, threadIdx.x = %d, bidb = %d, num_m_blocks = %d, next_tile_idx = %d, group_end_tile = %d, m_blocks_in_group = %d\n", blockIdx.x, threadIdx.x, bidb, num_m_blocks, next_tile_idx, group_end_tile, m_blocks_in_group);
+ // }
+ }
+ int group_start_tile = group_end_tile - m_blocks_in_group * params.num_head;
+ // The next problem to process is the first one that does not have ending tile position
+ // that is greater than or equal to tile index.
+ int batch_idx_in_group = __popc(__ballot_sync(0xffffffff, group_start_tile + num_m_blocks_cumulative * params.num_head <= next_tile_idx));
+ bidb += batch_idx_in_group;
+ num_m_blocks = __shfl_sync(0xffffffff, num_m_blocks, batch_idx_in_group);
+ int mh_block = next_tile_idx - group_start_tile - (batch_idx_in_group == 0 ? 0 : __shfl_sync(0xffffffff, num_m_blocks_cumulative, batch_idx_in_group - 1)) * params.num_head;
+ int bidh = mh_block / num_m_blocks;
+ int block = mh_block - bidh * num_m_blocks;
+ // if (blockIdx.x <= 9 && threadIdx.x == 0) {
+ // printf("blockIdx.x = %d, threadIdx.x = %d, batch_idx_in_group = %d, bidb = %d, num_m_blocks = %d, next_tile_idx = %d, group_end_tile = %d, m_blocks_in_group = %d, mh_block = %d, bidh = %d, block = %d\n", blockIdx.x, threadIdx.x, batch_idx_in_group, bidb, num_m_blocks, next_tile_idx, group_end_tile, m_blocks_in_group, mh_block, bidh, block);
+ // }
+ return {next_tile_idx, block, bidh, bidb};
+ }
+
+ template<bool IsProducerWarp=false>
+ CUTLASS_DEVICE
+ WorkTileInfo
+ get_initial_work(Params const& params) const {
+ if constexpr (IsProducerWarp) {
+ WorkTileInfo work_info = tile_idx_to_work_tile(params, int(blockIdx.x), {0, 0, 0, 0});
+ if (threadIdx.x % cutlass::NumThreadsPerWarp == 0) {
+ *work_info_smem = make_int4(work_info.tile_idx, work_info.block, work_info.bidh, work_info.bidb);
+ }
+ flash::named_barrier_arrive(NumThreads, static_cast<uint32_t>(FwdNamedBarriers::TileCountSmemFull) /*id*/);
+ return work_info;
+ } else {
+ return get_next_work<false>(params, {0, 0, 0, 0});
+ }
+ }
+
+ CUTLASS_DEVICE
+ void
+ init_consumer() const {
+ // Don't arrive at the TileCountSmemEmpty barrier here, because get_initial_work will do that
+ }
+
+ CUTLASS_DEVICE
+ void
+ prefetch_next_work(Params const& params, WorkTileInfo& current_work) const {
+ if (threadIdx.x % NumProducerThreads == 0) {
+ current_work.tile_idx = atomicAdd(params.tile_count_semaphore, 1) + int(gridDim.x);
+ }
+ }
+
+ template<bool IsProducerWarp=false>
+ CUTLASS_DEVICE
+ WorkTileInfo
+ get_next_work(Params const& params, WorkTileInfo const& current_work) const {
+ if constexpr (IsProducerWarp) {
+ // thread 0 has the next tile_idx, just need to broadcast to the rest of warp 0
+ int new_tile_idx = __shfl_sync(0xffffffff, current_work.tile_idx, 0 /*lane*/);
+ WorkTileInfo work_info = {__shfl_sync(0xffffffff, current_work.tile_idx, 1 /*lane*/), current_work.block, current_work.bidh, current_work.bidb};
+ work_info = tile_idx_to_work_tile(params, new_tile_idx, work_info);
+ flash::named_barrier_sync(NumThreads, static_cast<uint32_t>(FwdNamedBarriers::TileCountSmemEmpty) /*id*/);
+ if (threadIdx.x % cutlass::NumThreadsPerWarp == 0) {
+ *work_info_smem = make_int4(work_info.tile_idx, work_info.block, work_info.bidh, work_info.bidb);
+ }
+ flash::named_barrier_arrive(NumThreads, static_cast<uint32_t>(FwdNamedBarriers::TileCountSmemFull) /*id*/);
+ return work_info;
+ } else {
+ flash::named_barrier_sync(NumThreads, static_cast<uint32_t>(FwdNamedBarriers::TileCountSmemFull) /*id*/);
+ int4 work_info = *work_info_smem;
+ flash::named_barrier_arrive(NumThreads, static_cast<uint32_t>(FwdNamedBarriers::TileCountSmemEmpty) /*id*/);
+ return WorkTileInfo{work_info.x, work_info.y, work_info.z, work_info.w};
+ }
+ }
+
+};
+
+} // flash
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/tile_size.h b/src/solarwm/backends/minimax_h3/hopper_src/tile_size.h
new file mode 100644
index 0000000..424ec06
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/tile_size.h
@@ -0,0 +1,72 @@
+/******************************************************************************
+ * Copyright (c) 2024, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <tuple>
+
+// Return {kBlockM, kBlockN, Mma1_is_RS, IntraWGOverlap}
+constexpr std::tuple<int, int, bool, bool> tile_size_fwd_sm90(
+ int headdim, bool is_causal, bool is_local, int element_size=2,
+ bool v_colmajor=false, bool paged_kv=false, bool softcap=false) {
+ if (element_size == 2) {
+ if (headdim <= 64) {
+ return {192, 128, true, true};
+ // Good for long seqlen (>= 4k) but suffers from tile quantization at short seqlen
+ // return {192, is_causal || is_local ? 192 : 176, true, false};
+ } else if (headdim <= 96) {
+ return {192, is_local || paged_kv ? 128 : 144, false, true};
+ } else if (headdim <= 128) {
+#ifdef SOLAR_HOPPER_FWD192
+ if (!is_causal && !is_local && !paged_kv && !softcap) {
+ return {192, 128, false, true};
+ }
+#endif
+ return {128, is_causal || is_local || paged_kv ? 128 : 176, true, true};
+ // {128, 192, false, false} and {192, 128, false, true} are quite good too
+ // 128 x 192 hits the limit of smem if Mma1_is_RS, 128 x 144 hits the limit if !Mma1_is_RS
+ } else if (headdim <= 192) {
+ return {128, paged_kv || is_local ? 96 : 112, true, true}; // 128 x 112 hits the limit of smem
+ } else {
+ return {128, is_local ? 64 : 80, true, true}; // 128 x 80 hits the limit of smem
+ }
+ } else {
+ if (headdim <= 64) {
+ return {192, 160, true, true};
+ } else if (headdim <= 96) {
+ return {192, 128, true, true};
+ } else if (headdim <= 128) {
+ return {128, paged_kv ? 160 : (v_colmajor || (softcap && is_local) ? 192 : 224), true, true};
+ } else if (headdim <= 192) {
+ return {128, (paged_kv || softcap) && is_local ? 128 : 160, true, true};
+ } else {
+ return {128, is_local ? 64 : 128, true, !paged_kv}; // PagedKV uses more registers so we disabled IntraWGOverlap
+ }
+ }
+}
+
+// Return {kBlockM, kBlockN, kNWarps, kStages, Q_in_regs}
+constexpr std::tuple<int, int, int, int, bool> tile_size_fwd_sm8x(
+ bool sm86_or_89, int headdim, bool is_causal, bool is_local, int element_size=2,
+ bool paged_kv=false, bool varlen_and_split=false,
+ bool softcap=false, bool append_kv=false) {
+ if (element_size == 2) {
+ if (headdim <= 64) {
+ return {128, varlen_and_split ? 80 : (is_local ? 96 : 112), 4, 1, false};
+ } else if (headdim <= 96) {
+ return {128, varlen_and_split || is_local ? 48 : 64, 4, 1, false};
+ } else if (headdim <= 128) {
+ bool const use_8_warps = sm86_or_89 | varlen_and_split;
+ return {128, use_8_warps ? (varlen_and_split ? (is_local ? 96 : 112) : (is_local ? 96 : 128)) : (is_local ? 48 : 64), use_8_warps ? 8 : 4, 1, use_8_warps};
+ } else if (headdim <= 192) {
+ bool const kBlockN_64 = append_kv || is_local || varlen_and_split || paged_kv;
+ return {128, kBlockN_64 ? 64 : 96, 8, sm86_or_89 ? 1 : 2, !kBlockN_64};
+ } else {
+ return {128, sm86_or_89 ? (append_kv ? 32 : (varlen_and_split || is_local ? 48 : 64)) : (append_kv ? 48 : (varlen_and_split || is_local ? 64 : 96)), 8, 1, sm86_or_89 && !append_kv};
+ }
+ } else {
+ // Placeholder for now
+ return {128, 64, 8, 2, false};
+ }
+}
diff --git a/src/solarwm/backends/minimax_h3/hopper_src/utils.h b/src/solarwm/backends/minimax_h3/hopper_src/utils.h
new file mode 100644
index 0000000..fa8938c
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/hopper_src/utils.h
@@ -0,0 +1,572 @@
+/******************************************************************************
+ * Copyright (c) 2024, Tri Dao.
+ ******************************************************************************/
+
+#pragma once
+
+#include <assert.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <cuda_fp16.h>
+
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+#include <cuda_bf16.h>
+#endif
+
+#include <cute/tensor.hpp>
+
+#include <cutlass/cutlass.h>
+#include <cutlass/array.h>
+#include <cutlass/numeric_conversion.h>
+#include <cutlass/numeric_types.h>
+
+
+#define CHECK_CUDA(call) \
+ do { \
+ cudaError_t status_ = call; \
+ if (status_ != cudaSuccess) { \
+ fprintf(stderr, "CUDA error (%s:%d): %s\n", __FILE__, __LINE__, cudaGetErrorString(status_)); \
+ exit(1); \
+ } \
+ } while(0)
+
+#define CHECK_CUDA_KERNEL_LAUNCH() CHECK_CUDA(cudaGetLastError())
+
+namespace flash {
+
+using namespace cute;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// A wrapper for the kernel that is used to guard against compilation on
+// architectures that will never use the kernel. The purpose of this is to
+// reduce the size of the compiled binary.
+// Adapted from https://github.com/vllm-project/vllm/blob/4d29e91be84d27ca313d657eee92c067439a4c23/csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cuh#L55
+template <typename Kernel>
+struct enable_sm90_or_later : Kernel {
+ template <typename... Args>
+ CUTLASS_DEVICE void operator()(Args&&... args) {
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
+ Kernel::operator()(std::forward<Args>(args)...);
+#endif
+ }
+};
+
+template <typename Kernel>
+struct enable_sm80_to_sm89 : Kernel {
+ template <typename... Args>
+ CUTLASS_DEVICE void operator()(Args&&... args) {
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) && (__CUDA_ARCH__ <= 890)
+ Kernel::operator()(std::forward<Args>(args)...);
+#endif
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template<typename T>
+struct MaxOp {
+__device__ __forceinline__ T operator()(T const & x, T const & y) { return x > y ? x : y; }
+};
+
+template <>
+struct MaxOp<float> {
+// This is slightly faster
+__device__ __forceinline__ float operator()(float const &x, float const &y) { return max(x, y); }
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template<typename T>
+struct SumOp {
+__device__ __forceinline__ T operator()(T const & x, T const & y) { return x + y; }
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template<int THREADS>
+struct Allreduce {
+ static_assert(THREADS == 32 || THREADS == 16 || THREADS == 8 || THREADS == 4);
+ template<typename T, typename Operator>
+ static __device__ __forceinline__ T run(T x, Operator &op) {
+ constexpr int OFFSET = THREADS / 2;
+ x = op(x, __shfl_xor_sync(uint32_t(-1), x, OFFSET));
+ return Allreduce<OFFSET>::run(x, op);
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template<>
+struct Allreduce<2> {
+template<typename T, typename Operator>
+static __device__ __forceinline__ T run(T x, Operator &op) {
+ x = op(x, __shfl_xor_sync(uint32_t(-1), x, 1));
+ return x;
+}
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// For SM80, convert acc_layout from (MMA=4, MMA_M, MMA_N) to (nrow=(2, MMA_M), ncol=(2, MMA_N))
+// For SM90, convert acc_layout from ((2, 2, V), MMA_M, MMA_N) to (nrow=(2, MMA_M), ncol=(2, V, MMA_N))
+template<bool Transposed=false, typename Layout0>
+CUTLASS_DEVICE auto convert_layout_acc_rowcol(Layout0 acc_layout) {
+ if constexpr (decltype(rank<0>(acc_layout))::value == 3) { // SM90
+ static_assert(decltype(size<0, 0>(acc_layout))::value == 2);
+ static_assert(decltype(size<0, 1>(acc_layout))::value == 2);
+ static_assert(decltype(rank(acc_layout))::value == 3);
+ auto l = acc_layout;
+ if constexpr (!Transposed) {
+ return make_layout(make_layout(get<0, 1>(l), get<1>(l)), make_layout(get<0, 0>(l), get<0, 2>(l), get<2>(l)));
+ } else {
+ return make_layout(make_layout(get<0, 0>(l), get<0, 2>(l), get<2>(l)), make_layout(get<0, 1>(l), get<1>(l)));
+ }
+
+ } else { // SM80
+ static_assert(decltype(size<0>(acc_layout))::value == 4);
+ static_assert(decltype(rank(acc_layout))::value == 3);
+ auto l = logical_divide(acc_layout, Shape<_2>{}); // ((2, 2), MMA_M, MMA_N)
+ if constexpr (!Transposed) {
+ return make_layout(make_layout(get<0, 1>(l), get<1>(l)), make_layout(get<0, 0>(l), get<2>(l)));
+ } else {
+ return make_layout(make_layout(get<0, 0>(l), get<2>(l)), make_layout(get<0, 1>(l), get<1>(l)));
+ }
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// For SM80, convert acc_layout from (MMA=4, MMA_M, MMA_N) to ((4, 2), MMA_M, MMA_N / 2)
+// if using m16n8k16, or to (4, MMA_M, MMA_N) if using m16n8k8.
+// For SM90, FP16/BF16, convert acc_layout from ((2, 2, N / 8), MMA_M, MMA_N) to ((2, 2, 2), MMA_M, (N / 16, MMA_N))
+// For SM90, FP8, convert acc_layout from ((2, 2, N / 8), MMA_M, MMA_N) to ((4, 2, 2), MMA_M, (N / 32, MMA_N))
+template<typename MMA_Traits, typename Layout0>
+CUTLASS_DEVICE auto convert_layout_acc_Aregs(Layout0 acc_layout) {
+ using X = Underscore;
+ if constexpr (decltype(rank<0>(acc_layout))::value == 3) { // SM90
+ static_assert(decltype(size<0, 0>(acc_layout))::value == 2);
+ static_assert(decltype(size<0, 1>(acc_layout))::value == 2);
+ static_assert(decltype(rank(acc_layout))::value == 3);
+ static_assert(decltype(rank(get<0>(acc_layout)))::value == 3);
+ if constexpr (sizeof(typename MMA_Traits::ValTypeA) == 2) {
+ auto l = logical_divide(get<0, 2>(acc_layout), Tile<_2>{}); // ((2, N / 16))
+ return make_layout(make_layout(get<0, 0>(acc_layout), get<0, 1>(acc_layout), get<0, 0>(l)), get<1>(acc_layout), coalesce(make_layout(get<0, 1>(l), get<2>(acc_layout))));
+ } else {
+ static_assert(sizeof(typename MMA_Traits::ValTypeA) == 1);
+ static_assert(decltype(stride<0, 0>(acc_layout))::value == 1);
+ static_assert(decltype(stride<0, 1>(acc_layout))::value == 2);
+ auto l = logical_divide(get<0, 2>(acc_layout), Tile<Layout<Shape<_2, _2>>>{}); // (((2, 2), N / 32))
+ // This combines the first two modes (<0, 0> and <0, 1>) into one mode.
+ // Will require register shuffling later to be correct.
+ return make_layout(make_layout(Layout<_4>{}, get<0, 0, 0>(l), get<0, 0, 1>(l)),
+ get<1>(acc_layout),
+ coalesce(make_layout(get<0, 1>(l), get<2>(acc_layout)))); // ((4, 2, 2), MMA_M, N / 32 * MMA_N)
+ // This combination is right but doesn't work with register shuffling.
+ // return make_layout(make_layout(coalesce(make_layout(get<0, 0>(acc_layout), get<0, 0, 0>(l))), get<0, 1>(acc_layout), get<0, 0, 1>(l)),
+ // get<1>(acc_layout),
+ // coalesce(make_layout(get<0, 1>(l), get<2>(acc_layout))));
+ }
+ } else { // SM80
+ static_assert(decltype(size<0>(acc_layout))::value == 4);
+ static_assert(decltype(rank(acc_layout))::value == 3);
+ constexpr int mma_shape_K = get<2>(typename MMA_Traits::Shape_MNK{});
+ static_assert(mma_shape_K == 8 || mma_shape_K == 16);
+ if constexpr (mma_shape_K == 8) {
+ return acc_layout;
+ } else {
+ auto l = logical_divide(acc_layout, Shape<X, X, _2>{}); // (4, MMA_M, (2, MMA_N / 2)))
+ return make_layout(make_layout(get<0>(l), get<2, 0>(l)), get<1>(l), get<2, 1>(l));
+ }
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename To_type, typename Engine, typename Layout>
+CUTLASS_DEVICE auto convert_type_unsafe(Tensor<Engine, Layout> const &tensor) {
+ using From_type = typename Engine::value_type;
+ static constexpr int numel = decltype(size(tensor))::value;
+ cutlass::NumericArrayConverter<To_type, From_type, numel> convert_op;
+ // HACK: this requires tensor to be "contiguous"
+ auto frag = convert_op(*reinterpret_cast<const cutlass::Array<From_type, numel> *>(tensor.data()));
+ return make_tensor(make_rmem_ptr<To_type>(&frag), tensor.layout());
+ // Unsafe because we're returning a tensor with memory allocated on the stack. If the compiler does not
+ // inline this function, then the memory might not be valid.
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Engine, typename Layout, typename EngineOut>
+CUTLASS_DEVICE void convert_type_out(Tensor<Engine, Layout> const &tensor, Tensor<EngineOut, Layout> &out) {
+ // Somehow if we allocate out inside this function and return it, e2e is slower and the output can be wrong.
+ using From_type = typename Engine::value_type;
+ using To_type = typename EngineOut::value_type;
+ static constexpr int FragmentSize = std::max(sizeof(From_type) / sizeof(To_type), sizeof(To_type) / sizeof(From_type));
+ static_assert(CUTE_STATIC_V(size(tensor)) % FragmentSize == 0, "Fragment size does not vectorize properly");
+ Tensor frag = recast<cutlass::Array<From_type, FragmentSize> const>(tensor);
+ Tensor out_frg = recast<cutlass::Array<To_type, FragmentSize>>(out);
+ static_assert(size(frag) == size(out_frg));
+ cutlass::NumericArrayConverter<To_type, From_type, FragmentSize> convert_op;
+ #pragma unroll
+ for (int i = 0; i < size(frag); ++i) { out_frg[i] = convert_op(frag[i]); }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// Blocks until all but N previous cp.async.commit_group operations have committed.
+// This differs from cute::cp_async_wait in that when N = 0 we don't call cp.async.wait_all
+// (which is equivalent to commit_group then wait_group 0).
+// Instead we just call cp.async.wait_group 0, which is slightly faster.
+// https://github.com/NVIDIA/cutlass/blob/master/include/cute/arch/copy_sm80.hpp#L113
+template <int N>
+CUTE_HOST_DEVICE
+void cp_async_wait() {
+#if defined(CUTE_ARCH_CP_ASYNC_SM80_ENABLED)
+ asm volatile("cp.async.wait_group %0;\n" :: "n"(N));
+#endif
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <bool A, class Mma, class Tensor0>
+CUTLASS_DEVICE
+auto mma_partition_fragment_AB(Mma const& mma, Tensor0 const& tensor0) {
+ if constexpr (A) {
+ return mma.partition_fragment_A(tensor0);
+ } else {
+ return mma.partition_fragment_B(tensor0);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <bool zero_init=false, int wg_wait=0, bool SwapAB=false, int M_slice=-1,
+ typename Tensor0, typename Tensor1, typename Tensor2, typename TiledMma>
+CUTLASS_DEVICE void gemm(TiledMma& tiled_mma, Tensor0 const& tCrA, Tensor1 const& tCrB, Tensor2& tCrC) {
+ if constexpr (M_slice >= 0) {
+ static constexpr int MMA_M = decltype(size<1>(tCrC))::value;
+ static_assert(M_slice < MMA_M);
+ // After logical_divide, C has shape ((2,2,V), (MMA_M, 1), MMA_N)
+ Tensor tCrC_slice = cute::logical_divide(tCrC, Shape<cute::Underscore, Int<MMA_M>>{})(_, make_coord(Int<M_slice>{}, _), _);
+ if constexpr (!SwapAB) {
+ Tensor tCrA_slice = cute::logical_divide(tCrA, Shape<cute::Underscore, Int<MMA_M>>{})(_, make_coord(Int<M_slice>{}, _), _);
+ gemm<zero_init, wg_wait, SwapAB, /*M_slice=*/-1>(tiled_mma, tCrA_slice, tCrB, tCrC_slice);
+ } else {
+ Tensor tCrB_slice = cute::logical_divide(tCrB, Shape<cute::Underscore, Int<MMA_M>>{})(_, make_coord(Int<M_slice>{}, _), _);
+ gemm<zero_init, wg_wait, SwapAB, /*M_slice=*/-1>(tiled_mma, tCrA, tCrB_slice, tCrC_slice);
+ }
+ } else {
+ constexpr bool Is_RS = !cute::is_base_of<cute::GMMA::DescriptorIterator, typename TiledMma::FrgTypeA>::value;
+ // Need to cast away const on tCrA since warpgroup_fence_operand doesn't take const
+ if constexpr (Is_RS) {
+ if constexpr (!SwapAB) {
+ warpgroup_fence_operand(const_cast<Tensor0 &>(tCrA));
+ } else {
+ warpgroup_fence_operand(const_cast<Tensor1 &>(tCrB));
+ }
+ }
+ warpgroup_fence_operand(tCrC);
+ warpgroup_arrive();
+ if constexpr (zero_init) {
+ tiled_mma.accumulate_ = GMMA::ScaleOut::Zero;
+ }
+ // Unroll the K mode manually to set scale D to 1
+ CUTLASS_PRAGMA_UNROLL
+ for (int k_block = 0; k_block < size<2>(tCrA); ++k_block) {
+ if constexpr (!SwapAB) {
+ cute::gemm(tiled_mma, tCrA(_,_,k_block), tCrB(_,_,k_block), tCrC);
+ } else {
+ cute::gemm(tiled_mma, tCrB(_,_,k_block), tCrA(_,_,k_block), tCrC);
+ }
+ tiled_mma.accumulate_ = GMMA::ScaleOut::One;
+ }
+ warpgroup_commit_batch();
+ if constexpr (wg_wait >= 0) { warpgroup_wait<wg_wait>(); }
+ warpgroup_fence_operand(tCrC);
+ if constexpr (Is_RS) {
+ if constexpr (!SwapAB) {
+ warpgroup_fence_operand(const_cast<Tensor0 &>(tCrA));
+ } else {
+ warpgroup_fence_operand(const_cast<Tensor1 &>(tCrB));
+ }
+ }
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template<bool A_in_regs=false, bool B_in_regs=false, bool SwapAB=false,
+ typename Tensor0, typename Tensor1,
+ typename Tensor2, typename Tensor3, typename Tensor4,
+ typename TiledMma, typename TiledCopyA, typename TiledCopyB,
+ typename ThrCopyA, typename ThrCopyB, typename Hook>
+CUTLASS_DEVICE void gemm_sm80(Tensor0 &acc, Tensor1 &tCrA, Tensor2 &tCrB, Tensor3 const& tCsA,
+ Tensor4 const& tCsB, TiledMma tiled_mma,
+ TiledCopyA smem_tiled_copy_A, TiledCopyB smem_tiled_copy_B,
+ ThrCopyA smem_thr_copy_A, ThrCopyB smem_thr_copy_B, Hook fn) {
+ if constexpr (SwapAB) {
+ gemm_sm80<B_in_regs, A_in_regs>(acc, tCrB, tCrA, tCsB, tCsA, tiled_mma, smem_tiled_copy_B, smem_tiled_copy_A, smem_thr_copy_B, smem_thr_copy_A, fn);
+ } else {
+ CUTE_STATIC_ASSERT_V(size<1>(tCrA) == size<1>(acc)); // MMA_M
+ CUTE_STATIC_ASSERT_V(size<1>(tCrB) == size<2>(acc)); // MMA_N
+ CUTE_STATIC_ASSERT_V(size<2>(tCrA) == size<2>(tCrB)); // MMA_K
+ Tensor tCrA_copy_view = smem_thr_copy_A.retile_D(tCrA);
+ CUTE_STATIC_ASSERT_V(size<1>(tCsA) == size<1>(tCrA_copy_view)); // M
+ Tensor tCrB_copy_view = smem_thr_copy_B.retile_D(tCrB);
+ CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<1>(tCrB_copy_view)); // N
+ if (!A_in_regs) { cute::copy(smem_tiled_copy_A, tCsA(_, _, _0{}), tCrA_copy_view(_, _, _0{})); }
+ if (!B_in_regs) { cute::copy(smem_tiled_copy_B, tCsB(_, _, _0{}), tCrB_copy_view(_, _, _0{})); }
+ #pragma unroll
+ for (int i = 0; i < size<2>(tCrA); ++i) {
+ if (i < size<2>(tCrA) - 1) {
+ if (!A_in_regs) { cute::copy(smem_tiled_copy_A, tCsA(_, _, i + 1), tCrA_copy_view(_, _, i + 1)); }
+ if (!B_in_regs) { cute::copy(smem_tiled_copy_B, tCsB(_, _, i + 1), tCrB_copy_view(_, _, i + 1)); }
+ }
+ if constexpr (!std::is_same_v<Hook, std::nullptr_t>) {
+ if (i == 0) { fn(); }
+ }
+ cute::gemm(tiled_mma, tCrA(_, _, i), tCrB(_, _, i), acc);
+ }
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template<typename Tensor0, typename Tensor1, typename Tensor2, typename Tensor3,
+ typename TiledMma, typename TiledCopy, typename ThrCopy>
+CUTLASS_DEVICE void gemm_rs_sm80(Tensor0 &acc, Tensor1 &tCrA, Tensor2 &tCrB, Tensor3 const& tCsB,
+ TiledMma tiled_mma, TiledCopy smem_tiled_copy_B,
+ ThrCopy smem_thr_copy_B) {
+ CUTE_STATIC_ASSERT_V(size<1>(tCrA) == size<1>(acc)); // MMA_M
+ CUTE_STATIC_ASSERT_V(size<1>(tCrB) == size<2>(acc)); // MMA_N
+ CUTE_STATIC_ASSERT_V(size<2>(tCrA) == size<2>(tCrB)); // MMA_K
+ Tensor tCrB_copy_view = smem_thr_copy_B.retile_D(tCrB);
+ CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<1>(tCrB_copy_view)); // N
+ cute::copy(smem_tiled_copy_B, tCsB(_, _, _0{}), tCrB_copy_view(_, _, _0{}));
+ #pragma unroll
+ for (int i = 0; i < size<2>(tCrA); ++i) {
+ if (i < size<2>(tCrA) - 1) {
+ cute::copy(smem_tiled_copy_B, tCsB(_, _, i + 1), tCrB_copy_view(_, _, i + 1));
+ }
+ cute::gemm(tiled_mma, tCrA(_, _, i), tCrB(_, _, i), acc);
+ }
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <bool Is_even_MN=true, bool Is_even_K=true, bool Clear_OOB_MN=false, bool Clear_OOB_K=true,
+ class CopyAtom, class TV, class Tiler, typename Engine0, typename Layout0, typename Engine1, typename Layout1,
+ typename Engine2, typename Layout2, typename Engine3, typename Layout3>
+CUTLASS_DEVICE void copy(TiledCopy<CopyAtom, TV, Tiler> const &tiled_copy, Tensor<Engine0, Layout0> const &S,
+ Tensor<Engine1, Layout1> &D, Tensor<Engine2, Layout2> const &identity_MN,
+ Tensor<Engine3, Layout3> const &predicate_K, const int max_MN=0) {
+ // Decay TiledCopy to CopyAtom
+ auto copy_atom = static_cast<CopyAtom const&>(tiled_copy);
+ CUTE_STATIC_ASSERT_V(rank(S) == Int<3>{});
+ CUTE_STATIC_ASSERT_V(rank(D) == Int<3>{});
+ CUTE_STATIC_ASSERT_V(size<0>(S) == size<0>(D)); // MMA
+ CUTE_STATIC_ASSERT_V(size<1>(S) == size<1>(D)); // MMA_M
+ CUTE_STATIC_ASSERT_V(size<2>(S) == size<2>(D)); // MMA_K
+ // There's no case where !Clear_OOB_K && Clear_OOB_MN
+ static_assert(!(Clear_OOB_MN && !Clear_OOB_K));
+ auto has_with_bool = cute::is_valid([](auto t)->void_t<decltype(declval<typename decltype(t)::Traits>().with(true))>{}, copy_atom);
+ #pragma unroll
+ for (int m = 0; m < size<1>(S); ++m) {
+ bool predicate_mn = Is_even_MN || get<0>(identity_MN(_0{}, m, _0{})) < max_MN;
+ if constexpr (Is_even_MN || !Clear_OOB_MN) {
+ if (Is_even_MN || predicate_mn) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(S); ++k) {
+ if constexpr (Is_even_K || !Clear_OOB_K) {
+ if (Is_even_K || predicate_K(k)) { cute::copy(copy_atom, S(_, m, k), D(_, m, k)); }
+ } else { // Clear_OOB_K == true && Is_even_K == false
+ // If copy traits can be transformed with a predicate value, do it, otherwise branch here
+ if constexpr (has_with_bool) {
+ cute::copy(copy_atom.with(predicate_K(k)), S(_, m, k), D(_, m, k));
+ } else {
+ if (predicate_K(k)) {
+ cute::copy(copy_atom, S(_, m, k), D(_, m, k));
+ } else {
+ cute::clear(D(_, m, k));
+ }
+ }
+ }
+ }
+ }
+ } else { // Clear_OOB_MN == true && Is_even_MN == false, also implies Clear_OOB_K == true
+ if constexpr (!has_with_bool) {
+ if (predicate_mn) {
+ #pragma unroll
+ for (int k = 0; k < size<2>(S); ++k) {
+ if (Is_even_K || predicate_K(k)) {
+ cute::copy(copy_atom, S(_, m, k), D(_, m, k));
+ } else if (Clear_OOB_K) {
+ cute::clear(D(_, m, k));
+ }
+ }
+ } else {
+ cute::clear(D(_, m, _));
+ }
+ } else { // combine the mn predicate with the k predicate
+ #pragma unroll
+ for (int k = 0; k < size<2>(S); ++k) {
+ cute::copy(copy_atom.with(predicate_mn && (Is_even_K || predicate_K(k))), S(_, m, k), D(_, m, k));
+ }
+ }
+ }
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+// Byte permute and shuffle to match register layout of
+// (FP8 downcasted) accumulator of GEMM-I to FP8 operand A of GEMM-II.
+template <typename Fragment>
+CUTLASS_DEVICE void permute_Aregs_fp8(Fragment &frag) {
+ // frag has shape ((4, 2, 2), MMA_M, MMA_N), each element is 8 bits
+ static_assert(decltype(size<0, 0>(frag))::value == 4);
+ static_assert(decltype(size<0, 1>(frag))::value == 2);
+ static_assert(decltype(stride<0, 0>(frag))::value == 1);
+ static_assert(decltype(stride<0, 1>(frag))::value == 4);
+ static_assert(sizeof(typename Fragment::value_type) == 1);
+
+ int quad_idx = threadIdx.x % 4;
+ bool lane_03 = quad_idx == 0 || quad_idx == 3;
+ int selector_upper = lane_03 ? 0x5410 : 0x1054;
+ int selector_lower = lane_03 ? 0x7632 : 0x3276;
+
+ static constexpr int upper_map[4] = {0, 3, 1, 2};
+ // static constexpr int lower_map[4] = {1, 2, 0, 3};
+
+ Tensor frag_64b = recast<uint2>(frag); // ((1, 1, 2), MMA_M, MMA_N)
+ #pragma unroll
+ for (int i = 0; i < size(frag_64b); ++i) {
+ uint32_t upper = frag_64b[i].x;
+ uint32_t lower = frag_64b[i].y;
+ uint32_t upper0 = lane_03 ? upper : lower;
+ uint32_t lower0 = lane_03 ? lower : upper;
+ upper0 = __shfl_sync(uint32_t(-1), upper0, upper_map[quad_idx], 4);
+ // lower0 = __shfl_sync(uint32_t(-1), lower0, lower_map[quad_idx], 4);
+ lower0 = __shfl_sync(uint32_t(-1), lower0, upper_map[quad_idx] ^ 1, 4);
+ frag_64b[i].x = __byte_perm(upper0, lower0, selector_upper);
+ frag_64b[i].y = __byte_perm(upper0, lower0, selector_lower);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Fragment>
+CUTLASS_DEVICE void permute_Cregs_fp8(Fragment &frag) {
+ // frag has shape ((2, 2, N / 8), MMA_M, MMA_N), each element is 32 bits
+ static_assert(decltype(size<0, 0>(frag))::value == 2);
+ static_assert(decltype(size<0, 1>(frag))::value == 2);
+ static_assert(decltype(size<0, 2>(frag))::value % 2 == 0);
+ static_assert(decltype(stride<0, 0>(frag))::value == 1);
+ static_assert(sizeof(typename Fragment::value_type) == 4);
+ Tensor frag_64b = group_modes<1, 3>(recast<uint2>(frag)); // ((1, 2, N / 8), (MMA_M, MMA_N))
+ #pragma unroll
+ for (int mi = 0; mi < size<1>(frag_64b); ++mi) {
+ #pragma unroll
+ for (int i = 0; i < size<0, 2>(frag_64b) / 2; ++i) {
+ cutlass::swap(frag_64b(make_coord(_0{}, _1{}, 2 * i), mi), frag_64b(make_coord(_0{}, _0{}, 2 * i + 1), mi));
+ }
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Fragment>
+CUTLASS_DEVICE void permute_output_fp8(Fragment &out) {
+ // out has shape ((2, 2, N / 8), MMA_M, MMA_N), each element is 32 bits
+ static_assert(decltype(size<0, 0>(out))::value == 2);
+ static_assert(decltype(size<0, 1>(out))::value == 2);
+ static_assert(decltype(size<0, 2>(out))::value % 2 == 0);
+ static_assert(decltype(stride<0, 0>(out))::value == 1);
+ static_assert(sizeof(typename Fragment::value_type) == 4);
+ Tensor frag = group_modes<1, 3>(out); // ((2, 2, N / 8), (MMA_M, MMA_N))
+ #pragma unroll
+ for (int mi = 0; mi < size<1>(frag); ++mi) {
+ #pragma unroll
+ for (int j = 0; j < size<0, 1>(frag); ++j) {
+ #pragma unroll
+ for (int i = 0; i < size<0, 2>(frag) / 2; ++i) {
+ cutlass::swap(frag(make_coord(_1{}, j, 2 * i), mi), frag(make_coord(_0{}, j, 2 * i + 1), mi));
+ }
+ }
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Fragment>
+CUTLASS_DEVICE void permute_output_fp8_Vcolmajor(Fragment &frag) {
+ // frag has shape ((2, 2, N / 8), MMA_M, MMA_N), each element is 16 bits
+ static_assert(decltype(size<0, 0>(frag))::value == 2);
+ static_assert(decltype(size<0, 1>(frag))::value == 2);
+ static_assert(decltype(stride<0, 0>(frag))::value == 1);
+ static_assert(sizeof(typename Fragment::value_type) == 2 || sizeof(typename Fragment::value_type) == 4);
+
+ int quad_idx = threadIdx.x % 4;
+ bool lane_03 = quad_idx == 0 || quad_idx == 3;
+
+ static constexpr int upper_map[4] = {0, 2, 3, 1};
+ // static constexpr int lower_map[4] = {2, 0, 1, 3};
+
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(frag); }
+ using type2 = std::conditional_t<sizeof(typename Fragment::value_type) == 2, uint32_t, uint64_t>;
+ Tensor frag_2 = group_modes<1, 3>(recast<type2>(frag)); // ((1, 2, N / 8), (MMA_M, MMA_N))
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print(frag); printf("\n"); print(frag_2); }
+ #pragma unroll
+ for (int mi = 0; mi < size<1>(frag_2); ++mi) {
+ #pragma unroll
+ for (int j = 0; j < size<0, 1>(frag_2); ++j) {
+ #pragma unroll
+ for (int i = 0; i < size<0, 2>(frag_2) / 2; ++i) {
+ type2 upper = frag_2(make_coord(_0{}, j, 2 * i), mi);
+ type2 lower = frag_2(make_coord(_0{}, j, 2 * i + 1), mi);
+ type2 upper0 = lane_03 ? upper : lower;
+ type2 lower0 = lane_03 ? lower : upper;
+ upper0 = __shfl_sync(uint32_t(-1), upper0, upper_map[quad_idx], 4);
+ // lower0 = __shfl_sync(uint32_t(-1), lower0, lower_map[quad_idx], 4);
+ lower0 = __shfl_sync(uint32_t(-1), lower0, upper_map[quad_idx] ^ 2, 4);
+ frag_2(make_coord(_0{}, j, 2 * i), mi) = lane_03 ? upper0 : lower0;
+ frag_2(make_coord(_0{}, j, 2 * i + 1), mi) = lane_03 ? lower0 : upper0;
+ }
+ }
+ }
+ // if (blockIdx.x == 0 && threadIdx.x == 128) { print_tensor(frag); }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template <typename Engine, typename Layout>
+CUTLASS_DEVICE void apply_softcap(Tensor<Engine, Layout> &tensor, float const softcap){
+ #pragma unroll
+ for (int i = 0; i < size(tensor); ++i) {
+ tensor(i) = cutlass::fast_tanh(tensor(i) * softcap);
+ }
+}
+
+template <typename Engine, typename Layout>
+CUTLASS_DEVICE auto calculate_dtanh(Tensor<Engine, Layout> &tensor){
+ Tensor out = make_fragment_like<float>(tensor);
+ #pragma unroll
+ for (int i = 0; i < size(tensor); ++i) {
+ out(i) = 1.f - (tensor(i) * tensor(i));
+ }
+ return out;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+CUTLASS_DEVICE
+int canonical_warp_group_idx_nosync() {
+ return threadIdx.x / cutlass::NumThreadsPerWarpGroup;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+} // namespace flash
diff --git a/src/solarwm/backends/minimax_h3/model.py b/src/solarwm/backends/minimax_h3/model.py
index 46a58b8..691c6e0 100644
--- a/src/solarwm/backends/minimax_h3/model.py
+++ b/src/solarwm/backends/minimax_h3/model.py
@@ -575,6 +575,10 @@ class SolarMiniMaxH3Transformer3DModel(H3AnyFlowConditioningMixin, MiniMaxH3Tran
attention_mask,
training=attn.training,
).transpose(1, 2)
+ elif getattr(attn.processor, "_attention_backend", None) == "solar_hopper":
+ from .hopper_attention import attention
+
+ attended = attention(query, key, value, attn_mask=attention_mask)
else:
attended = dispatch_attention_fn(
query,
diff --git a/src/solarwm/backends/minimax_h3/optional.py b/src/solarwm/backends/minimax_h3/optional.py
index 337be14..1763b15 100644
--- a/src/solarwm/backends/minimax_h3/optional.py
+++ b/src/solarwm/backends/minimax_h3/optional.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import os
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
@@ -121,6 +122,12 @@ def load_transformer(model_cfg: Mapping[str, Any], *, device: Any) -> H3RuntimeM
# (including Qwen token refinement and SGF score models) remains Flash.
requested_backend = str(model_cfg.get("attention_backend", "flash"))
transformer.set_attention_backend("flash" if requested_backend == "flex" else requested_backend)
+ if os.environ.get("SOLAR_HOPPER_ATTENTION") == "1":
+ from .hopper_attention import initialize
+
+ initialize()
+ for block in transformer.transformer_blocks:
+ block.attn.processor._attention_backend = "solar_hopper"
if model_cfg.get("transformer_device_map") is None:
transformer = transformer.to(device)
transformer.camera_intrinsics_mode = str(model_cfg.get("camera_intrinsics_mode", "wan_fixed"))