LLM inference in C/C++
C++ ggml
CUDA: prefer whole-tile FlashAttention scheduling for efficient two-stage kernels (#29435)
anujjd89651a7b205
diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuhindex a194918219..41d948335e 100644--- a/ggml/src/ggml-cuda/fattn-common.cuh+++ b/ggml/src/ggml-cuda/fattn-common.cuh@@ -981,7 +981,7 @@ template <int DV, int ncols1, int ncols2>void launch_fattn(ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared,const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const bool use_sparse,- const int warp_size = WARP_SIZE+ const int warp_size = WARP_SIZE, const bool async_kv_preload = false) {constexpr int ncols = ncols1 * ncols2;@@ -1114,7 +1114,8 @@ void launch_fattn(// Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped.// Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or// multiple sequences of possibly different lengths.- if (!use_sparse && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) {+ const bool scan_mask = !use_sparse && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1);+ if (scan_mask) {const int64_t s31 = mask->nb[1] / sizeof(half2);const int64_t s33 = mask->nb[3] / sizeof(half2);@@ -1142,10 +1143,16 @@ void launch_fattn(dim3 blocks_num;if (stream_k) {- auto should_use_stream_k = [](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) {+ // Stream-K splits the work before the mask scan is applied, so skipped KV tiles make the blocks uneven.+ const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK && async_kv_preload && scan_mask;++ auto should_use_stream_k = [prefer_whole_tiles](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) {const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks;const int tiles_efficiency_percent = 100 * ntiles_dst / (max_blocks*tiles_nwaves);+ if (prefer_whole_tiles && tiles_efficiency_percent >= 75) {+ return false;+ }if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_ADA_LOVELACE) {return true;}diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuhindex 9d221d23eb..7f0f6abfdb 100644--- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh+++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh@@ -2100,8 +2100,9 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml}}+ const bool async_kv_preload = nstages == 2 && !use_sparse;launch_fattn<DV, ncols1, ncols2>- (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host);+ (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host, async_kv_preload);}